2 Bacula(R) - The Network Backup Solution
4 Copyright (C) 2000-2017 Kern Sibbald
6 The original author of Bacula is Kern Sibbald, with contributions
7 from many others, a complete list can be found in the file AUTHORS.
9 You may use this file and others of this release according to the
10 license defined in the LICENSE file, which includes the Affero General
11 Public License, v3.0 ("AGPLv3") and some additional permissions and
12 terms pursuant to its AGPLv3 Section 7.
14 This notice must be preserved when any source code is
15 conveyed and/or propagated.
17 Bacula(R) is a registered trademark of Kern Sibbald.
20 * Bacula Director -- vbackup.c -- responsible for doing virtual
21 * backup jobs or in other words, consolidation or synthetic
22 * backups. No connection to the Client is made.
24 * Kern Sibbald, July MMVIII
26 * Basic tasks done here:
27 * Open DB and create records for this job.
28 * Figure out what Jobs to copy.
29 * Open Message Channel with Storage daemon to tell him a
30 * job will be starting.
31 * Connect to the storage daemon and run the job.
38 static const int dbglevel = 10;
40 static bool create_bootstrap_file(JCR *jcr, char *jobids);
41 void vbackup_cleanup(JCR *jcr, int TermCode);
44 * Called here before the job is run to do the job
47 bool do_vbackup_init(JCR *jcr)
49 if (!get_or_create_fileset_record(jcr)) {
50 Dmsg1(dbglevel, "JobId=%d no FileSet\n", (int)jcr->JobId);
54 apply_pool_overrides(jcr);
56 if (!allow_duplicate_job(jcr)) {
61 * If the read pool has not been allocated yet due to the job
62 * being upgraded to a virtual full then allocate it now
64 if (!jcr->rpool_source) {
65 jcr->rpool_source = get_pool_memory(PM_MESSAGE);
67 jcr->jr.PoolId = get_or_create_pool_record(jcr, jcr->pool->name());
68 if (jcr->jr.PoolId == 0) {
69 Dmsg1(dbglevel, "JobId=%d no PoolId\n", (int)jcr->JobId);
70 Jmsg(jcr, M_FATAL, 0, _("Could not get or create a Pool record.\n"));
74 * Note, at this point, pool is the pool for this job. We
75 * transfer it to rpool (read pool), and a bit later,
76 * pool will be changed to point to the write pool,
77 * which comes from pool->NextPool.
79 jcr->rpool = jcr->pool; /* save read pool */
80 pm_strcpy(jcr->rpool_source, jcr->pool_source);
82 /* If pool storage specified, use it for virtual full */
83 copy_rstorage(jcr, jcr->pool->storage, _("Pool resource"));
85 Dmsg2(dbglevel, "Read pool=%s (From %s)\n", jcr->rpool->name(), jcr->rpool_source);
87 jcr->start_time = time(NULL);
88 jcr->jr.StartTime = jcr->start_time;
89 jcr->jr.JobLevel = L_FULL; /* we want this to appear as a Full backup */
90 if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) {
91 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db));
94 if (!apply_wstorage_overrides(jcr, jcr->pool)) {
98 Dmsg2(dbglevel, "Write pool=%s read rpool=%s\n", jcr->pool->name(), jcr->rpool->name());
104 * Do a virtual backup, which consolidates all previous backups into
105 * a sort of synthetic Full.
107 * Returns: false on failure
110 bool do_vbackup(JCR *jcr)
112 char level_computed = L_FULL;
120 Dmsg2(100, "rstorage=%p wstorage=%p\n", jcr->rstorage, jcr->wstorage);
121 Dmsg2(100, "Read store=%s, write store=%s\n",
122 ((STORE *)jcr->rstorage->first())->name(),
123 ((STORE *)jcr->wstorage->first())->name());
125 jcr->wasVirtualFull = true; /* remember where we came from */
127 /* Print Job Start message */
128 Jmsg(jcr, M_INFO, 0, _("Start Virtual Backup JobId %s, Job=%s\n"),
129 edit_uint64(jcr->JobId, ed1), jcr->Job);
130 if (!jcr->accurate) {
131 Jmsg(jcr, M_WARNING, 0,
132 _("This Job is not an Accurate backup so is not equivalent to a Full backup.\n"));
135 if (jcr->JobIds && *jcr->JobIds) {
138 POOL_MEM query(PM_MESSAGE);
140 memset(&jr, 0, sizeof(jr));
142 if (is_an_integer(jcr->JobIds)) {
143 /* Single JobId, so start the accurate code based on this id */
145 jr.JobId = str_to_int64(jcr->JobIds);
146 if (!db_get_job_record(jcr, jcr->db, &jr)) {
147 Jmsg(jcr, M_ERROR, 0,
148 _("Unable to get Job record for JobId=%s: ERR=%s\n"),
149 jcr->JobIds, db_strerror(jcr->db));
152 Jmsg(jcr, M_INFO,0,_("Selecting jobs to build the Full state at %s\n"),
155 jr.JobLevel = L_INCREMENTAL; /* Take Full+Diff+Incr */
156 db_get_accurate_jobids(jcr, jcr->db, &jr, &jobids);
158 } else if (sel.set_string(jcr->JobIds, true)) {
159 /* Found alljobid keyword */
160 if (jcr->use_all_JobIds) {
161 jobids.count = sel.size();
162 pm_strcpy(jobids.list, sel.get_expanded_list());
164 /* Need to apply some filter on the job name */
167 "SELECT JobId FROM Job "
168 "WHERE Job.Name = '%s' "
169 "AND Job.JobId IN (%s) "
170 "ORDER BY JobTDate ASC",
172 sel.get_expanded_list());
174 db_sql_query(jcr->db, query.c_str(), db_list_handler, &jobids);
177 if (jobids.count == 0) {
178 Jmsg(jcr, M_FATAL, 0, _("No valid Jobs found from user selection.\n"));
182 Jmsg(jcr, M_INFO, 0, _("Using user supplied JobIds=%s\n"),
187 "SELECT Level FROM Job "
188 "WHERE Job.JobId IN (%s) "
192 /* Will produce something like F,D,I or F,I */
193 db_sql_query(jcr->db, query.c_str(), db_list_handler, &status);
195 /* If no full found in the list, we build a "virtualdiff" or
198 if (strchr(status.list, L_FULL) == NULL) {
199 if (strchr(status.list, L_DIFFERENTIAL)) {
200 level_computed = L_DIFFERENTIAL;
201 Jmsg(jcr, M_INFO, 0, _("No previous Full found in list, "
202 "using Differential level\n"));
205 level_computed = L_INCREMENTAL;
206 Jmsg(jcr, M_INFO, 0, _("No previous Full found in list, "
207 "using Incremental level\n"));
212 } else { /* No argument provided */
213 jcr->jr.JobLevel = L_VIRTUAL_FULL;
214 db_get_accurate_jobids(jcr, jcr->db, &jcr->jr, &jobids);
215 Dmsg1(10, "Accurate jobids=%s\n", jobids.list);
218 if (jobids.count == 0) {
219 Jmsg(jcr, M_FATAL, 0, _("No previous Jobs found.\n"));
222 jobids.count -= jcr->job->BackupsToKeep;
223 if (jobids.count <= 0) {
224 Jmsg(jcr, M_WARNING, 0, _("Insufficient Backups to Keep.\n"));
227 if (jobids.count == 1) {
228 Jmsg(jcr, M_WARNING, 0, _("Only one Job found. Consolidation not needed.\n"));
232 /* Remove number of JobIds we want to keep */
233 for (int i=0; i < (int)jcr->job->BackupsToKeep; i++) {
234 p = strrchr(jobids.list, ','); /* find last jobid */
242 /* Full by default, or might be Incr/Diff when jobid= is used */
243 jcr->jr.JobLevel = level_computed;
245 Jmsg(jcr, M_INFO, 0, "Consolidating JobIds=%s\n", jobids.list);
248 * Now we find the last job that ran and store it's info in
249 * the previous_jr record. We will set our times to the
250 * values from that job so that anything changed after that
251 * time will be picked up on the next backup.
253 p = strrchr(jobids.list, ','); /* find last jobid */
259 memset(&jcr->previous_jr, 0, sizeof(jcr->previous_jr));
260 jcr->previous_jr.JobId = str_to_int64(p);
261 Dmsg1(10, "Previous JobId=%s\n", p);
262 if (!db_get_job_record(jcr, jcr->db, &jcr->previous_jr)) {
263 Jmsg(jcr, M_FATAL, 0, _("Error getting Job record for previous Job: ERR=%s"),
264 db_strerror(jcr->db));
268 if (!create_bootstrap_file(jcr, jobids.list)) {
269 Jmsg(jcr, M_FATAL, 0, _("Could not get or create the FileSet record.\n"));
274 * Open a message channel connection with the Storage
275 * daemon. This is to let him know that our client
276 * will be contacting him for a backup session.
279 Dmsg0(110, "Open connection with storage daemon\n");
280 jcr->setJobStatus(JS_WaitSD);
282 * Start conversation with Storage daemon
284 if (!connect_to_storage_daemon(jcr, 10, SDConnectTimeout, 1)) {
287 sd = jcr->store_bsock;
290 * Now start a job with the Storage daemon
292 if (!start_storage_daemon_job(jcr, jcr->rstorage, jcr->wstorage, /*send_bsr*/true)) {
295 Dmsg0(100, "Storage daemon connection OK\n");
298 * We re-update the job start record so that the start
299 * time is set after the run before job. This avoids
300 * that any files created by the run before job will
301 * be saved twice. They will be backed up in the current
302 * job, but not in the next one unless they are changed.
303 * Without this, they will be backed up in this job and
304 * in the next job run because in that case, their date
305 * is after the start of this run.
307 jcr->start_time = time(NULL);
308 jcr->jr.StartTime = jcr->start_time;
309 jcr->jr.JobTDate = jcr->start_time;
310 jcr->setJobStatus(JS_Running);
312 /* Update job start record */
313 if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) {
314 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db));
318 /* Declare the job started to start the MaxRunTime check */
319 jcr->setJobStarted();
322 * Start the job prior to starting the message thread below
323 * to avoid two threads from using the BSOCK structure at
326 if (!sd->fsend("run")) {
331 * Now start a Storage daemon message thread
333 if (!start_storage_daemon_message_thread(jcr)) {
337 jcr->setJobStatus(JS_Running);
339 /* Pickup Job termination data */
340 /* Note, the SD stores in jcr->JobFiles/ReadBytes/JobBytes/JobErrors */
341 wait_for_storage_daemon_termination(jcr);
342 jcr->setJobStatus(jcr->SDJobStatus);
343 db_write_batch_file_records(jcr); /* used by bulk batch file insert */
344 if (jcr->JobStatus != JS_Terminated) {
347 if (jcr->job->DeleteConsolidatedJobs) {
348 ua = new_ua_context(jcr);
349 purge_jobs_from_catalog(ua, jobids.list);
351 Jmsg(jcr, M_INFO, 0, _("Deleted consolidated JobIds=%s\n"), jobids.list);
354 vbackup_cleanup(jcr, jcr->JobStatus);
360 * Release resources allocated during backup.
362 void vbackup_cleanup(JCR *jcr, int TermCode)
364 char sdt[50], edt[50], schedt[50];
365 char ec1[30], ec3[30], ec4[30], compress[50];
366 char ec7[30], ec8[30], elapsed[50];
367 char term_code[100], sd_term_msg[100];
368 const char *term_msg;
369 int msg_type = M_INFO;
372 double kbps, compression;
374 POOL_MEM query(PM_MESSAGE);
376 Dmsg2(100, "Enter vbackup_cleanup %d %c\n", TermCode, TermCode);
377 memset(&cr, 0, sizeof(cr));
379 jcr->jr.JobLevel = L_FULL; /* we want this to appear as a Full backup */
380 jcr->JobFiles = jcr->SDJobFiles;
381 jcr->JobBytes = jcr->SDJobBytes;
382 update_job_end(jcr, TermCode);
384 /* Update final items to set them to the previous job's values */
385 Mmsg(query, "UPDATE Job SET StartTime='%s',EndTime='%s',"
386 "JobTDate=%s WHERE JobId=%s",
387 jcr->previous_jr.cStartTime, jcr->previous_jr.cEndTime,
388 edit_uint64(jcr->previous_jr.JobTDate, ec1),
389 edit_uint64(jcr->JobId, ec3));
390 db_sql_query(jcr->db, query.c_str(), NULL, NULL);
392 /* Get the fully updated job record */
393 if (!db_get_job_record(jcr, jcr->db, &jcr->jr)) {
394 Jmsg(jcr, M_WARNING, 0, _("Error getting Job record for Job report: ERR=%s"),
395 db_strerror(jcr->db));
396 jcr->setJobStatus(JS_ErrorTerminated);
399 bstrncpy(cr.Name, jcr->client->name(), sizeof(cr.Name));
400 if (!db_get_client_record(jcr, jcr->db, &cr)) {
401 Jmsg(jcr, M_WARNING, 0, _("Error getting Client record for Job report: ERR=%s"),
402 db_strerror(jcr->db));
405 bstrncpy(mr.VolumeName, jcr->VolumeName, sizeof(mr.VolumeName));
406 if (!db_get_media_record(jcr, jcr->db, &mr)) {
407 Jmsg(jcr, M_WARNING, 0, _("Error getting Media record for Volume \"%s\": ERR=%s"),
408 mr.VolumeName, db_strerror(jcr->db));
409 jcr->setJobStatus(JS_ErrorTerminated);
412 update_bootstrap_file(jcr);
414 switch (jcr->JobStatus) {
416 if (jcr->JobErrors || jcr->SDErrors) {
417 term_msg = _("Backup OK -- with warnings");
419 term_msg = _("Backup OK");
423 case JS_ErrorTerminated:
424 term_msg = _("*** Backup Error ***");
425 msg_type = M_ERROR; /* Generate error message */
426 if (jcr->store_bsock) {
427 jcr->store_bsock->signal(BNET_TERMINATE);
428 if (jcr->SD_msg_chan_started) {
429 pthread_cancel(jcr->SD_msg_chan);
434 term_msg = _("Backup Canceled");
435 if (jcr->store_bsock) {
436 jcr->store_bsock->signal(BNET_TERMINATE);
437 if (jcr->SD_msg_chan_started) {
438 pthread_cancel(jcr->SD_msg_chan);
443 term_msg = term_code;
444 sprintf(term_code, _("Inappropriate term code: %c\n"), jcr->JobStatus);
447 bstrftimes(schedt, sizeof(schedt), jcr->jr.SchedTime);
448 bstrftimes(sdt, sizeof(sdt), jcr->jr.StartTime);
449 bstrftimes(edt, sizeof(edt), jcr->jr.EndTime);
450 RunTime = jcr->jr.EndTime - jcr->jr.StartTime;
454 kbps = ((double)jcr->jr.JobBytes) / (1000.0 * (double)RunTime);
455 if (!db_get_job_volume_names(jcr, jcr->db, jcr->jr.JobId, &jcr->VolumeName)) {
457 * Note, if the job has erred, most likely it did not write any
458 * tape, so suppress this "error" message since in that case
459 * it is normal. Or look at it the other way, only for a
460 * normal exit should we complain about this error.
462 if (jcr->JobStatus == JS_Terminated && jcr->jr.JobBytes) {
463 Jmsg(jcr, M_ERROR, 0, "%s", db_strerror(jcr->db));
465 jcr->VolumeName[0] = 0; /* none */
468 if (jcr->ReadBytes == 0) {
469 bstrncpy(compress, "None", sizeof(compress));
471 compression = (double)100 - 100.0 * ((double)jcr->JobBytes / (double)jcr->ReadBytes);
472 if (compression < 0.5) {
473 bstrncpy(compress, "None", sizeof(compress));
475 bsnprintf(compress, sizeof(compress), "%.1f %%", compression);
478 jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg));
480 Jmsg(jcr, msg_type, 0, _("%s %s %s (%s):\n"
481 " Build OS: %s %s %s\n"
484 " Backup Level: Virtual Full\n"
485 " Client: \"%s\" %s\n"
486 " FileSet: \"%s\" %s\n"
487 " Pool: \"%s\" (From %s)\n"
488 " Catalog: \"%s\" (From %s)\n"
489 " Storage: \"%s\" (From %s)\n"
490 " Scheduled time: %s\n"
493 " Elapsed time: %s\n"
495 " SD Files Written: %s\n"
496 " SD Bytes Written: %s (%sB)\n"
498 " Volume name(s): %s\n"
499 " Volume Session Id: %d\n"
500 " Volume Session Time: %d\n"
501 " Last Volume Bytes: %s (%sB)\n"
503 " SD termination status: %s\n"
504 " Termination: %s\n\n"),
505 BACULA, my_name, VERSION, LSMDATE,
506 HOST_OS, DISTNAME, DISTVER,
509 jcr->client->name(), cr.Uname,
510 jcr->fileset->name(), jcr->FSCreateTime,
511 jcr->pool->name(), jcr->pool_source,
512 jcr->catalog->name(), jcr->catalog_source,
513 jcr->wstore->name(), jcr->wstore_source,
517 edit_utime(RunTime, elapsed, sizeof(elapsed)),
519 edit_uint64_with_commas(jcr->jr.JobFiles, ec1),
520 edit_uint64_with_commas(jcr->jr.JobBytes, ec3),
521 edit_uint64_with_suffix(jcr->jr.JobBytes, ec4),
526 edit_uint64_with_commas(mr.VolBytes, ec7),
527 edit_uint64_with_suffix(mr.VolBytes, ec8),
532 Dmsg0(100, "Leave vbackup_cleanup()\n");
536 * This callback routine is responsible for inserting the
537 * items it gets into the bootstrap structure. For each JobId selected
538 * this routine is called once for each file. We do not allow
539 * duplicate filenames, but instead keep the info from the most
540 * recent file entered (i.e. the JobIds are assumed to be sorted)
542 * See uar_sel_files in sql_cmds.c for query that calls us.
543 * row[0]=Path, row[1]=Filename, row[2]=FileIndex
544 * row[3]=JobId row[4]=LStat
546 int insert_bootstrap_handler(void *ctx, int num_fields, char **row)
550 rblist *bsr_list = (rblist *)ctx;
552 JobId = str_to_int64(row[3]);
553 FileIndex = str_to_int64(row[2]);
554 add_findex(bsr_list, JobId, FileIndex);
559 static bool create_bootstrap_file(JCR *jcr, char *jobids)
565 memset(&rx, 0, sizeof(rx));
566 rx.bsr_list = New(rblist(bsr, &bsr->link));
567 ua = new_ua_context(jcr);
570 #define new_get_file_list
571 #ifdef new_get_file_list
572 if (!db_open_batch_connexion(jcr, jcr->db)) {
573 Jmsg0(jcr, M_FATAL, 0, "Can't get batch sql connexion");
577 if (!db_get_file_list(jcr, jcr->db_batch, jobids, false /* don't use md5 */,
578 true /* use delta */,
579 insert_bootstrap_handler, (void *)rx.bsr_list))
581 Jmsg(jcr, M_ERROR, 0, "%s", db_strerror(jcr->db_batch));
585 JobId_t JobId, last_JobId = 0;
586 rx.query = get_pool_memory(PM_MESSAGE);
587 for (p=rx.JobIds; get_next_jobid_from_list(&p, &JobId) > 0; ) {
590 if (JobId == last_JobId) {
591 continue; /* eliminate duplicate JobIds */
595 * Find files for this JobId and insert them in the tree
597 Mmsg(rx.query, uar_sel_files, edit_int64(JobId, ed1));
598 Dmsg1(100, "uar_sel_files=%s\n", rx.query);
599 if (!db_sql_query(ua->db, rx.query, insert_bootstrap_handler, (void *)rx.bsr_list)) {
600 Jmsg(jcr, M_ERROR, 0, "%s", db_strerror(ua->db));
602 free_pool_memory(rx.query);
607 complete_bsr(ua, rx.bsr_list);
608 jcr->ExpectedFiles = write_bsr_file(ua, rx);
609 if (chk_dbglvl(10)) {
610 Pmsg1(000, "Found %d files to consolidate.\n", jcr->ExpectedFiles);
612 Jmsg(jcr, M_INFO, 0, _("Found %d files to consolidate into Virtual Full.\n"),
615 free_bsr(rx.bsr_list);
616 return jcr->ExpectedFiles==0?false:true;