2 Bacula(R) - The Network Backup Solution
4 Copyright (C) 2000-2016 Kern Sibbald
6 The original author of Bacula is Kern Sibbald, with contributions
7 from many others, a complete list can be found in the file AUTHORS.
9 You may use this file and others of this release according to the
10 license defined in the LICENSE file, which includes the Affero General
11 Public License, v3.0 ("AGPLv3") and some additional permissions and
12 terms pursuant to its AGPLv3 Section 7.
14 This notice must be preserved when any source code is
15 conveyed and/or propagated.
17 Bacula(R) is a registered trademark of Kern Sibbald.
21 * Bacula Director -- vbackup.c -- responsible for doing virtual
22 * backup jobs or in other words, consolidation or synthetic
25 * Kern Sibbald, July MMVIII
27 * Basic tasks done here:
28 * Open DB and create records for this job.
29 * Figure out what Jobs to copy.
30 * Open Message Channel with Storage daemon to tell him a job will be starting.
31 * Open connection with File daemon and pass him commands
33 * When the File daemon finishes the job, update the DB.
41 static const int dbglevel = 10;
43 static bool create_bootstrap_file(JCR *jcr, char *jobids);
44 void vbackup_cleanup(JCR *jcr, int TermCode);
47 * Called here before the job is run to do the job
50 bool do_vbackup_init(JCR *jcr)
54 * if the read pool has not been allocated yet due to the job
55 * being upgraded to a virtual full then allocate it now
57 if (!jcr->rpool_source)
58 jcr->rpool_source = get_pool_memory(PM_MESSAGE);
61 * Note, at this point, pool is the pool for this job. We
62 * transfer it to rpool (read pool), and a bit later,
63 * pool will be changed to point to the write pool,
64 * which comes from pool->NextPool.
66 jcr->rpool = jcr->pool; /* save read pool */
67 pm_strcpy(jcr->rpool_source, jcr->pool_source);
69 /* If pool storage specified, use it for virtual full */
70 copy_rstorage(jcr, jcr->pool->storage, _("Pool resource"));
72 Dmsg2(dbglevel, "Read pool=%s (From %s)\n", jcr->rpool->name(), jcr->rpool_source);
74 jcr->start_time = time(NULL);
75 jcr->jr.StartTime = jcr->start_time;
76 jcr->jr.JobLevel = L_FULL; /* we want this to appear as a Full backup */
77 if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) {
78 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db));
81 if (!apply_wstorage_overrides(jcr, jcr->pool)) {
85 Dmsg2(dbglevel, "Write pool=%s read rpool=%s\n", jcr->pool->name(), jcr->rpool->name());
91 * Do a virtual backup, which consolidates all previous backups into
92 * a sort of synthetic Full.
94 * Returns: false on failure
97 bool do_vbackup(JCR *jcr)
99 char level_computed = L_FULL;
106 Dmsg2(100, "rstorage=%p wstorage=%p\n", jcr->rstorage, jcr->wstorage);
107 Dmsg2(100, "Read store=%s, write store=%s\n",
108 ((STORE *)jcr->rstorage->first())->name(),
109 ((STORE *)jcr->wstorage->first())->name());
111 jcr->wasVirtualFull = true; /* remember where we came from */
113 /* Print Job Start message */
114 Jmsg(jcr, M_INFO, 0, _("Start Virtual Backup JobId %s, Job=%s\n"),
115 edit_uint64(jcr->JobId, ed1), jcr->Job);
116 if (!jcr->accurate) {
117 Jmsg(jcr, M_WARNING, 0,
118 _("This Job is not an Accurate backup so is not equivalent to a Full backup.\n"));
121 if (jcr->JobIds && *jcr->JobIds) {
124 POOL_MEM query(PM_MESSAGE);
126 memset(&jr, 0, sizeof(jr));
128 if (is_an_integer(jcr->JobIds)) {
129 /* Single JobId, so start the accurate code based on this id */
131 jr.JobId = str_to_int64(jcr->JobIds);
132 if (!db_get_job_record(jcr, jcr->db, &jr)) {
133 Jmsg(jcr, M_ERROR, 0,
134 _("Unable to get Job record for JobId=%s: ERR=%s\n"),
135 jcr->JobIds, db_strerror(jcr->db));
138 Jmsg(jcr, M_INFO,0,_("Selecting jobs to build the Full state at %s\n"),
141 jr.JobLevel = L_INCREMENTAL; /* Take Full+Diff+Incr */
142 db_get_accurate_jobids(jcr, jcr->db, &jr, &jobids);
144 } else if (sel.set_string(jcr->JobIds, true)) {
145 /* Found alljobid keyword */
146 if (jcr->use_all_JobIds) {
147 jobids.count = sel.size();
148 pm_strcpy(jobids.list, sel.get_expanded_list());
150 /* Need to apply some filter on the job name */
153 "SELECT JobId FROM Job "
154 "WHERE Job.Name = '%s' "
155 "AND Job.JobId IN (%s) "
156 "ORDER BY JobTDate ASC",
158 sel.get_expanded_list());
160 db_sql_query(jcr->db, query.c_str(), db_list_handler, &jobids);
163 if (jobids.count == 0) {
164 Jmsg(jcr, M_FATAL, 0, _("No valid Jobs found from user selection.\n"));
168 Jmsg(jcr, M_INFO, 0, _("Using user supplied JobIds=%s\n"),
173 "SELECT Level FROM Job "
174 "WHERE Job.JobId IN (%s) "
178 /* Will produce something like F,D,I or F,I */
179 db_sql_query(jcr->db, query.c_str(), db_list_handler, &status);
181 /* If no full found in the list, we build a "virtualdiff" or
184 if (strchr(status.list, L_FULL) == NULL) {
185 if (strchr(status.list, L_DIFFERENTIAL)) {
186 level_computed = L_DIFFERENTIAL;
187 Jmsg(jcr, M_INFO, 0, _("No previous Full found in list, "
188 "using Differential level\n"));
191 level_computed = L_INCREMENTAL;
192 Jmsg(jcr, M_INFO, 0, _("No previous Full found in list, "
193 "using Incremental level\n"));
198 } else { /* No argument provided */
199 jcr->jr.JobLevel = L_VIRTUAL_FULL;
200 db_get_accurate_jobids(jcr, jcr->db, &jcr->jr, &jobids);
201 Dmsg1(10, "Accurate jobids=%s\n", jobids.list);
204 if (jobids.count == 0) {
205 Jmsg(jcr, M_FATAL, 0, _("No previous Jobs found.\n"));
209 /* Full by default, or might be Incr/Diff when jobid= is used */
210 jcr->jr.JobLevel = level_computed;
213 * Now we find the last job that ran and store it's info in
214 * the previous_jr record. We will set our times to the
215 * values from that job so that anything changed after that
216 * time will be picked up on the next backup.
218 p = strrchr(jobids.list, ','); /* find last jobid */
224 memset(&jcr->previous_jr, 0, sizeof(jcr->previous_jr));
225 jcr->previous_jr.JobId = str_to_int64(p);
226 Dmsg1(10, "Previous JobId=%s\n", p);
227 if (!db_get_job_record(jcr, jcr->db, &jcr->previous_jr)) {
228 Jmsg(jcr, M_FATAL, 0, _("Error getting Job record for previous Job: ERR=%s"),
229 db_strerror(jcr->db));
233 if (!create_bootstrap_file(jcr, jobids.list)) {
234 Jmsg(jcr, M_FATAL, 0, _("Could not get or create the FileSet record.\n"));
239 * Open a message channel connection with the Storage
240 * daemon. This is to let him know that our client
241 * will be contacting him for a backup session.
244 Dmsg0(110, "Open connection with storage daemon\n");
245 jcr->setJobStatus(JS_WaitSD);
247 * Start conversation with Storage daemon
249 if (!connect_to_storage_daemon(jcr, 10, SDConnectTimeout, 1)) {
252 sd = jcr->store_bsock;
255 * Now start a job with the Storage daemon
257 if (!start_storage_daemon_job(jcr, jcr->rstorage, jcr->wstorage, /*send_bsr*/true)) {
260 Dmsg0(100, "Storage daemon connection OK\n");
263 * We re-update the job start record so that the start
264 * time is set after the run before job. This avoids
265 * that any files created by the run before job will
266 * be saved twice. They will be backed up in the current
267 * job, but not in the next one unless they are changed.
268 * Without this, they will be backed up in this job and
269 * in the next job run because in that case, their date
270 * is after the start of this run.
272 jcr->start_time = time(NULL);
273 jcr->jr.StartTime = jcr->start_time;
274 jcr->jr.JobTDate = jcr->start_time;
275 jcr->setJobStatus(JS_Running);
277 /* Add the following when support for base jobs is added to virtual full */
278 //jcr->HasBase = jcr->job->base != NULL;
279 //jcr->jr.HasBase = jcr->HasBase;
281 /* Update job start record */
282 if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) {
283 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db));
287 /* Declare the job started to start the MaxRunTime check */
288 jcr->setJobStarted();
291 * Start the job prior to starting the message thread below
292 * to avoid two threads from using the BSOCK structure at
295 if (!sd->fsend("run")) {
300 * Now start a Storage daemon message thread
302 if (!start_storage_daemon_message_thread(jcr)) {
306 jcr->setJobStatus(JS_Running);
308 /* Pickup Job termination data */
309 /* Note, the SD stores in jcr->JobFiles/ReadBytes/JobBytes/JobErrors */
310 wait_for_storage_daemon_termination(jcr);
311 jcr->setJobStatus(jcr->SDJobStatus);
312 db_write_batch_file_records(jcr); /* used by bulk batch file insert */
313 if (jcr->JobStatus != JS_Terminated) {
317 vbackup_cleanup(jcr, jcr->JobStatus);
323 * Release resources allocated during backup.
325 void vbackup_cleanup(JCR *jcr, int TermCode)
327 char sdt[50], edt[50], schedt[50];
328 char ec1[30], ec3[30], ec4[30], compress[50];
329 char ec7[30], ec8[30], elapsed[50];
330 char term_code[100], sd_term_msg[100];
331 const char *term_msg;
332 int msg_type = M_INFO;
335 double kbps, compression;
337 POOL_MEM query(PM_MESSAGE);
339 Dmsg2(100, "Enter backup_cleanup %d %c\n", TermCode, TermCode);
340 memset(&cr, 0, sizeof(cr));
342 jcr->setJobLevel(L_FULL); /* we want this to appear as a Full backup */
343 jcr->jr.JobLevel = L_FULL; /* we want this to appear as a Full backup */
344 jcr->JobFiles = jcr->SDJobFiles;
345 jcr->JobBytes = jcr->SDJobBytes;
346 update_job_end(jcr, TermCode);
348 /* Update final items to set them to the previous job's values */
349 Mmsg(query, "UPDATE Job SET StartTime='%s',EndTime='%s',"
350 "JobTDate=%s WHERE JobId=%s",
351 jcr->previous_jr.cStartTime, jcr->previous_jr.cEndTime,
352 edit_uint64(jcr->previous_jr.JobTDate, ec1),
353 edit_uint64(jcr->JobId, ec3));
354 db_sql_query(jcr->db, query.c_str(), NULL, NULL);
356 /* Get the fully updated job record */
357 if (!db_get_job_record(jcr, jcr->db, &jcr->jr)) {
358 Jmsg(jcr, M_WARNING, 0, _("Error getting Job record for Job report: ERR=%s"),
359 db_strerror(jcr->db));
360 jcr->setJobStatus(JS_ErrorTerminated);
363 bstrncpy(cr.Name, jcr->client->name(), sizeof(cr.Name));
364 if (!db_get_client_record(jcr, jcr->db, &cr)) {
365 Jmsg(jcr, M_WARNING, 0, _("Error getting Client record for Job report: ERR=%s"),
366 db_strerror(jcr->db));
369 bstrncpy(mr.VolumeName, jcr->VolumeName, sizeof(mr.VolumeName));
370 if (!db_get_media_record(jcr, jcr->db, &mr)) {
371 Jmsg(jcr, M_WARNING, 0, _("Error getting Media record for Volume \"%s\": ERR=%s"),
372 mr.VolumeName, db_strerror(jcr->db));
373 jcr->setJobStatus(JS_ErrorTerminated);
376 update_bootstrap_file(jcr);
378 switch (jcr->JobStatus) {
380 if (jcr->JobErrors || jcr->SDErrors) {
381 term_msg = _("Backup OK -- with warnings");
383 term_msg = _("Backup OK");
387 case JS_ErrorTerminated:
388 term_msg = _("*** Backup Error ***");
389 msg_type = M_ERROR; /* Generate error message */
390 if (jcr->store_bsock) {
391 jcr->store_bsock->signal(BNET_TERMINATE);
392 if (jcr->SD_msg_chan_started) {
393 pthread_cancel(jcr->SD_msg_chan);
398 term_msg = _("Backup Canceled");
399 if (jcr->store_bsock) {
400 jcr->store_bsock->signal(BNET_TERMINATE);
401 if (jcr->SD_msg_chan_started) {
402 pthread_cancel(jcr->SD_msg_chan);
407 term_msg = term_code;
408 sprintf(term_code, _("Inappropriate term code: %c\n"), jcr->JobStatus);
411 bstrftimes(schedt, sizeof(schedt), jcr->jr.SchedTime);
412 bstrftimes(sdt, sizeof(sdt), jcr->jr.StartTime);
413 bstrftimes(edt, sizeof(edt), jcr->jr.EndTime);
414 RunTime = jcr->jr.EndTime - jcr->jr.StartTime;
418 kbps = ((double)jcr->jr.JobBytes) / (1000.0 * (double)RunTime);
420 if (!db_get_job_volume_names(jcr, jcr->db, jcr->jr.JobId, &jcr->VolumeName)) {
422 * Note, if the job has erred, most likely it did not write any
423 * tape, so suppress this "error" message since in that case
424 * it is normal. Or look at it the other way, only for a
425 * normal exit should we complain about this error.
427 if (jcr->JobStatus == JS_Terminated && jcr->jr.JobBytes) {
428 Jmsg(jcr, M_ERROR, 0, "%s", db_strerror(jcr->db));
430 jcr->VolumeName[0] = 0; /* none */
433 if (jcr->ReadBytes == 0) {
434 bstrncpy(compress, "None", sizeof(compress));
436 compression = (double)100 - 100.0 * ((double)jcr->JobBytes / (double)jcr->ReadBytes);
437 if (compression < 0.5) {
438 bstrncpy(compress, "None", sizeof(compress));
440 bsnprintf(compress, sizeof(compress), "%.1f %%", compression);
443 jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg));
445 Jmsg(jcr, msg_type, 0, _("%s %s %s (%s):\n"
446 " Build OS: %s %s %s\n"
449 " Backup Level: Virtual Full\n"
450 " Client: \"%s\" %s\n"
451 " FileSet: \"%s\" %s\n"
452 " Pool: \"%s\" (From %s)\n"
453 " Catalog: \"%s\" (From %s)\n"
454 " Storage: \"%s\" (From %s)\n"
455 " Scheduled time: %s\n"
458 " Elapsed time: %s\n"
460 " SD Files Written: %s\n"
461 " SD Bytes Written: %s (%sB)\n"
463 " Volume name(s): %s\n"
464 " Volume Session Id: %d\n"
465 " Volume Session Time: %d\n"
466 " Last Volume Bytes: %s (%sB)\n"
468 " SD termination status: %s\n"
469 " Termination: %s\n\n"),
470 BACULA, my_name, VERSION, LSMDATE,
471 HOST_OS, DISTNAME, DISTVER,
474 jcr->client->name(), cr.Uname,
475 jcr->fileset->name(), jcr->FSCreateTime,
476 jcr->pool->name(), jcr->pool_source,
477 jcr->catalog->name(), jcr->catalog_source,
478 jcr->wstore->name(), jcr->wstore_source,
482 edit_utime(RunTime, elapsed, sizeof(elapsed)),
484 edit_uint64_with_commas(jcr->jr.JobFiles, ec1),
485 edit_uint64_with_commas(jcr->jr.JobBytes, ec3),
486 edit_uint64_with_suffix(jcr->jr.JobBytes, ec4),
491 edit_uint64_with_commas(mr.VolBytes, ec7),
492 edit_uint64_with_suffix(mr.VolBytes, ec8),
497 Dmsg0(100, "Leave vbackup_cleanup()\n");
501 * This callback routine is responsible for inserting the
502 * items it gets into the bootstrap structure. For each JobId selected
503 * this routine is called once for each file. We do not allow
504 * duplicate filenames, but instead keep the info from the most
505 * recent file entered (i.e. the JobIds are assumed to be sorted)
507 * See uar_sel_files in sql_cmds.c for query that calls us.
508 * row[0]=Path, row[1]=Filename, row[2]=FileIndex
509 * row[3]=JobId row[4]=LStat
511 int insert_bootstrap_handler(void *ctx, int num_fields, char **row)
515 RBSR *bsr = (RBSR *)ctx;
517 JobId = str_to_int64(row[3]);
518 FileIndex = str_to_int64(row[2]);
519 add_findex(bsr, JobId, FileIndex);
524 static bool create_bootstrap_file(JCR *jcr, char *jobids)
529 memset(&rx, 0, sizeof(rx));
531 ua = new_ua_context(jcr);
534 #define new_get_file_list
535 #ifdef new_get_file_list
536 if (!db_open_batch_connexion(jcr, jcr->db)) {
537 Jmsg0(jcr, M_FATAL, 0, "Can't get batch sql connexion");
541 if (!db_get_file_list(jcr, jcr->db_batch, jobids, false /* don't use md5 */,
542 true /* use delta */,
543 insert_bootstrap_handler, (void *)rx.bsr))
545 Jmsg(jcr, M_ERROR, 0, "%s", db_strerror(jcr->db_batch));
549 JobId_t JobId, last_JobId = 0;
550 rx.query = get_pool_memory(PM_MESSAGE);
551 for (p=rx.JobIds; get_next_jobid_from_list(&p, &JobId) > 0; ) {
554 if (JobId == last_JobId) {
555 continue; /* eliminate duplicate JobIds */
559 * Find files for this JobId and insert them in the tree
561 Mmsg(rx.query, uar_sel_files, edit_int64(JobId, ed1));
562 Dmsg1(100, "uar_sel_files=%s\n", rx.query);
563 if (!db_sql_query(ua->db, rx.query, insert_bootstrap_handler, (void *)rx.bsr)) {
564 Jmsg(jcr, M_ERROR, 0, "%s", db_strerror(ua->db));
566 free_pool_memory(rx.query);
571 complete_bsr(ua, rx.bsr);
572 jcr->ExpectedFiles = write_bsr_file(ua, rx);
573 Jmsg(jcr, M_INFO, 0, _("Found %d files to consolidate into Virtual Full.\n"),
577 return jcr->ExpectedFiles==0?false:true;