2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2008 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of John Walker.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
30 * Bacula Director -- verify.c -- responsible for running file verification
32 * Kern Sibbald, October MM
34 * Basic tasks done here:
36 * Open connection with File daemon and pass him commands
38 * When the File daemon sends the attributes, compare them to
47 #include "findlib/find.h"
49 /* Commands sent to File daemon */
50 static char verifycmd[] = "verify level=%s\n";
51 static char storaddr[] = "storage address=%s port=%d ssl=0\n";
53 /* Responses received from File daemon */
54 static char OKverify[] = "2000 OK verify\n";
55 static char OKstore[] = "2000 OK storage\n";
56 static char OKbootstrap[] = "2000 OK bootstrap\n";
58 /* Forward referenced functions */
59 static void prt_fname(JCR *jcr);
60 static int missing_handler(void *ctx, int num_fields, char **row);
64 * Called here before the job is run to do the job
67 bool do_verify_init(JCR *jcr)
69 if (!allow_duplicate_job(jcr)) {
77 * Do a verification of the specified files against the Catlaog
79 * Returns: false on failure
82 bool do_verify(JCR *jcr)
89 JobId_t verify_jobid = 0;
92 free_wstorage(jcr); /* we don't write */
94 memset(&jcr->previous_jr, 0, sizeof(jcr->previous_jr));
97 * Find JobId of last job that ran. Note, we do this when
98 * the job actually starts running, not at schedule time,
99 * so that we find the last job that terminated before
100 * this job runs rather than before it is scheduled. This
101 * permits scheduling a Backup and Verify at the same time,
102 * but with the Verify at a lower priority.
104 * For VERIFY_CATALOG we want the JobId of the last INIT.
105 * For VERIFY_VOLUME_TO_CATALOG, we want the JobId of the
108 if (jcr->JobLevel == L_VERIFY_CATALOG ||
109 jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG ||
110 jcr->JobLevel == L_VERIFY_DISK_TO_CATALOG) {
111 memcpy(&jr, &jcr->jr, sizeof(jr));
112 if (jcr->verify_job &&
113 (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG ||
114 jcr->JobLevel == L_VERIFY_DISK_TO_CATALOG)) {
115 Name = jcr->verify_job->name();
119 Dmsg1(100, "find last jobid for: %s\n", NPRT(Name));
120 if (!db_find_last_jobid(jcr, jcr->db, Name, &jr)) {
121 if (jcr->JobLevel == L_VERIFY_CATALOG) {
122 Jmsg(jcr, M_FATAL, 0, _(
123 "Unable to find JobId of previous InitCatalog Job.\n"
124 "Please run a Verify with Level=InitCatalog before\n"
125 "running the current Job.\n"));
127 Jmsg(jcr, M_FATAL, 0, _(
128 "Unable to find JobId of previous Job for this client.\n"));
132 verify_jobid = jr.JobId;
133 Dmsg1(100, "Last full jobid=%d\n", verify_jobid);
136 * Now get the job record for the previous backup that interests
137 * us. We use the verify_jobid that we found above.
139 if (jcr->JobLevel == L_VERIFY_CATALOG ||
140 jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG ||
141 jcr->JobLevel == L_VERIFY_DISK_TO_CATALOG) {
142 jcr->previous_jr.JobId = verify_jobid;
143 if (!db_get_job_record(jcr, jcr->db, &jcr->previous_jr)) {
144 Jmsg(jcr, M_FATAL, 0, _("Could not get job record for previous Job. ERR=%s"),
145 db_strerror(jcr->db));
148 if (jcr->previous_jr.JobStatus != 'T') {
149 Jmsg(jcr, M_FATAL, 0, _("Last Job %d did not terminate normally. JobStatus=%c\n"),
150 verify_jobid, jcr->previous_jr.JobStatus);
153 Jmsg(jcr, M_INFO, 0, _("Verifying against JobId=%d Job=%s\n"),
154 jcr->previous_jr.JobId, jcr->previous_jr.Job);
158 * If we are verifying a Volume, we need the Storage
159 * daemon, so open a connection, otherwise, just
160 * create a dummy authorization key (passed to
161 * File daemon but not used).
163 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG) {
164 if (!create_restore_bootstrap_file(jcr)) {
168 jcr->sd_auth_key = bstrdup("dummy"); /* dummy Storage daemon key */
171 if (jcr->JobLevel == L_VERIFY_DISK_TO_CATALOG && jcr->verify_job) {
172 jcr->fileset = jcr->verify_job->fileset;
174 Dmsg2(100, "ClientId=%u JobLevel=%c\n", jcr->previous_jr.ClientId, jcr->JobLevel);
176 if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) {
177 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db));
181 /* Print Job Start message */
182 Jmsg(jcr, M_INFO, 0, _("Start Verify JobId=%s Level=%s Job=%s\n"),
183 edit_uint64(jcr->JobId, ed1), level_to_str(jcr->JobLevel), jcr->Job);
185 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG) {
187 * Start conversation with Storage daemon
189 set_jcr_job_status(jcr, JS_Blocked);
190 if (!connect_to_storage_daemon(jcr, 10, SDConnectTimeout, 1)) {
194 * Now start a job with the Storage daemon
196 if (!start_storage_daemon_job(jcr, jcr->rstorage, NULL)) {
199 if (!jcr->store_bsock->fsend("run")) {
203 * Now start a Storage daemon message thread
205 if (!start_storage_daemon_message_thread(jcr)) {
208 Dmsg0(50, "Storage daemon connection OK\n");
212 * OK, now connect to the File daemon
213 * and ask him for the files.
215 set_jcr_job_status(jcr, JS_Blocked);
216 if (!connect_to_file_daemon(jcr, 10, FDConnectTimeout, 1)) {
220 set_jcr_job_status(jcr, JS_Running);
221 fd = jcr->file_bsock;
224 Dmsg0(30, ">filed: Send include list\n");
225 if (!send_include_list(jcr)) {
229 Dmsg0(30, ">filed: Send exclude list\n");
230 if (!send_exclude_list(jcr)) {
235 * Send Level command to File daemon, as well
236 * as the Storage address if appropriate.
238 switch (jcr->JobLevel) {
242 case L_VERIFY_CATALOG:
245 case L_VERIFY_VOLUME_TO_CATALOG:
247 * send Storage daemon address to the File daemon
249 if (jcr->rstore->SDDport == 0) {
250 jcr->rstore->SDDport = jcr->rstore->SDport;
252 bnet_fsend(fd, storaddr, jcr->rstore->address, jcr->rstore->SDDport);
253 if (!response(jcr, fd, OKstore, "Storage", DISPLAY_ERROR)) {
258 * Send the bootstrap file -- what Volumes/files to restore
260 if (!send_bootstrap_file(jcr, fd) ||
261 !response(jcr, fd, OKbootstrap, "Bootstrap", DISPLAY_ERROR)) {
265 if (!jcr->RestoreBootstrap) {
266 Jmsg0(jcr, M_FATAL, 0, _("Deprecated feature ... use bootstrap.\n"));
275 case L_VERIFY_DISK_TO_CATALOG:
276 level="disk_to_catalog";
279 Jmsg2(jcr, M_FATAL, 0, _("Unimplemented Verify level %d(%c)\n"), jcr->JobLevel,
284 if (!send_runscripts_commands(jcr)) {
289 * Send verify command/level to File daemon
291 fd->fsend(verifycmd, level);
292 if (!response(jcr, fd, OKverify, "Verify", DISPLAY_ERROR)) {
297 * Now get data back from File daemon and
298 * compare it to the catalog or store it in the
299 * catalog depending on the run type.
301 /* Compare to catalog */
302 switch (jcr->JobLevel) {
303 case L_VERIFY_CATALOG:
304 Dmsg0(10, "Verify level=catalog\n");
305 jcr->sd_msg_thread_done = true; /* no SD msg thread, so it is done */
306 jcr->SDJobStatus = JS_Terminated;
307 get_attributes_and_compare_to_catalog(jcr, jcr->previous_jr.JobId);
310 case L_VERIFY_VOLUME_TO_CATALOG:
311 Dmsg0(10, "Verify level=volume\n");
312 get_attributes_and_compare_to_catalog(jcr, jcr->previous_jr.JobId);
315 case L_VERIFY_DISK_TO_CATALOG:
316 Dmsg0(10, "Verify level=disk_to_catalog\n");
317 jcr->sd_msg_thread_done = true; /* no SD msg thread, so it is done */
318 jcr->SDJobStatus = JS_Terminated;
319 get_attributes_and_compare_to_catalog(jcr, jcr->previous_jr.JobId);
324 Dmsg0(10, "Verify level=init\n");
325 jcr->sd_msg_thread_done = true; /* no SD msg thread, so it is done */
326 jcr->SDJobStatus = JS_Terminated;
327 get_attributes_and_put_in_catalog(jcr);
328 db_end_transaction(jcr, jcr->db); /* terminate any open transaction */
329 db_write_batch_file_records(jcr);
333 Jmsg1(jcr, M_FATAL, 0, _("Unimplemented verify level %d\n"), jcr->JobLevel);
337 stat = wait_for_job_termination(jcr);
338 verify_cleanup(jcr, stat);
347 * Release resources allocated during backup.
350 void verify_cleanup(JCR *jcr, int TermCode)
352 char sdt[50], edt[50];
353 char ec1[30], ec2[30];
354 char term_code[100], fd_term_msg[100], sd_term_msg[100];
355 const char *term_msg;
360 // Dmsg1(100, "Enter verify_cleanup() TermCod=%d\n", TermCode);
362 Dmsg3(900, "JobLevel=%c Expected=%u JobFiles=%u\n", jcr->JobLevel,
363 jcr->ExpectedFiles, jcr->JobFiles);
364 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG &&
365 jcr->ExpectedFiles != jcr->JobFiles) {
366 TermCode = JS_ErrorTerminated;
369 /* If no files were expected, there can be no error */
370 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG &&
371 jcr->ExpectedFiles == 0) {
372 TermCode = JS_Terminated;
375 JobId = jcr->jr.JobId;
377 update_job_end(jcr, TermCode);
379 if (job_canceled(jcr)) {
380 cancel_storage_daemon_job(jcr);
383 if (jcr->unlink_bsr && jcr->RestoreBootstrap) {
384 unlink(jcr->RestoreBootstrap);
385 jcr->unlink_bsr = false;
388 msg_type = M_INFO; /* by default INFO message */
391 term_msg = _("Verify OK");
394 case JS_ErrorTerminated:
395 term_msg = _("*** Verify Error ***");
396 msg_type = M_ERROR; /* Generate error message */
399 term_msg = _("Verify warnings");
402 term_msg = _("Verify Canceled");
405 term_msg = _("Verify Differences");
408 term_msg = term_code;
409 bsnprintf(term_code, sizeof(term_code),
410 _("Inappropriate term code: %d %c\n"), TermCode, TermCode);
413 bstrftimes(sdt, sizeof(sdt), jcr->jr.StartTime);
414 bstrftimes(edt, sizeof(edt), jcr->jr.EndTime);
415 if (jcr->verify_job) {
416 Name = jcr->verify_job->hdr.name;
421 jobstatus_to_ascii(jcr->FDJobStatus, fd_term_msg, sizeof(fd_term_msg));
422 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG) {
423 jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg));
424 Jmsg(jcr, msg_type, 0, _("Bacula %s %s (%s): %s\n"
425 " Build OS: %s %s %s\n"
429 " Verify Level: %s\n"
431 " Verify JobId: %d\n"
435 " Files Expected: %s\n"
436 " Files Examined: %s\n"
437 " Non-fatal FD errors: %d\n"
438 " FD termination status: %s\n"
439 " SD termination status: %s\n"
440 " Termination: %s\n\n"),
441 my_name, VERSION, LSMDATE, edt,
442 HOST_OS, DISTNAME, DISTVER,
445 jcr->fileset->hdr.name,
446 level_to_str(jcr->JobLevel),
447 jcr->client->hdr.name,
448 jcr->previous_jr.JobId,
452 edit_uint64_with_commas(jcr->ExpectedFiles, ec1),
453 edit_uint64_with_commas(jcr->JobFiles, ec2),
459 Jmsg(jcr, msg_type, 0, _("Bacula %s %s (%s): %s\n"
464 " Verify Level: %s\n"
466 " Verify JobId: %d\n"
470 " Files Examined: %s\n"
471 " Non-fatal FD errors: %d\n"
472 " FD termination status: %s\n"
473 " Termination: %s\n\n"),
474 my_name, VERSION, LSMDATE, edt,
475 HOST_OS, DISTNAME, DISTVER,
478 jcr->fileset->hdr.name,
479 level_to_str(jcr->JobLevel),
481 jcr->previous_jr.JobId,
485 edit_uint64_with_commas(jcr->JobFiles, ec1),
490 Dmsg0(100, "Leave verify_cleanup()\n");
494 * This routine is called only during a Verify
496 int get_attributes_and_compare_to_catalog(JCR *jcr, JobId_t JobId)
501 struct stat statf; /* file stat */
502 struct stat statc; /* catalog stat */
503 int stat = JS_Terminated;
505 POOLMEM *fname = get_pool_memory(PM_MESSAGE);
506 int do_Digest = CRYPTO_DIGEST_NONE;
507 int32_t file_index = 0;
509 memset(&fdbr, 0, sizeof(FILE_DBR));
510 fd = jcr->file_bsock;
514 Dmsg0(20, "bdird: waiting to receive file attributes\n");
516 * Get Attributes and Signature from File daemon
520 * Options or Digest (MD5/SHA1)
525 while ((n=bget_dirmsg(fd)) >= 0 && !job_canceled(jcr)) {
528 char Opts_Digest[MAXSTRING]; /* Verify Opts or MD5/SHA1 digest */
530 if (job_canceled(jcr)) {
533 fname = check_pool_memory_size(fname, fd->msglen);
534 jcr->fname = check_pool_memory_size(jcr->fname, fd->msglen);
535 Dmsg1(200, "Atts+Digest=%s\n", fd->msg);
536 if ((len = sscanf(fd->msg, "%ld %d %100s", &file_index, &stream,
538 Jmsg3(jcr, M_FATAL, 0, _("bird<filed: bad attributes, expected 3 fields got %d\n"
539 " mslen=%d msg=%s\n"), len, fd->msglen, fd->msg);
543 * We read the Options or Signature into fname
544 * to prevent overrun, now copy it to proper location.
546 bstrncpy(Opts_Digest, fname, sizeof(Opts_Digest));
548 skip_nonspaces(&p); /* skip FileIndex */
550 skip_nonspaces(&p); /* skip Stream */
552 skip_nonspaces(&p); /* skip Opts_Digest */
553 p++; /* skip space */
556 *fn++ = *p++; /* copy filename */
558 *fn = *p++; /* term filename and point to attribs */
561 * Got attributes stream, decode it
563 if (stream == STREAM_UNIX_ATTRIBUTES || stream == STREAM_UNIX_ATTRIBUTES_EX) {
564 int32_t LinkFIf, LinkFIc;
565 Dmsg2(400, "file_index=%d attr=%s\n", file_index, attr);
567 jcr->FileIndex = file_index; /* remember attribute file_index */
568 decode_stat(attr, &statf, &LinkFIf); /* decode file stat packet */
569 do_Digest = CRYPTO_DIGEST_NONE;
570 jcr->fn_printed = false;
571 pm_strcpy(jcr->fname, fname); /* move filename into JCR */
573 Dmsg2(040, "dird<filed: stream=%d %s\n", stream, jcr->fname);
574 Dmsg1(020, "dird<filed: attr=%s\n", attr);
577 * Find equivalent record in the database
580 if (!db_get_file_attributes_record(jcr, jcr->db, jcr->fname,
581 &jcr->previous_jr, &fdbr)) {
582 Jmsg(jcr, M_INFO, 0, _("New file: %s\n"), jcr->fname);
583 Dmsg1(020, _("File not in catalog: %s\n"), jcr->fname);
584 stat = JS_Differences;
588 * mark file record as visited by stuffing the
589 * current JobId, which is unique, into the MarkId field.
591 db_mark_file_record(jcr, jcr->db, fdbr.FileId, jcr->JobId);
594 Dmsg3(400, "Found %s in catalog. inx=%d Opts=%s\n", jcr->fname,
595 file_index, Opts_Digest);
596 decode_stat(fdbr.LStat, &statc, &LinkFIc); /* decode catalog stat */
598 * Loop over options supplied by user and verify the
599 * fields he requests.
601 for (p=Opts_Digest; *p; p++) {
602 char ed1[30], ed2[30];
604 case 'i': /* compare INODEs */
605 if (statc.st_ino != statf.st_ino) {
607 Jmsg(jcr, M_INFO, 0, _(" st_ino differ. Cat: %s File: %s\n"),
608 edit_uint64((uint64_t)statc.st_ino, ed1),
609 edit_uint64((uint64_t)statf.st_ino, ed2));
610 stat = JS_Differences;
613 case 'p': /* permissions bits */
614 if (statc.st_mode != statf.st_mode) {
616 Jmsg(jcr, M_INFO, 0, _(" st_mode differ. Cat: %x File: %x\n"),
617 (uint32_t)statc.st_mode, (uint32_t)statf.st_mode);
618 stat = JS_Differences;
621 case 'n': /* number of links */
622 if (statc.st_nlink != statf.st_nlink) {
624 Jmsg(jcr, M_INFO, 0, _(" st_nlink differ. Cat: %d File: %d\n"),
625 (uint32_t)statc.st_nlink, (uint32_t)statf.st_nlink);
626 stat = JS_Differences;
629 case 'u': /* user id */
630 if (statc.st_uid != statf.st_uid) {
632 Jmsg(jcr, M_INFO, 0, _(" st_uid differ. Cat: %u File: %u\n"),
633 (uint32_t)statc.st_uid, (uint32_t)statf.st_uid);
634 stat = JS_Differences;
637 case 'g': /* group id */
638 if (statc.st_gid != statf.st_gid) {
640 Jmsg(jcr, M_INFO, 0, _(" st_gid differ. Cat: %u File: %u\n"),
641 (uint32_t)statc.st_gid, (uint32_t)statf.st_gid);
642 stat = JS_Differences;
646 if (statc.st_size != statf.st_size) {
648 Jmsg(jcr, M_INFO, 0, _(" st_size differ. Cat: %s File: %s\n"),
649 edit_uint64((uint64_t)statc.st_size, ed1),
650 edit_uint64((uint64_t)statf.st_size, ed2));
651 stat = JS_Differences;
654 case 'a': /* access time */
655 if (statc.st_atime != statf.st_atime) {
657 Jmsg(jcr, M_INFO, 0, _(" st_atime differs\n"));
658 stat = JS_Differences;
662 if (statc.st_mtime != statf.st_mtime) {
664 Jmsg(jcr, M_INFO, 0, _(" st_mtime differs\n"));
665 stat = JS_Differences;
668 case 'c': /* ctime */
669 if (statc.st_ctime != statf.st_ctime) {
671 Jmsg(jcr, M_INFO, 0, _(" st_ctime differs\n"));
672 stat = JS_Differences;
675 case 'd': /* file size decrease */
676 if (statc.st_size > statf.st_size) {
678 Jmsg(jcr, M_INFO, 0, _(" st_size decrease. Cat: %s File: %s\n"),
679 edit_uint64((uint64_t)statc.st_size, ed1),
680 edit_uint64((uint64_t)statf.st_size, ed2));
681 stat = JS_Differences;
684 case '5': /* compare MD5 */
685 Dmsg1(500, "set Do_MD5 for %s\n", jcr->fname);
686 do_Digest = CRYPTO_DIGEST_MD5;
688 case '1': /* compare SHA1 */
689 do_Digest = CRYPTO_DIGEST_SHA1;
698 * Got Digest Signature from Storage daemon
699 * It came across in the Opts_Digest field.
701 } else if (crypto_digest_stream_type(stream) != CRYPTO_DIGEST_NONE) {
702 Dmsg2(400, "stream=Digest inx=%d Digest=%s\n", file_index, Opts_Digest);
704 * When ever we get a digest it MUST have been
705 * preceded by an attributes record, which sets attr_file_index
707 if (jcr->FileIndex != (uint32_t)file_index) {
708 Jmsg2(jcr, M_FATAL, 0, _("MD5/SHA1 index %d not same as attributes %d\n"),
709 file_index, jcr->FileIndex);
712 if (do_Digest != CRYPTO_DIGEST_NONE) {
713 db_escape_string(jcr, jcr->db, buf, Opts_Digest, strlen(Opts_Digest));
714 if (strcmp(buf, fdbr.Digest) != 0) {
716 if (debug_level >= 10) {
717 Jmsg(jcr, M_INFO, 0, _(" %s not same. File=%s Cat=%s\n"),
718 stream_to_ascii(stream), buf, fdbr.Digest);
720 Jmsg(jcr, M_INFO, 0, _(" %s differs.\n"),
721 stream_to_ascii(stream));
723 stat = JS_Differences;
725 do_Digest = CRYPTO_DIGEST_NONE;
728 jcr->JobFiles = file_index;
730 if (is_bnet_error(fd)) {
732 Jmsg2(jcr, M_FATAL, 0, _("bdird<filed: bad attributes from filed n=%d : %s\n"),
737 /* Now find all the files that are missing -- i.e. all files in
738 * the database where the MarkId != current JobId
740 jcr->fn_printed = false;
741 bsnprintf(buf, sizeof(buf),
742 "SELECT Path.Path,Filename.Name FROM File,Path,Filename "
743 "WHERE File.JobId=%d "
744 "AND File.MarkId!=%d AND File.PathId=Path.PathId "
745 "AND File.FilenameId=Filename.FilenameId",
747 /* missing_handler is called for each file found */
748 db_sql_query(jcr->db, buf, missing_handler, (void *)jcr);
749 if (jcr->fn_printed) {
750 stat = JS_Differences;
752 free_pool_memory(fname);
753 if (!job_canceled(jcr)) {
754 jcr->JobStatus = stat;
756 return stat == JS_Terminated;
760 * We are called here for each record that matches the above
761 * SQL query -- that is for each file contained in the Catalog
762 * that was not marked earlier. This means that the file in
763 * question is a missing file (in the Catalog but not on Disk).
765 static int missing_handler(void *ctx, int num_fields, char **row)
767 JCR *jcr = (JCR *)ctx;
769 if (job_canceled(jcr)) {
772 if (!jcr->fn_printed) {
773 Qmsg(jcr, M_INFO, 0, _("\nThe following files are in the Catalog but not on %s:\n"),
774 jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG ? "the Volume(s)" : "disk");
775 jcr->fn_printed = true;
777 Qmsg(jcr, M_INFO, 0, " %s%s\n", row[0]?row[0]:"", row[1]?row[1]:"");
783 * Print filename for verify
785 static void prt_fname(JCR *jcr)
787 if (!jcr->fn_printed) {
788 Jmsg(jcr, M_INFO, 0, _("File: %s\n"), jcr->fname);
789 jcr->fn_printed = true;