3 * Bacula Director -- verify.c -- responsible for running file verification
5 * Kern Sibbald, October MM
7 * Basic tasks done here:
9 * Open connection with File daemon and pass him commands
11 * When the File daemon sends the attributes, compare them to
17 Bacula® - The Network Backup Solution
19 Copyright (C) 2000-2006 Free Software Foundation Europe e.V.
21 The main author of Bacula is Kern Sibbald, with contributions from
22 many others, a complete list can be found in the file AUTHORS.
23 This program is Free Software; you can redistribute it and/or
24 modify it under the terms of version two of the GNU General Public
25 License as published by the Free Software Foundation plus additions
26 that are listed in the file LICENSE.
28 This program is distributed in the hope that it will be useful, but
29 WITHOUT ANY WARRANTY; without even the implied warranty of
30 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
31 General Public License for more details.
33 You should have received a copy of the GNU General Public License
34 along with this program; if not, write to the Free Software
35 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
38 Bacula® is a registered trademark of John Walker.
39 The licensor of Bacula is the Free Software Foundation Europe
40 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
41 Switzerland, email:ftf@fsfeurope.org.
47 #include "findlib/find.h"
49 /* Commands sent to File daemon */
50 static char verifycmd[] = "verify level=%s\n";
51 static char storaddr[] = "storage address=%s port=%d ssl=0\n";
53 /* Responses received from File daemon */
54 static char OKverify[] = "2000 OK verify\n";
55 static char OKstore[] = "2000 OK storage\n";
56 static char OKbootstrap[] = "2000 OK bootstrap\n";
58 /* Forward referenced functions */
59 static void prt_fname(JCR *jcr);
60 static int missing_handler(void *ctx, int num_fields, char **row);
64 * Called here before the job is run to do the job
67 bool do_verify_init(JCR *jcr)
70 JobId_t verify_jobid = 0;
73 free_wstorage(jcr); /* we don't write */
75 memset(&jcr->previous_jr, 0, sizeof(jcr->previous_jr));
77 Dmsg1(9, "bdird: created client %s record\n", jcr->client->hdr.name);
80 * Find JobId of last job that ran. E.g.
81 * for VERIFY_CATALOG we want the JobId of the last INIT.
82 * for VERIFY_VOLUME_TO_CATALOG, we want the JobId of the
85 if (jcr->JobLevel == L_VERIFY_CATALOG ||
86 jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG ||
87 jcr->JobLevel == L_VERIFY_DISK_TO_CATALOG) {
88 memcpy(&jr, &jcr->jr, sizeof(jr));
89 if (jcr->verify_job &&
90 (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG ||
91 jcr->JobLevel == L_VERIFY_DISK_TO_CATALOG)) {
92 Name = jcr->verify_job->hdr.name;
96 Dmsg1(100, "find last jobid for: %s\n", NPRT(Name));
97 if (!db_find_last_jobid(jcr, jcr->db, Name, &jr)) {
98 if (jcr->JobLevel == L_VERIFY_CATALOG) {
99 Jmsg(jcr, M_FATAL, 0, _(
100 "Unable to find JobId of previous InitCatalog Job.\n"
101 "Please run a Verify with Level=InitCatalog before\n"
102 "running the current Job.\n"));
104 Jmsg(jcr, M_FATAL, 0, _(
105 "Unable to find JobId of previous Job for this client.\n"));
109 verify_jobid = jr.JobId;
110 Dmsg1(100, "Last full jobid=%d\n", verify_jobid);
113 * Now get the job record for the previous backup that interests
114 * us. We use the verify_jobid that we found above.
116 if (jcr->JobLevel == L_VERIFY_CATALOG ||
117 jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG ||
118 jcr->JobLevel == L_VERIFY_DISK_TO_CATALOG) {
119 jcr->previous_jr.JobId = verify_jobid;
120 if (!db_get_job_record(jcr, jcr->db, &jcr->previous_jr)) {
121 Jmsg(jcr, M_FATAL, 0, _("Could not get job record for previous Job. ERR=%s"),
122 db_strerror(jcr->db));
125 if (jcr->previous_jr.JobStatus != 'T') {
126 Jmsg(jcr, M_FATAL, 0, _("Last Job %d did not terminate normally. JobStatus=%c\n"),
127 verify_jobid, jcr->previous_jr.JobStatus);
130 Jmsg(jcr, M_INFO, 0, _("Verifying against JobId=%d Job=%s\n"),
131 jcr->previous_jr.JobId, jcr->previous_jr.Job);
135 * If we are verifying a Volume, we need the Storage
136 * daemon, so open a connection, otherwise, just
137 * create a dummy authorization key (passed to
138 * File daemon but not used).
140 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG) {
141 if (!create_restore_bootstrap_file(jcr)) {
145 jcr->sd_auth_key = bstrdup("dummy"); /* dummy Storage daemon key */
148 if (jcr->JobLevel == L_VERIFY_DISK_TO_CATALOG && jcr->verify_job) {
149 jcr->fileset = jcr->verify_job->fileset;
151 Dmsg2(100, "ClientId=%u JobLevel=%c\n", jcr->previous_jr.ClientId, jcr->JobLevel);
157 * Do a verification of the specified files against the Catlaog
159 * Returns: false on failure
162 bool do_verify(JCR *jcr)
169 if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) {
170 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db));
174 /* Print Job Start message */
175 Jmsg(jcr, M_INFO, 0, _("Start Verify JobId=%s Level=%s Job=%s\n"),
176 edit_uint64(jcr->JobId, ed1), level_to_str(jcr->JobLevel), jcr->Job);
178 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG) {
180 * Start conversation with Storage daemon
182 set_jcr_job_status(jcr, JS_Blocked);
183 if (!connect_to_storage_daemon(jcr, 10, SDConnectTimeout, 1)) {
187 * Now start a job with the Storage daemon
189 if (!start_storage_daemon_job(jcr, jcr->rstorage, NULL)) {
192 if (!bnet_fsend(jcr->store_bsock, "run")) {
196 * Now start a Storage daemon message thread
198 if (!start_storage_daemon_message_thread(jcr)) {
201 Dmsg0(50, "Storage daemon connection OK\n");
205 * OK, now connect to the File daemon
206 * and ask him for the files.
208 set_jcr_job_status(jcr, JS_Blocked);
209 if (!connect_to_file_daemon(jcr, 10, FDConnectTimeout, 1)) {
213 set_jcr_job_status(jcr, JS_Running);
214 fd = jcr->file_bsock;
217 Dmsg0(30, ">filed: Send include list\n");
218 if (!send_include_list(jcr)) {
222 Dmsg0(30, ">filed: Send exclude list\n");
223 if (!send_exclude_list(jcr)) {
228 * Send Level command to File daemon, as well
229 * as the Storage address if appropriate.
231 switch (jcr->JobLevel) {
235 case L_VERIFY_CATALOG:
238 case L_VERIFY_VOLUME_TO_CATALOG:
240 * send Storage daemon address to the File daemon
242 if (jcr->rstore->SDDport == 0) {
243 jcr->rstore->SDDport = jcr->rstore->SDport;
245 bnet_fsend(fd, storaddr, jcr->rstore->address, jcr->rstore->SDDport);
246 if (!response(jcr, fd, OKstore, "Storage", DISPLAY_ERROR)) {
251 * Send the bootstrap file -- what Volumes/files to restore
253 if (!send_bootstrap_file(jcr, fd) ||
254 !response(jcr, fd, OKbootstrap, "Bootstrap", DISPLAY_ERROR)) {
258 if (!jcr->RestoreBootstrap) {
259 Jmsg0(jcr, M_FATAL, 0, _("Deprecated feature ... use bootstrap.\n"));
268 case L_VERIFY_DISK_TO_CATALOG:
269 level="disk_to_catalog";
272 Jmsg2(jcr, M_FATAL, 0, _("Unimplemented Verify level %d(%c)\n"), jcr->JobLevel,
277 if (!send_runscripts_commands(jcr)) {
282 * Send verify command/level to File daemon
284 bnet_fsend(fd, verifycmd, level);
285 if (!response(jcr, fd, OKverify, "Verify", DISPLAY_ERROR)) {
290 * Now get data back from File daemon and
291 * compare it to the catalog or store it in the
292 * catalog depending on the run type.
294 /* Compare to catalog */
295 switch (jcr->JobLevel) {
296 case L_VERIFY_CATALOG:
297 Dmsg0(10, "Verify level=catalog\n");
298 jcr->sd_msg_thread_done = true; /* no SD msg thread, so it is done */
299 jcr->SDJobStatus = JS_Terminated;
300 get_attributes_and_compare_to_catalog(jcr, jcr->previous_jr.JobId);
303 case L_VERIFY_VOLUME_TO_CATALOG:
304 Dmsg0(10, "Verify level=volume\n");
305 get_attributes_and_compare_to_catalog(jcr, jcr->previous_jr.JobId);
308 case L_VERIFY_DISK_TO_CATALOG:
309 Dmsg0(10, "Verify level=disk_to_catalog\n");
310 jcr->sd_msg_thread_done = true; /* no SD msg thread, so it is done */
311 jcr->SDJobStatus = JS_Terminated;
312 get_attributes_and_compare_to_catalog(jcr, jcr->previous_jr.JobId);
317 Dmsg0(10, "Verify level=init\n");
318 jcr->sd_msg_thread_done = true; /* no SD msg thread, so it is done */
319 jcr->SDJobStatus = JS_Terminated;
320 get_attributes_and_put_in_catalog(jcr);
324 Jmsg1(jcr, M_FATAL, 0, _("Unimplemented verify level %d\n"), jcr->JobLevel);
328 stat = wait_for_job_termination(jcr);
329 if (stat == JS_Terminated) {
330 verify_cleanup(jcr, stat);
338 * Release resources allocated during backup.
341 void verify_cleanup(JCR *jcr, int TermCode)
343 char sdt[50], edt[50];
344 char ec1[30], ec2[30];
345 char term_code[100], fd_term_msg[100], sd_term_msg[100];
346 const char *term_msg;
351 // Dmsg1(100, "Enter verify_cleanup() TermCod=%d\n", TermCode);
353 Dmsg3(900, "JobLevel=%c Expected=%u JobFiles=%u\n", jcr->JobLevel,
354 jcr->ExpectedFiles, jcr->JobFiles);
355 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG &&
356 jcr->ExpectedFiles != jcr->JobFiles) {
357 TermCode = JS_ErrorTerminated;
360 /* If no files were expected, there can be no error */
361 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG &&
362 jcr->ExpectedFiles == 0) {
363 TermCode = JS_Terminated;
366 JobId = jcr->jr.JobId;
368 update_job_end(jcr, TermCode);
370 if (jcr->unlink_bsr && jcr->RestoreBootstrap) {
371 unlink(jcr->RestoreBootstrap);
372 jcr->unlink_bsr = false;
375 msg_type = M_INFO; /* by default INFO message */
378 term_msg = _("Verify OK");
381 case JS_ErrorTerminated:
382 term_msg = _("*** Verify Error ***");
383 msg_type = M_ERROR; /* Generate error message */
386 term_msg = _("Verify warnings");
389 term_msg = _("Verify Canceled");
392 term_msg = _("Verify Differences");
395 term_msg = term_code;
396 bsnprintf(term_code, sizeof(term_code),
397 _("Inappropriate term code: %d %c\n"), TermCode, TermCode);
400 bstrftimes(sdt, sizeof(sdt), jcr->jr.StartTime);
401 bstrftimes(edt, sizeof(edt), jcr->jr.EndTime);
402 if (jcr->verify_job) {
403 Name = jcr->verify_job->hdr.name;
408 jobstatus_to_ascii(jcr->FDJobStatus, fd_term_msg, sizeof(fd_term_msg));
409 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG) {
410 jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg));
411 Jmsg(jcr, msg_type, 0, _("Bacula %s (%s): %s\n"
415 " Verify Level: %s\n"
417 " Verify JobId: %d\n"
421 " Files Expected: %s\n"
422 " Files Examined: %s\n"
423 " Non-fatal FD errors: %d\n"
424 " FD termination status: %s\n"
425 " SD termination status: %s\n"
426 " Termination: %s\n\n"),
432 jcr->fileset->hdr.name,
433 level_to_str(jcr->JobLevel),
434 jcr->client->hdr.name,
435 jcr->previous_jr.JobId,
439 edit_uint64_with_commas(jcr->ExpectedFiles, ec1),
440 edit_uint64_with_commas(jcr->JobFiles, ec2),
446 Jmsg(jcr, msg_type, 0, _("Bacula %s (%s): %s\n"
450 " Verify Level: %s\n"
452 " Verify JobId: %d\n"
456 " Files Examined: %s\n"
457 " Non-fatal FD errors: %d\n"
458 " FD termination status: %s\n"
459 " Termination: %s\n\n"),
465 jcr->fileset->hdr.name,
466 level_to_str(jcr->JobLevel),
467 jcr->client->hdr.name,
468 jcr->previous_jr.JobId,
472 edit_uint64_with_commas(jcr->JobFiles, ec1),
477 Dmsg0(100, "Leave verify_cleanup()\n");
481 * This routine is called only during a Verify
483 int get_attributes_and_compare_to_catalog(JCR *jcr, JobId_t JobId)
488 struct stat statf; /* file stat */
489 struct stat statc; /* catalog stat */
490 int stat = JS_Terminated;
492 POOLMEM *fname = get_pool_memory(PM_MESSAGE);
493 int do_Digest = CRYPTO_DIGEST_NONE;
494 int32_t file_index = 0;
496 memset(&fdbr, 0, sizeof(FILE_DBR));
497 fd = jcr->file_bsock;
501 Dmsg0(20, "bdird: waiting to receive file attributes\n");
503 * Get Attributes and Signature from File daemon
507 * Options or Digest (MD5/SHA1)
512 while ((n=bget_dirmsg(fd)) >= 0 && !job_canceled(jcr)) {
515 char Opts_Digest[MAXSTRING]; /* Verify Opts or MD5/SHA1 digest */
517 if (job_canceled(jcr)) {
520 fname = check_pool_memory_size(fname, fd->msglen);
521 jcr->fname = check_pool_memory_size(jcr->fname, fd->msglen);
522 Dmsg1(200, "Atts+Digest=%s\n", fd->msg);
523 if ((len = sscanf(fd->msg, "%ld %d %100s", &file_index, &stream,
525 Jmsg3(jcr, M_FATAL, 0, _("bird<filed: bad attributes, expected 3 fields got %d\n"
526 " mslen=%d msg=%s\n"), len, fd->msglen, fd->msg);
530 * We read the Options or Signature into fname
531 * to prevent overrun, now copy it to proper location.
533 bstrncpy(Opts_Digest, fname, sizeof(Opts_Digest));
535 skip_nonspaces(&p); /* skip FileIndex */
537 skip_nonspaces(&p); /* skip Stream */
539 skip_nonspaces(&p); /* skip Opts_Digest */
540 p++; /* skip space */
543 *fn++ = *p++; /* copy filename */
545 *fn = *p++; /* term filename and point to attribs */
548 * Got attributes stream, decode it
550 if (stream == STREAM_UNIX_ATTRIBUTES || stream == STREAM_UNIX_ATTRIBUTES_EX) {
551 int32_t LinkFIf, LinkFIc;
552 Dmsg2(400, "file_index=%d attr=%s\n", file_index, attr);
554 jcr->FileIndex = file_index; /* remember attribute file_index */
555 decode_stat(attr, &statf, &LinkFIf); /* decode file stat packet */
556 do_Digest = CRYPTO_DIGEST_NONE;
557 jcr->fn_printed = false;
558 pm_strcpy(jcr->fname, fname); /* move filename into JCR */
560 Dmsg2(040, "dird<filed: stream=%d %s\n", stream, jcr->fname);
561 Dmsg1(020, "dird<filed: attr=%s\n", attr);
564 * Find equivalent record in the database
567 if (!db_get_file_attributes_record(jcr, jcr->db, jcr->fname,
568 &jcr->previous_jr, &fdbr)) {
569 Jmsg(jcr, M_INFO, 0, _("New file: %s\n"), jcr->fname);
570 Dmsg1(020, _("File not in catalog: %s\n"), jcr->fname);
571 stat = JS_Differences;
575 * mark file record as visited by stuffing the
576 * current JobId, which is unique, into the MarkId field.
578 db_mark_file_record(jcr, jcr->db, fdbr.FileId, jcr->JobId);
581 Dmsg3(400, "Found %s in catalog. inx=%d Opts=%s\n", jcr->fname,
582 file_index, Opts_Digest);
583 decode_stat(fdbr.LStat, &statc, &LinkFIc); /* decode catalog stat */
585 * Loop over options supplied by user and verify the
586 * fields he requests.
588 for (p=Opts_Digest; *p; p++) {
589 char ed1[30], ed2[30];
591 case 'i': /* compare INODEs */
592 if (statc.st_ino != statf.st_ino) {
594 Jmsg(jcr, M_INFO, 0, _(" st_ino differ. Cat: %s File: %s\n"),
595 edit_uint64((uint64_t)statc.st_ino, ed1),
596 edit_uint64((uint64_t)statf.st_ino, ed2));
597 stat = JS_Differences;
600 case 'p': /* permissions bits */
601 if (statc.st_mode != statf.st_mode) {
603 Jmsg(jcr, M_INFO, 0, _(" st_mode differ. Cat: %x File: %x\n"),
604 (uint32_t)statc.st_mode, (uint32_t)statf.st_mode);
605 stat = JS_Differences;
608 case 'n': /* number of links */
609 if (statc.st_nlink != statf.st_nlink) {
611 Jmsg(jcr, M_INFO, 0, _(" st_nlink differ. Cat: %d File: %d\n"),
612 (uint32_t)statc.st_nlink, (uint32_t)statf.st_nlink);
613 stat = JS_Differences;
616 case 'u': /* user id */
617 if (statc.st_uid != statf.st_uid) {
619 Jmsg(jcr, M_INFO, 0, _(" st_uid differ. Cat: %u File: %u\n"),
620 (uint32_t)statc.st_uid, (uint32_t)statf.st_uid);
621 stat = JS_Differences;
624 case 'g': /* group id */
625 if (statc.st_gid != statf.st_gid) {
627 Jmsg(jcr, M_INFO, 0, _(" st_gid differ. Cat: %u File: %u\n"),
628 (uint32_t)statc.st_gid, (uint32_t)statf.st_gid);
629 stat = JS_Differences;
633 if (statc.st_size != statf.st_size) {
635 Jmsg(jcr, M_INFO, 0, _(" st_size differ. Cat: %s File: %s\n"),
636 edit_uint64((uint64_t)statc.st_size, ed1),
637 edit_uint64((uint64_t)statf.st_size, ed2));
638 stat = JS_Differences;
641 case 'a': /* access time */
642 if (statc.st_atime != statf.st_atime) {
644 Jmsg(jcr, M_INFO, 0, _(" st_atime differs\n"));
645 stat = JS_Differences;
649 if (statc.st_mtime != statf.st_mtime) {
651 Jmsg(jcr, M_INFO, 0, _(" st_mtime differs\n"));
652 stat = JS_Differences;
655 case 'c': /* ctime */
656 if (statc.st_ctime != statf.st_ctime) {
658 Jmsg(jcr, M_INFO, 0, _(" st_ctime differs\n"));
659 stat = JS_Differences;
662 case 'd': /* file size decrease */
663 if (statc.st_size > statf.st_size) {
665 Jmsg(jcr, M_INFO, 0, _(" st_size decrease. Cat: %s File: %s\n"),
666 edit_uint64((uint64_t)statc.st_size, ed1),
667 edit_uint64((uint64_t)statf.st_size, ed2));
668 stat = JS_Differences;
671 case '5': /* compare MD5 */
672 Dmsg1(500, "set Do_MD5 for %s\n", jcr->fname);
673 do_Digest = CRYPTO_DIGEST_MD5;
675 case '1': /* compare SHA1 */
676 do_Digest = CRYPTO_DIGEST_SHA1;
685 * Got Digest Signature from Storage daemon
686 * It came across in the Opts_Digest field.
688 } else if (crypto_digest_stream_type(stream) != CRYPTO_DIGEST_NONE) {
689 Dmsg2(400, "stream=Digest inx=%d Digest=%s\n", file_index, Opts_Digest);
691 * When ever we get a digest is MUST have been
692 * preceded by an attributes record, which sets attr_file_index
694 if (jcr->FileIndex != (uint32_t)file_index) {
695 Jmsg2(jcr, M_FATAL, 0, _("MD5/SHA1 index %d not same as attributes %d\n"),
696 file_index, jcr->FileIndex);
699 if (do_Digest != CRYPTO_DIGEST_NONE) {
700 db_escape_string(buf, Opts_Digest, strlen(Opts_Digest));
701 if (strcmp(buf, fdbr.Digest) != 0) {
703 if (debug_level >= 10) {
704 Jmsg(jcr, M_INFO, 0, _(" %s not same. File=%s Cat=%s\n"),
705 stream_to_ascii(stream), buf, fdbr.Digest);
707 Jmsg(jcr, M_INFO, 0, _(" %s differs.\n"),
708 stream_to_ascii(stream));
710 stat = JS_Differences;
712 do_Digest = CRYPTO_DIGEST_NONE;
715 jcr->JobFiles = file_index;
717 if (is_bnet_error(fd)) {
719 Jmsg2(jcr, M_FATAL, 0, _("bdird<filed: bad attributes from filed n=%d : %s\n"),
724 /* Now find all the files that are missing -- i.e. all files in
725 * the database where the MarkId != current JobId
727 jcr->fn_printed = false;
728 bsnprintf(buf, sizeof(buf),
729 "SELECT Path.Path,Filename.Name FROM File,Path,Filename "
730 "WHERE File.JobId=%d "
731 "AND File.MarkId!=%d AND File.PathId=Path.PathId "
732 "AND File.FilenameId=Filename.FilenameId",
734 /* missing_handler is called for each file found */
735 db_sql_query(jcr->db, buf, missing_handler, (void *)jcr);
736 if (jcr->fn_printed) {
737 stat = JS_Differences;
739 free_pool_memory(fname);
740 set_jcr_job_status(jcr, stat);
741 return stat == JS_Terminated;
745 * We are called here for each record that matches the above
746 * SQL query -- that is for each file contained in the Catalog
747 * that was not marked earlier. This means that the file in
748 * question is a missing file (in the Catalog but not on Disk).
750 static int missing_handler(void *ctx, int num_fields, char **row)
752 JCR *jcr = (JCR *)ctx;
754 if (job_canceled(jcr)) {
757 if (!jcr->fn_printed) {
758 Jmsg(jcr, M_INFO, 0, "\n");
759 Jmsg(jcr, M_INFO, 0, _("The following files are in the Catalog but not on disk:\n"));
760 jcr->fn_printed = true;
762 Jmsg(jcr, M_INFO, 0, " %s%s\n", row[0]?row[0]:"", row[1]?row[1]:"");
768 * Print filename for verify
770 static void prt_fname(JCR *jcr)
772 if (!jcr->fn_printed) {
773 Jmsg(jcr, M_INFO, 0, _("File: %s\n"), jcr->fname);
774 jcr->fn_printed = TRUE;