3 * Bacula Director -- verify.c -- responsible for running file verification
5 * Kern Sibbald, October MM
7 * Basic tasks done here:
9 * Open connection with File daemon and pass him commands
11 * When the File daemon sends the attributes, compare them to
17 Copyright (C) 2000-2005 Kern Sibbald
19 This program is free software; you can redistribute it and/or
20 modify it under the terms of the GNU General Public License
21 version 2 as amended with additional clauses defined in the
22 file LICENSE in the main source directory.
24 This program is distributed in the hope that it will be useful,
25 but WITHOUT ANY WARRANTY; without even the implied warranty of
26 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
27 the file LICENSE for additional details.
34 #include "findlib/find.h"
36 /* Imported Global Variables */
37 extern int debug_level;
39 /* Commands sent to File daemon */
40 static char verifycmd[] = "verify level=%s\n";
41 static char storaddr[] = "storage address=%s port=%d ssl=0\n";
43 /* Responses received from File daemon */
44 static char OKverify[] = "2000 OK verify\n";
45 static char OKstore[] = "2000 OK storage\n";
47 /* Forward referenced functions */
48 static void prt_fname(JCR *jcr);
49 static int missing_handler(void *ctx, int num_fields, char **row);
53 * Called here before the job is run to do the job
56 bool do_verify_init(JCR *jcr)
59 JobId_t verify_jobid = 0;
62 memset(&jcr->target_jr, 0, sizeof(jcr->target_jr));
64 Dmsg1(9, "bdird: created client %s record\n", jcr->client->hdr.name);
67 * Find JobId of last job that ran. E.g.
68 * for VERIFY_CATALOG we want the JobId of the last INIT.
69 * for VERIFY_VOLUME_TO_CATALOG, we want the JobId of the
72 if (jcr->JobLevel == L_VERIFY_CATALOG ||
73 jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG ||
74 jcr->JobLevel == L_VERIFY_DISK_TO_CATALOG) {
75 memcpy(&jr, &jcr->jr, sizeof(jr));
76 if (jcr->verify_job &&
77 (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG ||
78 jcr->JobLevel == L_VERIFY_DISK_TO_CATALOG)) {
79 Name = jcr->verify_job->hdr.name;
83 Dmsg1(100, "find last jobid for: %s\n", NPRT(Name));
84 if (!db_find_last_jobid(jcr, jcr->db, Name, &jr)) {
85 if (jcr->JobLevel == L_VERIFY_CATALOG) {
86 Jmsg(jcr, M_FATAL, 0, _(
87 "Unable to find JobId of previous InitCatalog Job.\n"
88 "Please run a Verify with Level=InitCatalog before\n"
89 "running the current Job.\n"));
91 Jmsg(jcr, M_FATAL, 0, _(
92 "Unable to find JobId of previous Job for this client.\n"));
96 verify_jobid = jr.JobId;
97 Dmsg1(100, "Last full jobid=%d\n", verify_jobid);
100 * Now get the job record for the previous backup that interests
101 * us. We use the verify_jobid that we found above.
103 if (jcr->JobLevel == L_VERIFY_CATALOG ||
104 jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG ||
105 jcr->JobLevel == L_VERIFY_DISK_TO_CATALOG) {
106 jcr->target_jr.JobId = verify_jobid;
107 if (!db_get_job_record(jcr, jcr->db, &jcr->target_jr)) {
108 Jmsg(jcr, M_FATAL, 0, _("Could not get job record for previous Job. ERR=%s"),
109 db_strerror(jcr->db));
112 if (jcr->target_jr.JobStatus != 'T') {
113 Jmsg(jcr, M_FATAL, 0, _("Last Job %d did not terminate normally. JobStatus=%c\n"),
114 verify_jobid, jcr->target_jr.JobStatus);
117 Jmsg(jcr, M_INFO, 0, _("Verifying against JobId=%d Job=%s\n"),
118 jcr->target_jr.JobId, jcr->target_jr.Job);
122 * If we are verifying a Volume, we need the Storage
123 * daemon, so open a connection, otherwise, just
124 * create a dummy authorization key (passed to
125 * File daemon but not used).
127 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG) {
130 memset(&rx, 0, sizeof(rx));
133 rx.bsr->JobId = jcr->target_jr.JobId;
134 ua = new_ua_context(jcr);
135 complete_bsr(ua, rx.bsr);
136 rx.bsr->fi = new_findex();
137 rx.bsr->fi->findex = 1;
138 rx.bsr->fi->findex2 = jcr->target_jr.JobFiles;
139 jcr->ExpectedFiles = write_bsr_file(ua, rx);
140 if (jcr->ExpectedFiles == 0) {
145 if (jcr->RestoreBootstrap) {
146 free(jcr->RestoreBootstrap);
148 POOLMEM *fname = get_pool_memory(PM_MESSAGE);
149 make_unique_restore_filename(ua, &fname);
150 jcr->RestoreBootstrap = bstrdup(fname);
153 free_pool_memory(fname);
154 jcr->needs_sd = true;
157 jcr->sd_auth_key = bstrdup("dummy"); /* dummy Storage daemon key */
160 if (jcr->JobLevel == L_VERIFY_DISK_TO_CATALOG && jcr->verify_job) {
161 jcr->fileset = jcr->verify_job->fileset;
163 Dmsg2(100, "ClientId=%u JobLevel=%c\n", jcr->target_jr.ClientId, jcr->JobLevel);
169 * Do a verification of the specified files against the Catlaog
171 * Returns: false on failure
174 bool do_verify(JCR *jcr)
180 if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) {
181 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db));
185 /* Print Job Start message */
186 Jmsg(jcr, M_INFO, 0, _("Start Verify JobId=%d Level=%s Job=%s\n"),
187 jcr->JobId, level_to_str(jcr->JobLevel), jcr->Job);
189 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG) {
191 * Start conversation with Storage daemon
193 set_jcr_job_status(jcr, JS_Blocked);
194 if (!connect_to_storage_daemon(jcr, 10, SDConnectTimeout, 1)) {
198 * Now start a job with the Storage daemon
200 if (!start_storage_daemon_job(jcr, jcr->storage, SD_READ)) {
204 * Now start a Storage daemon message thread
206 if (!start_storage_daemon_message_thread(jcr)) {
209 Dmsg0(50, "Storage daemon connection OK\n");
212 * OK, now connect to the File daemon
213 * and ask him for the files.
215 set_jcr_job_status(jcr, JS_Blocked);
216 if (!connect_to_file_daemon(jcr, 10, FDConnectTimeout, 1)) {
220 set_jcr_job_status(jcr, JS_Running);
221 fd = jcr->file_bsock;
224 Dmsg0(30, ">filed: Send include list\n");
225 if (!send_include_list(jcr)) {
229 Dmsg0(30, ">filed: Send exclude list\n");
230 if (!send_exclude_list(jcr)) {
235 * Send Level command to File daemon, as well
236 * as the Storage address if appropriate.
238 switch (jcr->JobLevel) {
242 case L_VERIFY_CATALOG:
245 case L_VERIFY_VOLUME_TO_CATALOG:
247 * send Storage daemon address to the File daemon
249 if (jcr->store->SDDport == 0) {
250 jcr->store->SDDport = jcr->store->SDport;
252 bnet_fsend(fd, storaddr, jcr->store->address, jcr->store->SDDport);
253 if (!response(jcr, fd, OKstore, "Storage", DISPLAY_ERROR)) {
258 * Send the bootstrap file -- what Volumes/files to restore
260 if (!send_bootstrap_file(jcr)) {
264 if (!jcr->RestoreBootstrap) {
265 Jmsg0(jcr, M_FATAL, 0, _("Deprecated feature ... use bootstrap.\n"));
274 case L_VERIFY_DISK_TO_CATALOG:
275 level="disk_to_catalog";
278 Jmsg2(jcr, M_FATAL, 0, _("Unimplemented Verify level %d(%c)\n"), jcr->JobLevel,
283 if (!send_run_before_and_after_commands(jcr)) {
288 * Send verify command/level to File daemon
290 bnet_fsend(fd, verifycmd, level);
291 if (!response(jcr, fd, OKverify, "Verify", DISPLAY_ERROR)) {
296 * Now get data back from File daemon and
297 * compare it to the catalog or store it in the
298 * catalog depending on the run type.
300 /* Compare to catalog */
301 switch (jcr->JobLevel) {
302 case L_VERIFY_CATALOG:
303 Dmsg0(10, "Verify level=catalog\n");
304 jcr->sd_msg_thread_done = true; /* no SD msg thread, so it is done */
305 jcr->SDJobStatus = JS_Terminated;
306 get_attributes_and_compare_to_catalog(jcr, jcr->target_jr.JobId);
309 case L_VERIFY_VOLUME_TO_CATALOG:
310 Dmsg0(10, "Verify level=volume\n");
311 get_attributes_and_compare_to_catalog(jcr, jcr->target_jr.JobId);
314 case L_VERIFY_DISK_TO_CATALOG:
315 Dmsg0(10, "Verify level=disk_to_catalog\n");
316 jcr->sd_msg_thread_done = true; /* no SD msg thread, so it is done */
317 jcr->SDJobStatus = JS_Terminated;
318 get_attributes_and_compare_to_catalog(jcr, jcr->target_jr.JobId);
323 Dmsg0(10, "Verify level=init\n");
324 jcr->sd_msg_thread_done = true; /* no SD msg thread, so it is done */
325 jcr->SDJobStatus = JS_Terminated;
326 get_attributes_and_put_in_catalog(jcr);
330 Jmsg1(jcr, M_FATAL, 0, _("Unimplemented verify level %d\n"), jcr->JobLevel);
334 stat = wait_for_job_termination(jcr);
335 if (stat == JS_Terminated) {
336 verify_cleanup(jcr, stat);
344 * Release resources allocated during backup.
347 void verify_cleanup(JCR *jcr, int TermCode)
349 char sdt[50], edt[50];
350 char ec1[30], ec2[30];
351 char term_code[100], fd_term_msg[100], sd_term_msg[100];
352 const char *term_msg;
357 // Dmsg1(100, "Enter verify_cleanup() TermCod=%d\n", TermCode);
358 dequeue_messages(jcr); /* display any queued messages */
360 Dmsg3(900, "JobLevel=%c Expected=%u JobFiles=%u\n", jcr->JobLevel,
361 jcr->ExpectedFiles, jcr->JobFiles);
362 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG &&
363 jcr->ExpectedFiles != jcr->JobFiles) {
364 TermCode = JS_ErrorTerminated;
367 /* If no files were expected, there can be no error */
368 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG &&
369 jcr->ExpectedFiles == 0) {
370 TermCode = JS_Terminated;
373 JobId = jcr->jr.JobId;
374 set_jcr_job_status(jcr, TermCode);
376 update_job_end_record(jcr);
377 if (jcr->unlink_bsr && jcr->RestoreBootstrap) {
378 unlink(jcr->RestoreBootstrap);
379 jcr->unlink_bsr = false;
382 msg_type = M_INFO; /* by default INFO message */
385 term_msg = _("Verify OK");
388 case JS_ErrorTerminated:
389 term_msg = _("*** Verify Error ***");
390 msg_type = M_ERROR; /* Generate error message */
393 term_msg = _("Verify warnings");
396 term_msg = _("Verify Canceled");
399 term_msg = _("Verify Differences");
402 term_msg = term_code;
403 bsnprintf(term_code, sizeof(term_code),
404 _("Inappropriate term code: %d %c\n"), TermCode, TermCode);
407 bstrftimes(sdt, sizeof(sdt), jcr->jr.StartTime);
408 bstrftimes(edt, sizeof(edt), jcr->jr.EndTime);
409 if (jcr->verify_job) {
410 Name = jcr->verify_job->hdr.name;
415 jobstatus_to_ascii(jcr->FDJobStatus, fd_term_msg, sizeof(fd_term_msg));
416 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG) {
417 jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg));
418 Jmsg(jcr, msg_type, 0, _("Bacula %s (%s): %s\n"
422 " Verify Level: %s\n"
424 " Verify JobId: %d\n"
428 " Files Expected: %s\n"
429 " Files Examined: %s\n"
430 " Non-fatal FD errors: %d\n"
431 " FD termination status: %s\n"
432 " SD termination status: %s\n"
433 " Termination: %s\n\n"),
439 jcr->fileset->hdr.name,
440 level_to_str(jcr->JobLevel),
441 jcr->client->hdr.name,
442 jcr->target_jr.JobId,
446 edit_uint64_with_commas(jcr->ExpectedFiles, ec1),
447 edit_uint64_with_commas(jcr->JobFiles, ec2),
453 Jmsg(jcr, msg_type, 0, _("Bacula %s (%s): %s\n"
457 " Verify Level: %s\n"
459 " Verify JobId: %d\n"
463 " Files Examined: %s\n"
464 " Non-fatal FD errors: %d\n"
465 " FD termination status: %s\n"
466 " Termination: %s\n\n"),
472 jcr->fileset->hdr.name,
473 level_to_str(jcr->JobLevel),
474 jcr->client->hdr.name,
475 jcr->target_jr.JobId,
479 edit_uint64_with_commas(jcr->JobFiles, ec1),
484 Dmsg0(100, "Leave verify_cleanup()\n");
488 * This routine is called only during a Verify
490 int get_attributes_and_compare_to_catalog(JCR *jcr, JobId_t JobId)
495 struct stat statf; /* file stat */
496 struct stat statc; /* catalog stat */
497 int stat = JS_Terminated;
499 POOLMEM *fname = get_pool_memory(PM_MESSAGE);
501 int32_t file_index = 0;
503 memset(&fdbr, 0, sizeof(FILE_DBR));
504 fd = jcr->file_bsock;
508 Dmsg0(20, "bdird: waiting to receive file attributes\n");
510 * Get Attributes and Signature from File daemon
514 * Options or SIG (MD5/SHA1)
519 while ((n=bget_dirmsg(fd)) >= 0 && !job_canceled(jcr)) {
522 char Opts_SIG[MAXSTRING]; /* Verify Opts or MD5/SHA1 signature */
524 fname = check_pool_memory_size(fname, fd->msglen);
525 jcr->fname = check_pool_memory_size(jcr->fname, fd->msglen);
526 Dmsg1(200, "Atts+SIG=%s\n", fd->msg);
527 if ((len = sscanf(fd->msg, "%ld %d %100s", &file_index, &stream,
529 Jmsg3(jcr, M_FATAL, 0, _("bird<filed: bad attributes, expected 3 fields got %d\n"
530 " mslen=%d msg=%s\n"), len, fd->msglen, fd->msg);
534 * We read the Options or Signature into fname
535 * to prevent overrun, now copy it to proper location.
537 bstrncpy(Opts_SIG, fname, sizeof(Opts_SIG));
539 skip_nonspaces(&p); /* skip FileIndex */
541 skip_nonspaces(&p); /* skip Stream */
543 skip_nonspaces(&p); /* skip Opts_SIG */
544 p++; /* skip space */
547 *fn++ = *p++; /* copy filename */
549 *fn = *p++; /* term filename and point to attribs */
552 * Got attributes stream, decode it
554 if (stream == STREAM_UNIX_ATTRIBUTES || stream == STREAM_UNIX_ATTRIBUTES_EX) {
555 int32_t LinkFIf, LinkFIc;
556 Dmsg2(400, "file_index=%d attr=%s\n", file_index, attr);
558 jcr->FileIndex = file_index; /* remember attribute file_index */
559 decode_stat(attr, &statf, &LinkFIf); /* decode file stat packet */
561 jcr->fn_printed = false;
562 pm_strcpy(jcr->fname, fname); /* move filename into JCR */
564 Dmsg2(040, "dird<filed: stream=%d %s\n", stream, jcr->fname);
565 Dmsg1(020, "dird<filed: attr=%s\n", attr);
568 * Find equivalent record in the database
571 if (!db_get_file_attributes_record(jcr, jcr->db, jcr->fname,
572 &jcr->target_jr, &fdbr)) {
573 Jmsg(jcr, M_INFO, 0, _("New file: %s\n"), jcr->fname);
574 Dmsg1(020, _("File not in catalog: %s\n"), jcr->fname);
575 stat = JS_Differences;
579 * mark file record as visited by stuffing the
580 * current JobId, which is unique, into the MarkId field.
582 db_mark_file_record(jcr, jcr->db, fdbr.FileId, jcr->JobId);
585 Dmsg3(400, "Found %s in catalog. inx=%d Opts=%s\n", jcr->fname,
586 file_index, Opts_SIG);
587 decode_stat(fdbr.LStat, &statc, &LinkFIc); /* decode catalog stat */
589 * Loop over options supplied by user and verify the
590 * fields he requests.
592 for (p=Opts_SIG; *p; p++) {
593 char ed1[30], ed2[30];
595 case 'i': /* compare INODEs */
596 if (statc.st_ino != statf.st_ino) {
598 Jmsg(jcr, M_INFO, 0, _(" st_ino differ. Cat: %s File: %s\n"),
599 edit_uint64((uint64_t)statc.st_ino, ed1),
600 edit_uint64((uint64_t)statf.st_ino, ed2));
601 stat = JS_Differences;
604 case 'p': /* permissions bits */
605 if (statc.st_mode != statf.st_mode) {
607 Jmsg(jcr, M_INFO, 0, _(" st_mode differ. Cat: %x File: %x\n"),
608 (uint32_t)statc.st_mode, (uint32_t)statf.st_mode);
609 stat = JS_Differences;
612 case 'n': /* number of links */
613 if (statc.st_nlink != statf.st_nlink) {
615 Jmsg(jcr, M_INFO, 0, _(" st_nlink differ. Cat: %d File: %d\n"),
616 (uint32_t)statc.st_nlink, (uint32_t)statf.st_nlink);
617 stat = JS_Differences;
620 case 'u': /* user id */
621 if (statc.st_uid != statf.st_uid) {
623 Jmsg(jcr, M_INFO, 0, _(" st_uid differ. Cat: %u File: %u\n"),
624 (uint32_t)statc.st_uid, (uint32_t)statf.st_uid);
625 stat = JS_Differences;
628 case 'g': /* group id */
629 if (statc.st_gid != statf.st_gid) {
631 Jmsg(jcr, M_INFO, 0, _(" st_gid differ. Cat: %u File: %u\n"),
632 (uint32_t)statc.st_gid, (uint32_t)statf.st_gid);
633 stat = JS_Differences;
637 if (statc.st_size != statf.st_size) {
639 Jmsg(jcr, M_INFO, 0, _(" st_size differ. Cat: %s File: %s\n"),
640 edit_uint64((uint64_t)statc.st_size, ed1),
641 edit_uint64((uint64_t)statf.st_size, ed2));
642 stat = JS_Differences;
645 case 'a': /* access time */
646 if (statc.st_atime != statf.st_atime) {
648 Jmsg(jcr, M_INFO, 0, _(" st_atime differs\n"));
649 stat = JS_Differences;
653 if (statc.st_mtime != statf.st_mtime) {
655 Jmsg(jcr, M_INFO, 0, _(" st_mtime differs\n"));
656 stat = JS_Differences;
659 case 'c': /* ctime */
660 if (statc.st_ctime != statf.st_ctime) {
662 Jmsg(jcr, M_INFO, 0, _(" st_ctime differs\n"));
663 stat = JS_Differences;
666 case 'd': /* file size decrease */
667 if (statc.st_size > statf.st_size) {
669 Jmsg(jcr, M_INFO, 0, _(" st_size decrease. Cat: %s File: %s\n"),
670 edit_uint64((uint64_t)statc.st_size, ed1),
671 edit_uint64((uint64_t)statf.st_size, ed2));
672 stat = JS_Differences;
675 case '5': /* compare MD5 */
676 Dmsg1(500, "set Do_MD5 for %s\n", jcr->fname);
679 case '1': /* compare SHA1 */
689 * Got SIG Signature from Storage daemon
690 * It came across in the Opts_SIG field.
692 } else if (stream == STREAM_MD5_SIGNATURE || stream == STREAM_SHA1_SIGNATURE) {
693 Dmsg2(400, "stream=SIG inx=%d SIG=%s\n", file_index, Opts_SIG);
695 * When ever we get a signature is MUST have been
696 * preceded by an attributes record, which sets attr_file_index
698 if (jcr->FileIndex != (uint32_t)file_index) {
699 Jmsg2(jcr, M_FATAL, 0, _("MD5/SHA1 index %d not same as attributes %d\n"),
700 file_index, jcr->FileIndex);
704 db_escape_string(buf, Opts_SIG, strlen(Opts_SIG));
705 if (strcmp(buf, fdbr.SIG) != 0) {
707 if (debug_level >= 10) {
708 Jmsg(jcr, M_INFO, 0, _(" %s not same. File=%s Cat=%s\n"),
709 stream==STREAM_MD5_SIGNATURE?"MD5":"SHA1", buf, fdbr.SIG);
711 Jmsg(jcr, M_INFO, 0, _(" %s differs.\n"),
712 stream==STREAM_MD5_SIGNATURE?"MD5":"SHA1");
714 stat = JS_Differences;
719 jcr->JobFiles = file_index;
721 if (is_bnet_error(fd)) {
723 Jmsg2(jcr, M_FATAL, 0, _("bdird<filed: bad attributes from filed n=%d : %s\n"),
728 /* Now find all the files that are missing -- i.e. all files in
729 * the database where the MarkedId != current JobId
731 jcr->fn_printed = false;
732 bsnprintf(buf, sizeof(buf),
733 "SELECT Path.Path,Filename.Name FROM File,Path,Filename "
734 "WHERE File.JobId=%d "
735 "AND File.MarkedId!=%d AND File.PathId=Path.PathId "
736 "AND File.FilenameId=Filename.FilenameId",
738 /* missing_handler is called for each file found */
739 db_sql_query(jcr->db, buf, missing_handler, (void *)jcr);
740 if (jcr->fn_printed) {
741 stat = JS_Differences;
743 free_pool_memory(fname);
744 set_jcr_job_status(jcr, stat);
745 return stat == JS_Terminated;
749 * We are called here for each record that matches the above
750 * SQL query -- that is for each file contained in the Catalog
751 * that was not marked earlier. This means that the file in
752 * question is a missing file (in the Catalog but not on Disk).
754 static int missing_handler(void *ctx, int num_fields, char **row)
756 JCR *jcr = (JCR *)ctx;
758 if (!jcr->fn_printed) {
759 Jmsg(jcr, M_INFO, 0, "\n");
760 Jmsg(jcr, M_INFO, 0, _("The following files are missing:\n"));
761 jcr->fn_printed = true;
763 Jmsg(jcr, M_INFO, 0, " %s%s\n", row[0]?row[0]:"", row[1]?row[1]:"");
769 * Print filename for verify
771 static void prt_fname(JCR *jcr)
773 if (!jcr->fn_printed) {
774 Jmsg(jcr, M_INFO, 0, _("File: %s\n"), jcr->fname);
775 jcr->fn_printed = TRUE;