3 * Bacula Director -- verify.c -- responsible for running file verification
5 * Kern Sibbald, October MM
7 * Basic tasks done here:
9 * Open connection with File daemon and pass him commands
11 * When the File daemon sends the attributes, compare them to
17 Copyright (C) 2000-2005 Kern Sibbald
19 This program is free software; you can redistribute it and/or
20 modify it under the terms of the GNU General Public License
21 version 2 as amended with additional clauses defined in the
22 file LICENSE in the main source directory.
24 This program is distributed in the hope that it will be useful,
25 but WITHOUT ANY WARRANTY; without even the implied warranty of
26 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
27 the file LICENSE for additional details.
34 #include "findlib/find.h"
36 /* Imported Global Variables */
37 extern int debug_level;
39 /* Commands sent to File daemon */
40 static char verifycmd[] = "verify level=%s\n";
41 static char storaddr[] = "storage address=%s port=%d ssl=0\n";
43 /* Responses received from File daemon */
44 static char OKverify[] = "2000 OK verify\n";
45 static char OKstore[] = "2000 OK storage\n";
47 /* Forward referenced functions */
48 static void prt_fname(JCR *jcr);
49 static int missing_handler(void *ctx, int num_fields, char **row);
53 * Called here before the job is run to do the job
56 bool do_verify_init(JCR *jcr)
59 JobId_t verify_jobid = 0;
62 memset(&jcr->target_jr, 0, sizeof(jcr->target_jr));
64 Dmsg1(9, "bdird: created client %s record\n", jcr->client->hdr.name);
67 * Find JobId of last job that ran. E.g.
68 * for VERIFY_CATALOG we want the JobId of the last INIT.
69 * for VERIFY_VOLUME_TO_CATALOG, we want the JobId of the
72 if (jcr->JobLevel == L_VERIFY_CATALOG ||
73 jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG ||
74 jcr->JobLevel == L_VERIFY_DISK_TO_CATALOG) {
75 memcpy(&jr, &jcr->jr, sizeof(jr));
76 if (jcr->verify_job &&
77 (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG ||
78 jcr->JobLevel == L_VERIFY_DISK_TO_CATALOG)) {
79 Name = jcr->verify_job->hdr.name;
83 Dmsg1(100, "find last jobid for: %s\n", NPRT(Name));
84 if (!db_find_last_jobid(jcr, jcr->db, Name, &jr)) {
85 if (jcr->JobLevel == L_VERIFY_CATALOG) {
86 Jmsg(jcr, M_FATAL, 0, _(
87 "Unable to find JobId of previous InitCatalog Job.\n"
88 "Please run a Verify with Level=InitCatalog before\n"
89 "running the current Job.\n"));
91 Jmsg(jcr, M_FATAL, 0, _(
92 "Unable to find JobId of previous Job for this client.\n"));
96 verify_jobid = jr.JobId;
97 Dmsg1(100, "Last full jobid=%d\n", verify_jobid);
100 * Now get the job record for the previous backup that interests
101 * us. We use the verify_jobid that we found above.
103 if (jcr->JobLevel == L_VERIFY_CATALOG ||
104 jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG ||
105 jcr->JobLevel == L_VERIFY_DISK_TO_CATALOG) {
106 jcr->target_jr.JobId = verify_jobid;
107 if (!db_get_job_record(jcr, jcr->db, &jcr->target_jr)) {
108 Jmsg(jcr, M_FATAL, 0, _("Could not get job record for previous Job. ERR=%s"),
109 db_strerror(jcr->db));
112 if (jcr->target_jr.JobStatus != 'T') {
113 Jmsg(jcr, M_FATAL, 0, _("Last Job %d did not terminate normally. JobStatus=%c\n"),
114 verify_jobid, jcr->target_jr.JobStatus);
117 Jmsg(jcr, M_INFO, 0, _("Verifying against JobId=%d Job=%s\n"),
118 jcr->target_jr.JobId, jcr->target_jr.Job);
122 * If we are verifying a Volume, we need the Storage
123 * daemon, so open a connection, otherwise, just
124 * create a dummy authorization key (passed to
125 * File daemon but not used).
127 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG) {
130 memset(&rx, 0, sizeof(rx));
133 rx.bsr->JobId = jcr->target_jr.JobId;
134 ua = new_ua_context(jcr);
135 complete_bsr(ua, rx.bsr);
136 rx.bsr->fi = new_findex();
137 rx.bsr->fi->findex = 1;
138 rx.bsr->fi->findex2 = jcr->target_jr.JobFiles;
139 jcr->ExpectedFiles = write_bsr_file(ua, rx);
140 if (jcr->ExpectedFiles == 0) {
145 if (jcr->RestoreBootstrap) {
146 free(jcr->RestoreBootstrap);
148 POOLMEM *fname = get_pool_memory(PM_MESSAGE);
149 make_unique_restore_filename(ua, &fname);
150 jcr->RestoreBootstrap = bstrdup(fname);
153 free_pool_memory(fname);
154 jcr->needs_sd = true;
157 jcr->sd_auth_key = bstrdup("dummy"); /* dummy Storage daemon key */
160 if (jcr->JobLevel == L_VERIFY_DISK_TO_CATALOG && jcr->verify_job) {
161 jcr->fileset = jcr->verify_job->fileset;
163 Dmsg2(100, "ClientId=%u JobLevel=%c\n", jcr->target_jr.ClientId, jcr->JobLevel);
169 * Do a verification of the specified files against the Catlaog
171 * Returns: false on failure
174 bool do_verify(JCR *jcr)
181 if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) {
182 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db));
186 /* Print Job Start message */
187 Jmsg(jcr, M_INFO, 0, _("Start Verify JobId=%s Level=%s Job=%s\n"),
188 edit_uint64(jcr->JobId, ed1), level_to_str(jcr->JobLevel), jcr->Job);
190 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG) {
192 * Start conversation with Storage daemon
194 set_jcr_job_status(jcr, JS_Blocked);
195 if (!connect_to_storage_daemon(jcr, 10, SDConnectTimeout, 1)) {
199 * Now start a job with the Storage daemon
201 if (!start_storage_daemon_job(jcr, jcr->storage, NULL)) {
205 * Now start a Storage daemon message thread
207 if (!start_storage_daemon_message_thread(jcr)) {
210 Dmsg0(50, "Storage daemon connection OK\n");
213 * OK, now connect to the File daemon
214 * and ask him for the files.
216 set_jcr_job_status(jcr, JS_Blocked);
217 if (!connect_to_file_daemon(jcr, 10, FDConnectTimeout, 1)) {
221 set_jcr_job_status(jcr, JS_Running);
222 fd = jcr->file_bsock;
225 Dmsg0(30, ">filed: Send include list\n");
226 if (!send_include_list(jcr)) {
230 Dmsg0(30, ">filed: Send exclude list\n");
231 if (!send_exclude_list(jcr)) {
236 * Send Level command to File daemon, as well
237 * as the Storage address if appropriate.
239 switch (jcr->JobLevel) {
243 case L_VERIFY_CATALOG:
246 case L_VERIFY_VOLUME_TO_CATALOG:
248 * send Storage daemon address to the File daemon
250 if (jcr->store->SDDport == 0) {
251 jcr->store->SDDport = jcr->store->SDport;
253 bnet_fsend(fd, storaddr, jcr->store->address, jcr->store->SDDport);
254 if (!response(jcr, fd, OKstore, "Storage", DISPLAY_ERROR)) {
259 * Send the bootstrap file -- what Volumes/files to restore
261 if (!send_bootstrap_file(jcr)) {
265 if (!jcr->RestoreBootstrap) {
266 Jmsg0(jcr, M_FATAL, 0, _("Deprecated feature ... use bootstrap.\n"));
275 case L_VERIFY_DISK_TO_CATALOG:
276 level="disk_to_catalog";
279 Jmsg2(jcr, M_FATAL, 0, _("Unimplemented Verify level %d(%c)\n"), jcr->JobLevel,
284 if (!send_run_before_and_after_commands(jcr)) {
289 * Send verify command/level to File daemon
291 bnet_fsend(fd, verifycmd, level);
292 if (!response(jcr, fd, OKverify, "Verify", DISPLAY_ERROR)) {
297 * Now get data back from File daemon and
298 * compare it to the catalog or store it in the
299 * catalog depending on the run type.
301 /* Compare to catalog */
302 switch (jcr->JobLevel) {
303 case L_VERIFY_CATALOG:
304 Dmsg0(10, "Verify level=catalog\n");
305 jcr->sd_msg_thread_done = true; /* no SD msg thread, so it is done */
306 jcr->SDJobStatus = JS_Terminated;
307 get_attributes_and_compare_to_catalog(jcr, jcr->target_jr.JobId);
310 case L_VERIFY_VOLUME_TO_CATALOG:
311 Dmsg0(10, "Verify level=volume\n");
312 get_attributes_and_compare_to_catalog(jcr, jcr->target_jr.JobId);
315 case L_VERIFY_DISK_TO_CATALOG:
316 Dmsg0(10, "Verify level=disk_to_catalog\n");
317 jcr->sd_msg_thread_done = true; /* no SD msg thread, so it is done */
318 jcr->SDJobStatus = JS_Terminated;
319 get_attributes_and_compare_to_catalog(jcr, jcr->target_jr.JobId);
324 Dmsg0(10, "Verify level=init\n");
325 jcr->sd_msg_thread_done = true; /* no SD msg thread, so it is done */
326 jcr->SDJobStatus = JS_Terminated;
327 get_attributes_and_put_in_catalog(jcr);
331 Jmsg1(jcr, M_FATAL, 0, _("Unimplemented verify level %d\n"), jcr->JobLevel);
335 stat = wait_for_job_termination(jcr);
336 if (stat == JS_Terminated) {
337 verify_cleanup(jcr, stat);
345 * Release resources allocated during backup.
348 void verify_cleanup(JCR *jcr, int TermCode)
350 char sdt[50], edt[50];
351 char ec1[30], ec2[30];
352 char term_code[100], fd_term_msg[100], sd_term_msg[100];
353 const char *term_msg;
358 // Dmsg1(100, "Enter verify_cleanup() TermCod=%d\n", TermCode);
359 dequeue_messages(jcr); /* display any queued messages */
361 Dmsg3(900, "JobLevel=%c Expected=%u JobFiles=%u\n", jcr->JobLevel,
362 jcr->ExpectedFiles, jcr->JobFiles);
363 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG &&
364 jcr->ExpectedFiles != jcr->JobFiles) {
365 TermCode = JS_ErrorTerminated;
368 /* If no files were expected, there can be no error */
369 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG &&
370 jcr->ExpectedFiles == 0) {
371 TermCode = JS_Terminated;
374 JobId = jcr->jr.JobId;
375 set_jcr_job_status(jcr, TermCode);
377 update_job_end_record(jcr);
378 if (jcr->unlink_bsr && jcr->RestoreBootstrap) {
379 unlink(jcr->RestoreBootstrap);
380 jcr->unlink_bsr = false;
383 msg_type = M_INFO; /* by default INFO message */
386 term_msg = _("Verify OK");
389 case JS_ErrorTerminated:
390 term_msg = _("*** Verify Error ***");
391 msg_type = M_ERROR; /* Generate error message */
394 term_msg = _("Verify warnings");
397 term_msg = _("Verify Canceled");
400 term_msg = _("Verify Differences");
403 term_msg = term_code;
404 bsnprintf(term_code, sizeof(term_code),
405 _("Inappropriate term code: %d %c\n"), TermCode, TermCode);
408 bstrftimes(sdt, sizeof(sdt), jcr->jr.StartTime);
409 bstrftimes(edt, sizeof(edt), jcr->jr.EndTime);
410 if (jcr->verify_job) {
411 Name = jcr->verify_job->hdr.name;
416 jobstatus_to_ascii(jcr->FDJobStatus, fd_term_msg, sizeof(fd_term_msg));
417 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG) {
418 jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg));
419 Jmsg(jcr, msg_type, 0, _("Bacula %s (%s): %s\n"
423 " Verify Level: %s\n"
425 " Verify JobId: %d\n"
429 " Files Expected: %s\n"
430 " Files Examined: %s\n"
431 " Non-fatal FD errors: %d\n"
432 " FD termination status: %s\n"
433 " SD termination status: %s\n"
434 " Termination: %s\n\n"),
440 jcr->fileset->hdr.name,
441 level_to_str(jcr->JobLevel),
442 jcr->client->hdr.name,
443 jcr->target_jr.JobId,
447 edit_uint64_with_commas(jcr->ExpectedFiles, ec1),
448 edit_uint64_with_commas(jcr->JobFiles, ec2),
454 Jmsg(jcr, msg_type, 0, _("Bacula %s (%s): %s\n"
458 " Verify Level: %s\n"
460 " Verify JobId: %d\n"
464 " Files Examined: %s\n"
465 " Non-fatal FD errors: %d\n"
466 " FD termination status: %s\n"
467 " Termination: %s\n\n"),
473 jcr->fileset->hdr.name,
474 level_to_str(jcr->JobLevel),
475 jcr->client->hdr.name,
476 jcr->target_jr.JobId,
480 edit_uint64_with_commas(jcr->JobFiles, ec1),
485 Dmsg0(100, "Leave verify_cleanup()\n");
489 * This routine is called only during a Verify
491 int get_attributes_and_compare_to_catalog(JCR *jcr, JobId_t JobId)
496 struct stat statf; /* file stat */
497 struct stat statc; /* catalog stat */
498 int stat = JS_Terminated;
500 POOLMEM *fname = get_pool_memory(PM_MESSAGE);
502 int32_t file_index = 0;
504 memset(&fdbr, 0, sizeof(FILE_DBR));
505 fd = jcr->file_bsock;
509 Dmsg0(20, "bdird: waiting to receive file attributes\n");
511 * Get Attributes and Signature from File daemon
515 * Options or SIG (MD5/SHA1)
520 while ((n=bget_dirmsg(fd)) >= 0 && !job_canceled(jcr)) {
523 char Opts_SIG[MAXSTRING]; /* Verify Opts or MD5/SHA1 signature */
525 fname = check_pool_memory_size(fname, fd->msglen);
526 jcr->fname = check_pool_memory_size(jcr->fname, fd->msglen);
527 Dmsg1(200, "Atts+SIG=%s\n", fd->msg);
528 if ((len = sscanf(fd->msg, "%ld %d %100s", &file_index, &stream,
530 Jmsg3(jcr, M_FATAL, 0, _("bird<filed: bad attributes, expected 3 fields got %d\n"
531 " mslen=%d msg=%s\n"), len, fd->msglen, fd->msg);
535 * We read the Options or Signature into fname
536 * to prevent overrun, now copy it to proper location.
538 bstrncpy(Opts_SIG, fname, sizeof(Opts_SIG));
540 skip_nonspaces(&p); /* skip FileIndex */
542 skip_nonspaces(&p); /* skip Stream */
544 skip_nonspaces(&p); /* skip Opts_SIG */
545 p++; /* skip space */
548 *fn++ = *p++; /* copy filename */
550 *fn = *p++; /* term filename and point to attribs */
553 * Got attributes stream, decode it
555 if (stream == STREAM_UNIX_ATTRIBUTES || stream == STREAM_UNIX_ATTRIBUTES_EX) {
556 int32_t LinkFIf, LinkFIc;
557 Dmsg2(400, "file_index=%d attr=%s\n", file_index, attr);
559 jcr->FileIndex = file_index; /* remember attribute file_index */
560 decode_stat(attr, &statf, &LinkFIf); /* decode file stat packet */
562 jcr->fn_printed = false;
563 pm_strcpy(jcr->fname, fname); /* move filename into JCR */
565 Dmsg2(040, "dird<filed: stream=%d %s\n", stream, jcr->fname);
566 Dmsg1(020, "dird<filed: attr=%s\n", attr);
569 * Find equivalent record in the database
572 if (!db_get_file_attributes_record(jcr, jcr->db, jcr->fname,
573 &jcr->target_jr, &fdbr)) {
574 Jmsg(jcr, M_INFO, 0, _("New file: %s\n"), jcr->fname);
575 Dmsg1(020, _("File not in catalog: %s\n"), jcr->fname);
576 stat = JS_Differences;
580 * mark file record as visited by stuffing the
581 * current JobId, which is unique, into the MarkId field.
583 db_mark_file_record(jcr, jcr->db, fdbr.FileId, jcr->JobId);
586 Dmsg3(400, "Found %s in catalog. inx=%d Opts=%s\n", jcr->fname,
587 file_index, Opts_SIG);
588 decode_stat(fdbr.LStat, &statc, &LinkFIc); /* decode catalog stat */
590 * Loop over options supplied by user and verify the
591 * fields he requests.
593 for (p=Opts_SIG; *p; p++) {
594 char ed1[30], ed2[30];
596 case 'i': /* compare INODEs */
597 if (statc.st_ino != statf.st_ino) {
599 Jmsg(jcr, M_INFO, 0, _(" st_ino differ. Cat: %s File: %s\n"),
600 edit_uint64((uint64_t)statc.st_ino, ed1),
601 edit_uint64((uint64_t)statf.st_ino, ed2));
602 stat = JS_Differences;
605 case 'p': /* permissions bits */
606 if (statc.st_mode != statf.st_mode) {
608 Jmsg(jcr, M_INFO, 0, _(" st_mode differ. Cat: %x File: %x\n"),
609 (uint32_t)statc.st_mode, (uint32_t)statf.st_mode);
610 stat = JS_Differences;
613 case 'n': /* number of links */
614 if (statc.st_nlink != statf.st_nlink) {
616 Jmsg(jcr, M_INFO, 0, _(" st_nlink differ. Cat: %d File: %d\n"),
617 (uint32_t)statc.st_nlink, (uint32_t)statf.st_nlink);
618 stat = JS_Differences;
621 case 'u': /* user id */
622 if (statc.st_uid != statf.st_uid) {
624 Jmsg(jcr, M_INFO, 0, _(" st_uid differ. Cat: %u File: %u\n"),
625 (uint32_t)statc.st_uid, (uint32_t)statf.st_uid);
626 stat = JS_Differences;
629 case 'g': /* group id */
630 if (statc.st_gid != statf.st_gid) {
632 Jmsg(jcr, M_INFO, 0, _(" st_gid differ. Cat: %u File: %u\n"),
633 (uint32_t)statc.st_gid, (uint32_t)statf.st_gid);
634 stat = JS_Differences;
638 if (statc.st_size != statf.st_size) {
640 Jmsg(jcr, M_INFO, 0, _(" st_size differ. Cat: %s File: %s\n"),
641 edit_uint64((uint64_t)statc.st_size, ed1),
642 edit_uint64((uint64_t)statf.st_size, ed2));
643 stat = JS_Differences;
646 case 'a': /* access time */
647 if (statc.st_atime != statf.st_atime) {
649 Jmsg(jcr, M_INFO, 0, _(" st_atime differs\n"));
650 stat = JS_Differences;
654 if (statc.st_mtime != statf.st_mtime) {
656 Jmsg(jcr, M_INFO, 0, _(" st_mtime differs\n"));
657 stat = JS_Differences;
660 case 'c': /* ctime */
661 if (statc.st_ctime != statf.st_ctime) {
663 Jmsg(jcr, M_INFO, 0, _(" st_ctime differs\n"));
664 stat = JS_Differences;
667 case 'd': /* file size decrease */
668 if (statc.st_size > statf.st_size) {
670 Jmsg(jcr, M_INFO, 0, _(" st_size decrease. Cat: %s File: %s\n"),
671 edit_uint64((uint64_t)statc.st_size, ed1),
672 edit_uint64((uint64_t)statf.st_size, ed2));
673 stat = JS_Differences;
676 case '5': /* compare MD5 */
677 Dmsg1(500, "set Do_MD5 for %s\n", jcr->fname);
680 case '1': /* compare SHA1 */
690 * Got SIG Signature from Storage daemon
691 * It came across in the Opts_SIG field.
693 } else if (stream == STREAM_MD5_SIGNATURE || stream == STREAM_SHA1_SIGNATURE) {
694 Dmsg2(400, "stream=SIG inx=%d SIG=%s\n", file_index, Opts_SIG);
696 * When ever we get a signature is MUST have been
697 * preceded by an attributes record, which sets attr_file_index
699 if (jcr->FileIndex != (uint32_t)file_index) {
700 Jmsg2(jcr, M_FATAL, 0, _("MD5/SHA1 index %d not same as attributes %d\n"),
701 file_index, jcr->FileIndex);
705 db_escape_string(buf, Opts_SIG, strlen(Opts_SIG));
706 if (strcmp(buf, fdbr.SIG) != 0) {
708 if (debug_level >= 10) {
709 Jmsg(jcr, M_INFO, 0, _(" %s not same. File=%s Cat=%s\n"),
710 stream==STREAM_MD5_SIGNATURE?"MD5":"SHA1", buf, fdbr.SIG);
712 Jmsg(jcr, M_INFO, 0, _(" %s differs.\n"),
713 stream==STREAM_MD5_SIGNATURE?"MD5":"SHA1");
715 stat = JS_Differences;
720 jcr->JobFiles = file_index;
722 if (is_bnet_error(fd)) {
724 Jmsg2(jcr, M_FATAL, 0, _("bdird<filed: bad attributes from filed n=%d : %s\n"),
729 /* Now find all the files that are missing -- i.e. all files in
730 * the database where the MarkedId != current JobId
732 jcr->fn_printed = false;
733 bsnprintf(buf, sizeof(buf),
734 "SELECT Path.Path,Filename.Name FROM File,Path,Filename "
735 "WHERE File.JobId=%d "
736 "AND File.MarkedId!=%d AND File.PathId=Path.PathId "
737 "AND File.FilenameId=Filename.FilenameId",
739 /* missing_handler is called for each file found */
740 db_sql_query(jcr->db, buf, missing_handler, (void *)jcr);
741 if (jcr->fn_printed) {
742 stat = JS_Differences;
744 free_pool_memory(fname);
745 set_jcr_job_status(jcr, stat);
746 return stat == JS_Terminated;
750 * We are called here for each record that matches the above
751 * SQL query -- that is for each file contained in the Catalog
752 * that was not marked earlier. This means that the file in
753 * question is a missing file (in the Catalog but not on Disk).
755 static int missing_handler(void *ctx, int num_fields, char **row)
757 JCR *jcr = (JCR *)ctx;
759 if (!jcr->fn_printed) {
760 Jmsg(jcr, M_INFO, 0, "\n");
761 Jmsg(jcr, M_INFO, 0, _("The following files are missing:\n"));
762 jcr->fn_printed = true;
764 Jmsg(jcr, M_INFO, 0, " %s%s\n", row[0]?row[0]:"", row[1]?row[1]:"");
770 * Print filename for verify
772 static void prt_fname(JCR *jcr)
774 if (!jcr->fn_printed) {
775 Jmsg(jcr, M_INFO, 0, _("File: %s\n"), jcr->fname);
776 jcr->fn_printed = TRUE;