3 * Bacula Director -- verify.c -- responsible for running file verification
5 * Kern Sibbald, October MM
7 * Basic tasks done here:
9 * Open connection with File daemon and pass him commands
11 * When the File daemon sends the attributes, compare them to
17 Copyright (C) 2000-2005 Kern Sibbald
19 This program is free software; you can redistribute it and/or
20 modify it under the terms of the GNU General Public License
21 version 2 as amended with additional clauses defined in the
22 file LICENSE in the main source directory.
24 This program is distributed in the hope that it will be useful,
25 but WITHOUT ANY WARRANTY; without even the implied warranty of
26 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
27 the file LICENSE for additional details.
34 #include "findlib/find.h"
36 /* Imported Global Variables */
37 extern int debug_level;
39 /* Commands sent to File daemon */
40 static char verifycmd[] = "verify level=%s\n";
41 static char storaddr[] = "storage address=%s port=%d ssl=0\n";
43 /* Responses received from File daemon */
44 static char OKverify[] = "2000 OK verify\n";
45 static char OKstore[] = "2000 OK storage\n";
47 /* Forward referenced functions */
48 static void prt_fname(JCR *jcr);
49 static int missing_handler(void *ctx, int num_fields, char **row);
53 * Called here before the job is run to do the job
56 bool do_verify_init(JCR *jcr)
59 JobId_t verify_jobid = 0;
62 memset(&jcr->target_jr, 0, sizeof(jcr->target_jr));
64 Dmsg1(9, "bdird: created client %s record\n", jcr->client->hdr.name);
67 * Find JobId of last job that ran. E.g.
68 * for VERIFY_CATALOG we want the JobId of the last INIT.
69 * for VERIFY_VOLUME_TO_CATALOG, we want the JobId of the
72 if (jcr->JobLevel == L_VERIFY_CATALOG ||
73 jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG ||
74 jcr->JobLevel == L_VERIFY_DISK_TO_CATALOG) {
75 memcpy(&jr, &jcr->jr, sizeof(jr));
76 if (jcr->verify_job &&
77 (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG ||
78 jcr->JobLevel == L_VERIFY_DISK_TO_CATALOG)) {
79 Name = jcr->verify_job->hdr.name;
83 Dmsg1(100, "find last jobid for: %s\n", NPRT(Name));
84 if (!db_find_last_jobid(jcr, jcr->db, Name, &jr)) {
85 if (jcr->JobLevel == L_VERIFY_CATALOG) {
86 Jmsg(jcr, M_FATAL, 0, _(
87 "Unable to find JobId of previous InitCatalog Job.\n"
88 "Please run a Verify with Level=InitCatalog before\n"
89 "running the current Job.\n"));
91 Jmsg(jcr, M_FATAL, 0, _(
92 "Unable to find JobId of previous Job for this client.\n"));
96 verify_jobid = jr.JobId;
97 Dmsg1(100, "Last full jobid=%d\n", verify_jobid);
100 * Now get the job record for the previous backup that interests
101 * us. We use the verify_jobid that we found above.
103 if (jcr->JobLevel == L_VERIFY_CATALOG ||
104 jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG ||
105 jcr->JobLevel == L_VERIFY_DISK_TO_CATALOG) {
106 jcr->target_jr.JobId = verify_jobid;
107 if (!db_get_job_record(jcr, jcr->db, &jcr->target_jr)) {
108 Jmsg(jcr, M_FATAL, 0, _("Could not get job record for previous Job. ERR=%s"),
109 db_strerror(jcr->db));
112 if (jcr->target_jr.JobStatus != 'T') {
113 Jmsg(jcr, M_FATAL, 0, _("Last Job %d did not terminate normally. JobStatus=%c\n"),
114 verify_jobid, jcr->target_jr.JobStatus);
117 Jmsg(jcr, M_INFO, 0, _("Verifying against JobId=%d Job=%s\n"),
118 jcr->target_jr.JobId, jcr->target_jr.Job);
122 * If we are verifying a Volume, we need the Storage
123 * daemon, so open a connection, otherwise, just
124 * create a dummy authorization key (passed to
125 * File daemon but not used).
127 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG) {
128 RBSR *bsr = new_bsr();
130 bsr->JobId = jcr->target_jr.JobId;
131 ua = new_ua_context(jcr);
132 complete_bsr(ua, bsr);
133 bsr->fi = new_findex();
135 bsr->fi->findex2 = jcr->target_jr.JobFiles;
136 jcr->ExpectedFiles = write_bsr_file(ua, bsr);
137 if (jcr->ExpectedFiles == 0) {
142 if (jcr->RestoreBootstrap) {
143 free(jcr->RestoreBootstrap);
145 POOLMEM *fname = get_pool_memory(PM_MESSAGE);
146 make_unique_restore_filename(ua, &fname);
147 jcr->RestoreBootstrap = bstrdup(fname);
150 free_pool_memory(fname);
151 jcr->needs_sd = true;
154 jcr->sd_auth_key = bstrdup("dummy"); /* dummy Storage daemon key */
157 if (jcr->JobLevel == L_VERIFY_DISK_TO_CATALOG && jcr->verify_job) {
158 jcr->fileset = jcr->verify_job->fileset;
160 Dmsg2(100, "ClientId=%u JobLevel=%c\n", jcr->target_jr.ClientId, jcr->JobLevel);
166 * Do a verification of the specified files against the Catlaog
168 * Returns: false on failure
171 bool do_verify(JCR *jcr)
177 if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) {
178 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db));
182 /* Print Job Start message */
183 Jmsg(jcr, M_INFO, 0, _("Start Verify JobId=%d Level=%s Job=%s\n"),
184 jcr->JobId, level_to_str(jcr->JobLevel), jcr->Job);
186 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG) {
188 * Start conversation with Storage daemon
190 set_jcr_job_status(jcr, JS_Blocked);
191 if (!connect_to_storage_daemon(jcr, 10, SDConnectTimeout, 1)) {
195 * Now start a job with the Storage daemon
197 if (!start_storage_daemon_job(jcr, jcr->storage, SD_READ)) {
201 * Now start a Storage daemon message thread
203 if (!start_storage_daemon_message_thread(jcr)) {
206 Dmsg0(50, "Storage daemon connection OK\n");
209 * OK, now connect to the File daemon
210 * and ask him for the files.
212 set_jcr_job_status(jcr, JS_Blocked);
213 if (!connect_to_file_daemon(jcr, 10, FDConnectTimeout, 1)) {
217 set_jcr_job_status(jcr, JS_Running);
218 fd = jcr->file_bsock;
221 Dmsg0(30, ">filed: Send include list\n");
222 if (!send_include_list(jcr)) {
226 Dmsg0(30, ">filed: Send exclude list\n");
227 if (!send_exclude_list(jcr)) {
232 * Send Level command to File daemon, as well
233 * as the Storage address if appropriate.
235 switch (jcr->JobLevel) {
239 case L_VERIFY_CATALOG:
242 case L_VERIFY_VOLUME_TO_CATALOG:
244 * send Storage daemon address to the File daemon
246 if (jcr->store->SDDport == 0) {
247 jcr->store->SDDport = jcr->store->SDport;
249 bnet_fsend(fd, storaddr, jcr->store->address, jcr->store->SDDport);
250 if (!response(jcr, fd, OKstore, "Storage", DISPLAY_ERROR)) {
255 * Send the bootstrap file -- what Volumes/files to restore
257 if (!send_bootstrap_file(jcr)) {
261 if (!jcr->RestoreBootstrap) {
262 Jmsg0(jcr, M_FATAL, 0, _("Deprecated feature ... use bootstrap.\n"));
271 case L_VERIFY_DISK_TO_CATALOG:
272 level="disk_to_catalog";
275 Jmsg2(jcr, M_FATAL, 0, _("Unimplemented Verify level %d(%c)\n"), jcr->JobLevel,
280 if (!send_run_before_and_after_commands(jcr)) {
285 * Send verify command/level to File daemon
287 bnet_fsend(fd, verifycmd, level);
288 if (!response(jcr, fd, OKverify, "Verify", DISPLAY_ERROR)) {
293 * Now get data back from File daemon and
294 * compare it to the catalog or store it in the
295 * catalog depending on the run type.
297 /* Compare to catalog */
298 switch (jcr->JobLevel) {
299 case L_VERIFY_CATALOG:
300 Dmsg0(10, "Verify level=catalog\n");
301 jcr->sd_msg_thread_done = true; /* no SD msg thread, so it is done */
302 jcr->SDJobStatus = JS_Terminated;
303 get_attributes_and_compare_to_catalog(jcr, jcr->target_jr.JobId);
306 case L_VERIFY_VOLUME_TO_CATALOG:
307 Dmsg0(10, "Verify level=volume\n");
308 get_attributes_and_compare_to_catalog(jcr, jcr->target_jr.JobId);
311 case L_VERIFY_DISK_TO_CATALOG:
312 Dmsg0(10, "Verify level=disk_to_catalog\n");
313 jcr->sd_msg_thread_done = true; /* no SD msg thread, so it is done */
314 jcr->SDJobStatus = JS_Terminated;
315 get_attributes_and_compare_to_catalog(jcr, jcr->target_jr.JobId);
320 Dmsg0(10, "Verify level=init\n");
321 jcr->sd_msg_thread_done = true; /* no SD msg thread, so it is done */
322 jcr->SDJobStatus = JS_Terminated;
323 get_attributes_and_put_in_catalog(jcr);
327 Jmsg1(jcr, M_FATAL, 0, _("Unimplemented verify level %d\n"), jcr->JobLevel);
331 stat = wait_for_job_termination(jcr);
332 if (stat == JS_Terminated) {
333 verify_cleanup(jcr, stat);
341 * Release resources allocated during backup.
344 void verify_cleanup(JCR *jcr, int TermCode)
346 char sdt[50], edt[50];
347 char ec1[30], ec2[30];
348 char term_code[100], fd_term_msg[100], sd_term_msg[100];
349 const char *term_msg;
354 // Dmsg1(100, "Enter verify_cleanup() TermCod=%d\n", TermCode);
355 dequeue_messages(jcr); /* display any queued messages */
357 Dmsg3(900, "JobLevel=%c Expected=%u JobFiles=%u\n", jcr->JobLevel,
358 jcr->ExpectedFiles, jcr->JobFiles);
359 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG &&
360 jcr->ExpectedFiles != jcr->JobFiles) {
361 TermCode = JS_ErrorTerminated;
364 /* If no files were expected, there can be no error */
365 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG &&
366 jcr->ExpectedFiles == 0) {
367 TermCode = JS_Terminated;
370 JobId = jcr->jr.JobId;
371 set_jcr_job_status(jcr, TermCode);
373 update_job_end_record(jcr);
374 if (jcr->unlink_bsr && jcr->RestoreBootstrap) {
375 unlink(jcr->RestoreBootstrap);
376 jcr->unlink_bsr = false;
379 msg_type = M_INFO; /* by default INFO message */
382 term_msg = _("Verify OK");
385 case JS_ErrorTerminated:
386 term_msg = _("*** Verify Error ***");
387 msg_type = M_ERROR; /* Generate error message */
390 term_msg = _("Verify warnings");
393 term_msg = _("Verify Canceled");
396 term_msg = _("Verify Differences");
399 term_msg = term_code;
400 bsnprintf(term_code, sizeof(term_code),
401 _("Inappropriate term code: %d %c\n"), TermCode, TermCode);
404 bstrftimes(sdt, sizeof(sdt), jcr->jr.StartTime);
405 bstrftimes(edt, sizeof(edt), jcr->jr.EndTime);
406 if (jcr->verify_job) {
407 Name = jcr->verify_job->hdr.name;
412 jobstatus_to_ascii(jcr->FDJobStatus, fd_term_msg, sizeof(fd_term_msg));
413 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG) {
414 jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg));
415 Jmsg(jcr, msg_type, 0, _("Bacula " VERSION " (" LSMDATE "): %s\n"
419 " Verify Level: %s\n"
421 " Verify JobId: %d\n"
425 " Files Expected: %s\n"
426 " Files Examined: %s\n"
427 " Non-fatal FD errors: %d\n"
428 " FD termination status: %s\n"
429 " SD termination status: %s\n"
430 " Termination: %s\n\n"),
434 jcr->fileset->hdr.name,
435 level_to_str(jcr->JobLevel),
436 jcr->client->hdr.name,
437 jcr->target_jr.JobId,
441 edit_uint64_with_commas(jcr->ExpectedFiles, ec1),
442 edit_uint64_with_commas(jcr->JobFiles, ec2),
448 Jmsg(jcr, msg_type, 0, _("Bacula " VERSION " (" LSMDATE "): %s\n"
452 " Verify Level: %s\n"
454 " Verify JobId: %d\n"
458 " Files Examined: %s\n"
459 " Non-fatal FD errors: %d\n"
460 " FD termination status: %s\n"
461 " Termination: %s\n\n"),
465 jcr->fileset->hdr.name,
466 level_to_str(jcr->JobLevel),
467 jcr->client->hdr.name,
468 jcr->target_jr.JobId,
472 edit_uint64_with_commas(jcr->JobFiles, ec1),
477 Dmsg0(100, "Leave verify_cleanup()\n");
481 * This routine is called only during a Verify
483 int get_attributes_and_compare_to_catalog(JCR *jcr, JobId_t JobId)
488 struct stat statf; /* file stat */
489 struct stat statc; /* catalog stat */
490 int stat = JS_Terminated;
492 POOLMEM *fname = get_pool_memory(PM_MESSAGE);
494 int32_t file_index = 0;
496 memset(&fdbr, 0, sizeof(FILE_DBR));
497 fd = jcr->file_bsock;
501 Dmsg0(20, "bdird: waiting to receive file attributes\n");
503 * Get Attributes and Signature from File daemon
507 * Options or SIG (MD5/SHA1)
512 while ((n=bget_dirmsg(fd)) >= 0 && !job_canceled(jcr)) {
515 char Opts_SIG[MAXSTRING]; /* Verify Opts or MD5/SHA1 signature */
517 fname = check_pool_memory_size(fname, fd->msglen);
518 jcr->fname = check_pool_memory_size(jcr->fname, fd->msglen);
519 Dmsg1(200, "Atts+SIG=%s\n", fd->msg);
520 if ((len = sscanf(fd->msg, "%ld %d %100s", &file_index, &stream,
522 Jmsg3(jcr, M_FATAL, 0, _("bird<filed: bad attributes, expected 3 fields got %d\n"
523 " mslen=%d msg=%s\n"), len, fd->msglen, fd->msg);
527 * We read the Options or Signature into fname
528 * to prevent overrun, now copy it to proper location.
530 bstrncpy(Opts_SIG, fname, sizeof(Opts_SIG));
532 skip_nonspaces(&p); /* skip FileIndex */
534 skip_nonspaces(&p); /* skip Stream */
536 skip_nonspaces(&p); /* skip Opts_SIG */
537 p++; /* skip space */
540 *fn++ = *p++; /* copy filename */
542 *fn = *p++; /* term filename and point to attribs */
545 * Got attributes stream, decode it
547 if (stream == STREAM_UNIX_ATTRIBUTES || stream == STREAM_UNIX_ATTRIBUTES_EX) {
548 int32_t LinkFIf, LinkFIc;
549 Dmsg2(400, "file_index=%d attr=%s\n", file_index, attr);
551 jcr->FileIndex = file_index; /* remember attribute file_index */
552 decode_stat(attr, &statf, &LinkFIf); /* decode file stat packet */
554 jcr->fn_printed = false;
555 pm_strcpy(jcr->fname, fname); /* move filename into JCR */
557 Dmsg2(040, "dird<filed: stream=%d %s\n", stream, jcr->fname);
558 Dmsg1(020, "dird<filed: attr=%s\n", attr);
561 * Find equivalent record in the database
564 if (!db_get_file_attributes_record(jcr, jcr->db, jcr->fname,
565 &jcr->target_jr, &fdbr)) {
566 Jmsg(jcr, M_INFO, 0, _("New file: %s\n"), jcr->fname);
567 Dmsg1(020, _("File not in catalog: %s\n"), jcr->fname);
568 stat = JS_Differences;
572 * mark file record as visited by stuffing the
573 * current JobId, which is unique, into the MarkId field.
575 db_mark_file_record(jcr, jcr->db, fdbr.FileId, jcr->JobId);
578 Dmsg3(400, "Found %s in catalog. inx=%d Opts=%s\n", jcr->fname,
579 file_index, Opts_SIG);
580 decode_stat(fdbr.LStat, &statc, &LinkFIc); /* decode catalog stat */
582 * Loop over options supplied by user and verify the
583 * fields he requests.
585 for (p=Opts_SIG; *p; p++) {
586 char ed1[30], ed2[30];
588 case 'i': /* compare INODEs */
589 if (statc.st_ino != statf.st_ino) {
591 Jmsg(jcr, M_INFO, 0, _(" st_ino differ. Cat: %s File: %s\n"),
592 edit_uint64((uint64_t)statc.st_ino, ed1),
593 edit_uint64((uint64_t)statf.st_ino, ed2));
594 stat = JS_Differences;
597 case 'p': /* permissions bits */
598 if (statc.st_mode != statf.st_mode) {
600 Jmsg(jcr, M_INFO, 0, _(" st_mode differ. Cat: %x File: %x\n"),
601 (uint32_t)statc.st_mode, (uint32_t)statf.st_mode);
602 stat = JS_Differences;
605 case 'n': /* number of links */
606 if (statc.st_nlink != statf.st_nlink) {
608 Jmsg(jcr, M_INFO, 0, _(" st_nlink differ. Cat: %d File: %d\n"),
609 (uint32_t)statc.st_nlink, (uint32_t)statf.st_nlink);
610 stat = JS_Differences;
613 case 'u': /* user id */
614 if (statc.st_uid != statf.st_uid) {
616 Jmsg(jcr, M_INFO, 0, _(" st_uid differ. Cat: %u File: %u\n"),
617 (uint32_t)statc.st_uid, (uint32_t)statf.st_uid);
618 stat = JS_Differences;
621 case 'g': /* group id */
622 if (statc.st_gid != statf.st_gid) {
624 Jmsg(jcr, M_INFO, 0, _(" st_gid differ. Cat: %u File: %u\n"),
625 (uint32_t)statc.st_gid, (uint32_t)statf.st_gid);
626 stat = JS_Differences;
630 if (statc.st_size != statf.st_size) {
632 Jmsg(jcr, M_INFO, 0, _(" st_size differ. Cat: %s File: %s\n"),
633 edit_uint64((uint64_t)statc.st_size, ed1),
634 edit_uint64((uint64_t)statf.st_size, ed2));
635 stat = JS_Differences;
638 case 'a': /* access time */
639 if (statc.st_atime != statf.st_atime) {
641 Jmsg(jcr, M_INFO, 0, _(" st_atime differs\n"));
642 stat = JS_Differences;
646 if (statc.st_mtime != statf.st_mtime) {
648 Jmsg(jcr, M_INFO, 0, _(" st_mtime differs\n"));
649 stat = JS_Differences;
652 case 'c': /* ctime */
653 if (statc.st_ctime != statf.st_ctime) {
655 Jmsg(jcr, M_INFO, 0, _(" st_ctime differs\n"));
656 stat = JS_Differences;
659 case 'd': /* file size decrease */
660 if (statc.st_size > statf.st_size) {
662 Jmsg(jcr, M_INFO, 0, _(" st_size decrease. Cat: %s File: %s\n"),
663 edit_uint64((uint64_t)statc.st_size, ed1),
664 edit_uint64((uint64_t)statf.st_size, ed2));
665 stat = JS_Differences;
668 case '5': /* compare MD5 */
669 Dmsg1(500, "set Do_MD5 for %s\n", jcr->fname);
672 case '1': /* compare SHA1 */
682 * Got SIG Signature from Storage daemon
683 * It came across in the Opts_SIG field.
685 } else if (stream == STREAM_MD5_SIGNATURE || stream == STREAM_SHA1_SIGNATURE) {
686 Dmsg2(400, "stream=SIG inx=%d SIG=%s\n", file_index, Opts_SIG);
688 * When ever we get a signature is MUST have been
689 * preceded by an attributes record, which sets attr_file_index
691 if (jcr->FileIndex != (uint32_t)file_index) {
692 Jmsg2(jcr, M_FATAL, 0, _("MD5/SHA1 index %d not same as attributes %d\n"),
693 file_index, jcr->FileIndex);
697 db_escape_string(buf, Opts_SIG, strlen(Opts_SIG));
698 if (strcmp(buf, fdbr.SIG) != 0) {
700 if (debug_level >= 10) {
701 Jmsg(jcr, M_INFO, 0, _(" %s not same. File=%s Cat=%s\n"),
702 stream==STREAM_MD5_SIGNATURE?"MD5":"SHA1", buf, fdbr.SIG);
704 Jmsg(jcr, M_INFO, 0, _(" %s differs.\n"),
705 stream==STREAM_MD5_SIGNATURE?"MD5":"SHA1");
707 stat = JS_Differences;
712 jcr->JobFiles = file_index;
714 if (is_bnet_error(fd)) {
716 Jmsg2(jcr, M_FATAL, 0, _("bdird<filed: bad attributes from filed n=%d : %s\n"),
721 /* Now find all the files that are missing -- i.e. all files in
722 * the database where the MarkedId != current JobId
724 jcr->fn_printed = false;
725 bsnprintf(buf, sizeof(buf),
726 "SELECT Path.Path,Filename.Name FROM File,Path,Filename "
727 "WHERE File.JobId=%d "
728 "AND File.MarkedId!=%d AND File.PathId=Path.PathId "
729 "AND File.FilenameId=Filename.FilenameId",
731 /* missing_handler is called for each file found */
732 db_sql_query(jcr->db, buf, missing_handler, (void *)jcr);
733 if (jcr->fn_printed) {
734 stat = JS_Differences;
736 free_pool_memory(fname);
737 set_jcr_job_status(jcr, stat);
738 return stat == JS_Terminated;
742 * We are called here for each record that matches the above
743 * SQL query -- that is for each file contained in the Catalog
744 * that was not marked earlier. This means that the file in
745 * question is a missing file (in the Catalog but not on Disk).
747 static int missing_handler(void *ctx, int num_fields, char **row)
749 JCR *jcr = (JCR *)ctx;
751 if (!jcr->fn_printed) {
752 Jmsg(jcr, M_INFO, 0, "\n");
753 Jmsg(jcr, M_INFO, 0, _("The following files are missing:\n"));
754 jcr->fn_printed = true;
756 Jmsg(jcr, M_INFO, 0, " %s%s\n", row[0]?row[0]:"", row[1]?row[1]:"");
762 * Print filename for verify
764 static void prt_fname(JCR *jcr)
766 if (!jcr->fn_printed) {
767 Jmsg(jcr, M_INFO, 0, _("File: %s\n"), jcr->fname);
768 jcr->fn_printed = TRUE;