3 * Bacula Director -- verify.c -- responsible for running file verification
5 * Kern Sibbald, October MM
7 * Basic tasks done here:
9 * Open connection with File daemon and pass him commands
11 * When the File daemon sends the attributes, compare them to
18 Copyright (C) 2000-2005 Kern Sibbald
20 This program is free software; you can redistribute it and/or
21 modify it under the terms of the GNU General Public License as
22 published by the Free Software Foundation; either version 2 of
23 the License, or (at your option) any later version.
25 This program is distributed in the hope that it will be useful,
26 but WITHOUT ANY WARRANTY; without even the implied warranty of
27 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
28 General Public License for more details.
30 You should have received a copy of the GNU General Public
31 License along with this program; if not, write to the Free
32 Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
39 #include "findlib/find.h"
41 /* Imported Global Variables */
42 extern int debug_level;
44 /* Commands sent to File daemon */
45 static char verifycmd[] = "verify level=%s\n";
46 static char storaddr[] = "storage address=%s port=%d ssl=0\n";
48 /* Responses received from File daemon */
49 static char OKverify[] = "2000 OK verify\n";
50 static char OKstore[] = "2000 OK storage\n";
52 /* Forward referenced functions */
53 static void prt_fname(JCR *jcr);
54 static int missing_handler(void *ctx, int num_fields, char **row);
58 * Called here before the job is run to do the job
61 bool do_verify_init(JCR *jcr)
64 JobId_t verify_jobid = 0;
67 memset(&jcr->target_jr, 0, sizeof(jcr->target_jr));
69 Dmsg1(9, "bdird: created client %s record\n", jcr->client->hdr.name);
72 * Find JobId of last job that ran. E.g.
73 * for VERIFY_CATALOG we want the JobId of the last INIT.
74 * for VERIFY_VOLUME_TO_CATALOG, we want the JobId of the
77 if (jcr->JobLevel == L_VERIFY_CATALOG ||
78 jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG ||
79 jcr->JobLevel == L_VERIFY_DISK_TO_CATALOG) {
80 memcpy(&jr, &jcr->jr, sizeof(jr));
81 if (jcr->verify_job &&
82 (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG ||
83 jcr->JobLevel == L_VERIFY_DISK_TO_CATALOG)) {
84 Name = jcr->verify_job->hdr.name;
88 Dmsg1(100, "find last jobid for: %s\n", NPRT(Name));
89 if (!db_find_last_jobid(jcr, jcr->db, Name, &jr)) {
90 if (jcr->JobLevel == L_VERIFY_CATALOG) {
91 Jmsg(jcr, M_FATAL, 0, _(
92 "Unable to find JobId of previous InitCatalog Job.\n"
93 "Please run a Verify with Level=InitCatalog before\n"
94 "running the current Job.\n"));
96 Jmsg(jcr, M_FATAL, 0, _(
97 "Unable to find JobId of previous Job for this client.\n"));
101 verify_jobid = jr.JobId;
102 Dmsg1(100, "Last full jobid=%d\n", verify_jobid);
105 * Now get the job record for the previous backup that interests
106 * us. We use the verify_jobid that we found above.
108 if (jcr->JobLevel == L_VERIFY_CATALOG ||
109 jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG ||
110 jcr->JobLevel == L_VERIFY_DISK_TO_CATALOG) {
111 jcr->target_jr.JobId = verify_jobid;
112 if (!db_get_job_record(jcr, jcr->db, &jcr->target_jr)) {
113 Jmsg(jcr, M_FATAL, 0, _("Could not get job record for previous Job. ERR=%s"),
114 db_strerror(jcr->db));
117 if (jcr->target_jr.JobStatus != 'T') {
118 Jmsg(jcr, M_FATAL, 0, _("Last Job %d did not terminate normally. JobStatus=%c\n"),
119 verify_jobid, jcr->target_jr.JobStatus);
122 Jmsg(jcr, M_INFO, 0, _("Verifying against JobId=%d Job=%s\n"),
123 jcr->target_jr.JobId, jcr->target_jr.Job);
127 * If we are verifying a Volume, we need the Storage
128 * daemon, so open a connection, otherwise, just
129 * create a dummy authorization key (passed to
130 * File daemon but not used).
132 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG) {
133 RBSR *bsr = new_bsr();
135 bsr->JobId = jcr->target_jr.JobId;
136 ua = new_ua_context(jcr);
137 complete_bsr(ua, bsr);
138 bsr->fi = new_findex();
140 bsr->fi->findex2 = jcr->target_jr.JobFiles;
141 jcr->ExpectedFiles = write_bsr_file(ua, bsr);
142 if (jcr->ExpectedFiles == 0) {
149 if (jcr->RestoreBootstrap) {
150 free(jcr->RestoreBootstrap);
152 POOLMEM *fname = get_pool_memory(PM_MESSAGE);
153 Mmsg(fname, "%s/restore.bsr", working_directory);
154 jcr->RestoreBootstrap = bstrdup(fname);
155 free_pool_memory(fname);
156 jcr->needs_sd = true;
159 jcr->sd_auth_key = bstrdup("dummy"); /* dummy Storage daemon key */
162 if (jcr->JobLevel == L_VERIFY_DISK_TO_CATALOG && jcr->verify_job) {
163 jcr->fileset = jcr->verify_job->fileset;
165 Dmsg2(100, "ClientId=%u JobLevel=%c\n", jcr->target_jr.ClientId, jcr->JobLevel);
171 * Do a verification of the specified files against the Catlaog
173 * Returns: false on failure
176 bool do_verify(JCR *jcr)
182 if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) {
183 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db));
187 /* Print Job Start message */
188 Jmsg(jcr, M_INFO, 0, _("Start Verify JobId=%d Level=%s Job=%s\n"),
189 jcr->JobId, level_to_str(jcr->JobLevel), jcr->Job);
191 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG) {
193 * Start conversation with Storage daemon
195 set_jcr_job_status(jcr, JS_Blocked);
196 if (!connect_to_storage_daemon(jcr, 10, SDConnectTimeout, 1)) {
200 * Now start a job with the Storage daemon
202 if (!start_storage_daemon_job(jcr, jcr->storage, SD_READ)) {
206 * Now start a Storage daemon message thread
208 if (!start_storage_daemon_message_thread(jcr)) {
211 Dmsg0(50, "Storage daemon connection OK\n");
214 * OK, now connect to the File daemon
215 * and ask him for the files.
217 set_jcr_job_status(jcr, JS_Blocked);
218 if (!connect_to_file_daemon(jcr, 10, FDConnectTimeout, 1)) {
222 set_jcr_job_status(jcr, JS_Running);
223 fd = jcr->file_bsock;
226 Dmsg0(30, ">filed: Send include list\n");
227 if (!send_include_list(jcr)) {
231 Dmsg0(30, ">filed: Send exclude list\n");
232 if (!send_exclude_list(jcr)) {
237 * Send Level command to File daemon, as well
238 * as the Storage address if appropriate.
240 switch (jcr->JobLevel) {
244 case L_VERIFY_CATALOG:
247 case L_VERIFY_VOLUME_TO_CATALOG:
249 * send Storage daemon address to the File daemon
251 if (jcr->store->SDDport == 0) {
252 jcr->store->SDDport = jcr->store->SDport;
254 bnet_fsend(fd, storaddr, jcr->store->address, jcr->store->SDDport);
255 if (!response(jcr, fd, OKstore, "Storage", DISPLAY_ERROR)) {
260 * Send the bootstrap file -- what Volumes/files to restore
262 if (!send_bootstrap_file(jcr)) {
266 if (!jcr->RestoreBootstrap) {
267 Jmsg0(jcr, M_FATAL, 0, _("Deprecated feature ... use bootstrap.\n"));
276 case L_VERIFY_DISK_TO_CATALOG:
277 level="disk_to_catalog";
280 Jmsg2(jcr, M_FATAL, 0, _("Unimplemented Verify level %d(%c)\n"), jcr->JobLevel,
285 if (!send_run_before_and_after_commands(jcr)) {
290 * Send verify command/level to File daemon
292 bnet_fsend(fd, verifycmd, level);
293 if (!response(jcr, fd, OKverify, "Verify", DISPLAY_ERROR)) {
298 * Now get data back from File daemon and
299 * compare it to the catalog or store it in the
300 * catalog depending on the run type.
302 /* Compare to catalog */
303 switch (jcr->JobLevel) {
304 case L_VERIFY_CATALOG:
305 Dmsg0(10, "Verify level=catalog\n");
306 jcr->sd_msg_thread_done = true; /* no SD msg thread, so it is done */
307 jcr->SDJobStatus = JS_Terminated;
308 get_attributes_and_compare_to_catalog(jcr, jcr->target_jr.JobId);
311 case L_VERIFY_VOLUME_TO_CATALOG:
312 Dmsg0(10, "Verify level=volume\n");
313 get_attributes_and_compare_to_catalog(jcr, jcr->target_jr.JobId);
316 case L_VERIFY_DISK_TO_CATALOG:
317 Dmsg0(10, "Verify level=disk_to_catalog\n");
318 jcr->sd_msg_thread_done = true; /* no SD msg thread, so it is done */
319 jcr->SDJobStatus = JS_Terminated;
320 get_attributes_and_compare_to_catalog(jcr, jcr->target_jr.JobId);
325 Dmsg0(10, "Verify level=init\n");
326 jcr->sd_msg_thread_done = true; /* no SD msg thread, so it is done */
327 jcr->SDJobStatus = JS_Terminated;
328 get_attributes_and_put_in_catalog(jcr);
332 Jmsg1(jcr, M_FATAL, 0, _("Unimplemented verify level %d\n"), jcr->JobLevel);
336 stat = wait_for_job_termination(jcr);
337 if (stat == JS_Terminated) {
338 verify_cleanup(jcr, stat);
346 * Release resources allocated during backup.
349 void verify_cleanup(JCR *jcr, int TermCode)
351 char sdt[50], edt[50];
352 char ec1[30], ec2[30];
353 char term_code[100], fd_term_msg[100], sd_term_msg[100];
354 const char *term_msg;
359 // Dmsg1(100, "Enter verify_cleanup() TermCod=%d\n", TermCode);
360 dequeue_messages(jcr); /* display any queued messages */
362 Dmsg3(900, "JobLevel=%c Expected=%u JobFiles=%u\n", jcr->JobLevel,
363 jcr->ExpectedFiles, jcr->JobFiles);
364 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG &&
365 jcr->ExpectedFiles != jcr->JobFiles) {
366 TermCode = JS_ErrorTerminated;
369 JobId = jcr->jr.JobId;
370 set_jcr_job_status(jcr, TermCode);
372 update_job_end_record(jcr);
374 msg_type = M_INFO; /* by default INFO message */
377 term_msg = _("Verify OK");
380 case JS_ErrorTerminated:
381 term_msg = _("*** Verify Error ***");
382 msg_type = M_ERROR; /* Generate error message */
385 term_msg = _("Verify warnings");
388 term_msg = _("Verify Canceled");
391 term_msg = _("Verify Differences");
394 term_msg = term_code;
395 bsnprintf(term_code, sizeof(term_code),
396 _("Inappropriate term code: %d %c\n"), TermCode, TermCode);
399 bstrftimes(sdt, sizeof(sdt), jcr->jr.StartTime);
400 bstrftimes(edt, sizeof(edt), jcr->jr.EndTime);
401 if (jcr->verify_job) {
402 Name = jcr->verify_job->hdr.name;
407 jobstatus_to_ascii(jcr->FDJobStatus, fd_term_msg, sizeof(fd_term_msg));
408 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG) {
409 jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg));
410 Jmsg(jcr, msg_type, 0, _("Bacula " VERSION " (" LSMDATE "): %s\n"
414 " Verify Level: %s\n"
416 " Verify JobId: %d\n"
420 " Files Expected: %s\n"
421 " Files Examined: %s\n"
422 " Non-fatal FD errors: %d\n"
423 " FD termination status: %s\n"
424 " SD termination status: %s\n"
425 " Termination: %s\n\n"),
429 jcr->fileset->hdr.name,
430 level_to_str(jcr->JobLevel),
431 jcr->client->hdr.name,
432 jcr->target_jr.JobId,
436 edit_uint64_with_commas(jcr->ExpectedFiles, ec1),
437 edit_uint64_with_commas(jcr->JobFiles, ec2),
443 Jmsg(jcr, msg_type, 0, _("Bacula " VERSION " (" LSMDATE "): %s\n"
447 " Verify Level: %s\n"
449 " Verify JobId: %d\n"
453 " Files Examined: %s\n"
454 " Non-fatal FD errors: %d\n"
455 " FD termination status: %s\n"
456 " Termination: %s\n\n"),
460 jcr->fileset->hdr.name,
461 level_to_str(jcr->JobLevel),
462 jcr->client->hdr.name,
463 jcr->target_jr.JobId,
467 edit_uint64_with_commas(jcr->JobFiles, ec1),
472 Dmsg0(100, "Leave verify_cleanup()\n");
476 * This routine is called only during a Verify
478 int get_attributes_and_compare_to_catalog(JCR *jcr, JobId_t JobId)
483 struct stat statf; /* file stat */
484 struct stat statc; /* catalog stat */
485 int stat = JS_Terminated;
487 POOLMEM *fname = get_pool_memory(PM_MESSAGE);
489 int32_t file_index = 0;
491 memset(&fdbr, 0, sizeof(FILE_DBR));
492 fd = jcr->file_bsock;
496 Dmsg0(20, "bdird: waiting to receive file attributes\n");
498 * Get Attributes and Signature from File daemon
502 * Options or SIG (MD5/SHA1)
507 while ((n=bget_dirmsg(fd)) >= 0 && !job_canceled(jcr)) {
510 char Opts_SIG[MAXSTRING]; /* Verify Opts or MD5/SHA1 signature */
512 fname = check_pool_memory_size(fname, fd->msglen);
513 jcr->fname = check_pool_memory_size(jcr->fname, fd->msglen);
514 Dmsg1(200, "Atts+SIG=%s\n", fd->msg);
515 if ((len = sscanf(fd->msg, "%ld %d %100s", &file_index, &stream,
517 Jmsg3(jcr, M_FATAL, 0, _("bird<filed: bad attributes, expected 3 fields got %d\n"
518 " mslen=%d msg=%s\n"), len, fd->msglen, fd->msg);
522 * We read the Options or Signature into fname
523 * to prevent overrun, now copy it to proper location.
525 bstrncpy(Opts_SIG, fname, sizeof(Opts_SIG));
527 skip_nonspaces(&p); /* skip FileIndex */
529 skip_nonspaces(&p); /* skip Stream */
531 skip_nonspaces(&p); /* skip Opts_SIG */
532 p++; /* skip space */
535 *fn++ = *p++; /* copy filename */
537 *fn = *p++; /* term filename and point to attribs */
540 * Got attributes stream, decode it
542 if (stream == STREAM_UNIX_ATTRIBUTES || stream == STREAM_UNIX_ATTRIBUTES_EX) {
543 int32_t LinkFIf, LinkFIc;
544 Dmsg2(400, "file_index=%d attr=%s\n", file_index, attr);
546 jcr->FileIndex = file_index; /* remember attribute file_index */
547 decode_stat(attr, &statf, &LinkFIf); /* decode file stat packet */
549 jcr->fn_printed = false;
550 pm_strcpy(jcr->fname, fname); /* move filename into JCR */
552 Dmsg2(040, "dird<filed: stream=%d %s\n", stream, jcr->fname);
553 Dmsg1(020, "dird<filed: attr=%s\n", attr);
556 * Find equivalent record in the database
559 if (!db_get_file_attributes_record(jcr, jcr->db, jcr->fname,
560 &jcr->target_jr, &fdbr)) {
561 Jmsg(jcr, M_INFO, 0, _("New file: %s\n"), jcr->fname);
562 Dmsg1(020, _("File not in catalog: %s\n"), jcr->fname);
563 stat = JS_Differences;
567 * mark file record as visited by stuffing the
568 * current JobId, which is unique, into the MarkId field.
570 db_mark_file_record(jcr, jcr->db, fdbr.FileId, jcr->JobId);
573 Dmsg3(400, "Found %s in catalog. inx=%d Opts=%s\n", jcr->fname,
574 file_index, Opts_SIG);
575 decode_stat(fdbr.LStat, &statc, &LinkFIc); /* decode catalog stat */
577 * Loop over options supplied by user and verify the
578 * fields he requests.
580 for (p=Opts_SIG; *p; p++) {
581 char ed1[30], ed2[30];
583 case 'i': /* compare INODEs */
584 if (statc.st_ino != statf.st_ino) {
586 Jmsg(jcr, M_INFO, 0, _(" st_ino differ. Cat: %s File: %s\n"),
587 edit_uint64((uint64_t)statc.st_ino, ed1),
588 edit_uint64((uint64_t)statf.st_ino, ed2));
589 stat = JS_Differences;
592 case 'p': /* permissions bits */
593 if (statc.st_mode != statf.st_mode) {
595 Jmsg(jcr, M_INFO, 0, _(" st_mode differ. Cat: %x File: %x\n"),
596 (uint32_t)statc.st_mode, (uint32_t)statf.st_mode);
597 stat = JS_Differences;
600 case 'n': /* number of links */
601 if (statc.st_nlink != statf.st_nlink) {
603 Jmsg(jcr, M_INFO, 0, _(" st_nlink differ. Cat: %d File: %d\n"),
604 (uint32_t)statc.st_nlink, (uint32_t)statf.st_nlink);
605 stat = JS_Differences;
608 case 'u': /* user id */
609 if (statc.st_uid != statf.st_uid) {
611 Jmsg(jcr, M_INFO, 0, _(" st_uid differ. Cat: %u File: %u\n"),
612 (uint32_t)statc.st_uid, (uint32_t)statf.st_uid);
613 stat = JS_Differences;
616 case 'g': /* group id */
617 if (statc.st_gid != statf.st_gid) {
619 Jmsg(jcr, M_INFO, 0, _(" st_gid differ. Cat: %u File: %u\n"),
620 (uint32_t)statc.st_gid, (uint32_t)statf.st_gid);
621 stat = JS_Differences;
625 if (statc.st_size != statf.st_size) {
627 Jmsg(jcr, M_INFO, 0, _(" st_size differ. Cat: %s File: %s\n"),
628 edit_uint64((uint64_t)statc.st_size, ed1),
629 edit_uint64((uint64_t)statf.st_size, ed2));
630 stat = JS_Differences;
633 case 'a': /* access time */
634 if (statc.st_atime != statf.st_atime) {
636 Jmsg(jcr, M_INFO, 0, _(" st_atime differs\n"));
637 stat = JS_Differences;
641 if (statc.st_mtime != statf.st_mtime) {
643 Jmsg(jcr, M_INFO, 0, _(" st_mtime differs\n"));
644 stat = JS_Differences;
647 case 'c': /* ctime */
648 if (statc.st_ctime != statf.st_ctime) {
650 Jmsg(jcr, M_INFO, 0, _(" st_ctime differs\n"));
651 stat = JS_Differences;
654 case 'd': /* file size decrease */
655 if (statc.st_size > statf.st_size) {
657 Jmsg(jcr, M_INFO, 0, _(" st_size decrease. Cat: %s File: %s\n"),
658 edit_uint64((uint64_t)statc.st_size, ed1),
659 edit_uint64((uint64_t)statf.st_size, ed2));
660 stat = JS_Differences;
663 case '5': /* compare MD5 */
664 Dmsg1(500, "set Do_MD5 for %s\n", jcr->fname);
667 case '1': /* compare SHA1 */
677 * Got SIG Signature from Storage daemon
678 * It came across in the Opts_SIG field.
680 } else if (stream == STREAM_MD5_SIGNATURE || stream == STREAM_SHA1_SIGNATURE) {
681 Dmsg2(400, "stream=SIG inx=%d SIG=%s\n", file_index, Opts_SIG);
683 * When ever we get a signature is MUST have been
684 * preceded by an attributes record, which sets attr_file_index
686 if (jcr->FileIndex != (uint32_t)file_index) {
687 Jmsg2(jcr, M_FATAL, 0, _("MD5/SHA1 index %d not same as attributes %d\n"),
688 file_index, jcr->FileIndex);
692 db_escape_string(buf, Opts_SIG, strlen(Opts_SIG));
693 if (strcmp(buf, fdbr.SIG) != 0) {
695 if (debug_level >= 10) {
696 Jmsg(jcr, M_INFO, 0, _(" %s not same. File=%s Cat=%s\n"),
697 stream==STREAM_MD5_SIGNATURE?"MD5":"SHA1", buf, fdbr.SIG);
699 Jmsg(jcr, M_INFO, 0, _(" %s differs.\n"),
700 stream==STREAM_MD5_SIGNATURE?"MD5":"SHA1");
702 stat = JS_Differences;
707 jcr->JobFiles = file_index;
709 if (is_bnet_error(fd)) {
711 Jmsg2(jcr, M_FATAL, 0, _("bdird<filed: bad attributes from filed n=%d : %s\n"),
716 /* Now find all the files that are missing -- i.e. all files in
717 * the database where the MarkedId != current JobId
719 jcr->fn_printed = false;
720 bsnprintf(buf, sizeof(buf),
721 "SELECT Path.Path,Filename.Name FROM File,Path,Filename "
722 "WHERE File.JobId=%d "
723 "AND File.MarkedId!=%d AND File.PathId=Path.PathId "
724 "AND File.FilenameId=Filename.FilenameId",
726 /* missing_handler is called for each file found */
727 db_sql_query(jcr->db, buf, missing_handler, (void *)jcr);
728 if (jcr->fn_printed) {
729 stat = JS_Differences;
731 free_pool_memory(fname);
732 set_jcr_job_status(jcr, stat);
733 return stat == JS_Terminated;
737 * We are called here for each record that matches the above
738 * SQL query -- that is for each file contained in the Catalog
739 * that was not marked earlier. This means that the file in
740 * question is a missing file (in the Catalog but not on Disk).
742 static int missing_handler(void *ctx, int num_fields, char **row)
744 JCR *jcr = (JCR *)ctx;
746 if (!jcr->fn_printed) {
747 Jmsg(jcr, M_INFO, 0, "\n");
748 Jmsg(jcr, M_INFO, 0, _("The following files are missing:\n"));
749 jcr->fn_printed = true;
751 Jmsg(jcr, M_INFO, 0, " %s%s\n", row[0]?row[0]:"", row[1]?row[1]:"");
757 * Print filename for verify
759 static void prt_fname(JCR *jcr)
761 if (!jcr->fn_printed) {
762 Jmsg(jcr, M_INFO, 0, _("File: %s\n"), jcr->fname);
763 jcr->fn_printed = TRUE;