3 * Bacula Director -- verify.c -- responsible for running file verification
5 * Kern Sibbald, October MM
7 * Basic tasks done here:
9 * Open connection with File daemon and pass him commands
11 * When the File daemon sends the attributes, compare them to
17 Copyright (C) 2000-2006 Kern Sibbald
19 This program is free software; you can redistribute it and/or
20 modify it under the terms of the GNU General Public License
21 version 2 as amended with additional clauses defined in the
22 file LICENSE in the main source directory.
24 This program is distributed in the hope that it will be useful,
25 but WITHOUT ANY WARRANTY; without even the implied warranty of
26 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
27 the file LICENSE for additional details.
34 #include "findlib/find.h"
36 /* Commands sent to File daemon */
37 static char verifycmd[] = "verify level=%s\n";
38 static char storaddr[] = "storage address=%s port=%d ssl=0\n";
40 /* Responses received from File daemon */
41 static char OKverify[] = "2000 OK verify\n";
42 static char OKstore[] = "2000 OK storage\n";
43 static char OKbootstrap[] = "2000 OK bootstrap\n";
45 /* Forward referenced functions */
46 static void prt_fname(JCR *jcr);
47 static int missing_handler(void *ctx, int num_fields, char **row);
51 * Called here before the job is run to do the job
54 bool do_verify_init(JCR *jcr)
57 JobId_t verify_jobid = 0;
60 memset(&jcr->previous_jr, 0, sizeof(jcr->previous_jr));
62 Dmsg1(9, "bdird: created client %s record\n", jcr->client->hdr.name);
65 * Find JobId of last job that ran. E.g.
66 * for VERIFY_CATALOG we want the JobId of the last INIT.
67 * for VERIFY_VOLUME_TO_CATALOG, we want the JobId of the
70 if (jcr->JobLevel == L_VERIFY_CATALOG ||
71 jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG ||
72 jcr->JobLevel == L_VERIFY_DISK_TO_CATALOG) {
73 memcpy(&jr, &jcr->jr, sizeof(jr));
74 if (jcr->verify_job &&
75 (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG ||
76 jcr->JobLevel == L_VERIFY_DISK_TO_CATALOG)) {
77 Name = jcr->verify_job->hdr.name;
81 Dmsg1(100, "find last jobid for: %s\n", NPRT(Name));
82 if (!db_find_last_jobid(jcr, jcr->db, Name, &jr)) {
83 if (jcr->JobLevel == L_VERIFY_CATALOG) {
84 Jmsg(jcr, M_FATAL, 0, _(
85 "Unable to find JobId of previous InitCatalog Job.\n"
86 "Please run a Verify with Level=InitCatalog before\n"
87 "running the current Job.\n"));
89 Jmsg(jcr, M_FATAL, 0, _(
90 "Unable to find JobId of previous Job for this client.\n"));
94 verify_jobid = jr.JobId;
95 Dmsg1(100, "Last full jobid=%d\n", verify_jobid);
98 * Now get the job record for the previous backup that interests
99 * us. We use the verify_jobid that we found above.
101 if (jcr->JobLevel == L_VERIFY_CATALOG ||
102 jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG ||
103 jcr->JobLevel == L_VERIFY_DISK_TO_CATALOG) {
104 jcr->previous_jr.JobId = verify_jobid;
105 if (!db_get_job_record(jcr, jcr->db, &jcr->previous_jr)) {
106 Jmsg(jcr, M_FATAL, 0, _("Could not get job record for previous Job. ERR=%s"),
107 db_strerror(jcr->db));
110 if (jcr->previous_jr.JobStatus != 'T') {
111 Jmsg(jcr, M_FATAL, 0, _("Last Job %d did not terminate normally. JobStatus=%c\n"),
112 verify_jobid, jcr->previous_jr.JobStatus);
115 Jmsg(jcr, M_INFO, 0, _("Verifying against JobId=%d Job=%s\n"),
116 jcr->previous_jr.JobId, jcr->previous_jr.Job);
120 * If we are verifying a Volume, we need the Storage
121 * daemon, so open a connection, otherwise, just
122 * create a dummy authorization key (passed to
123 * File daemon but not used).
125 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG) {
126 if (!create_restore_bootstrap_file(jcr)) {
130 jcr->sd_auth_key = bstrdup("dummy"); /* dummy Storage daemon key */
133 if (jcr->JobLevel == L_VERIFY_DISK_TO_CATALOG && jcr->verify_job) {
134 jcr->fileset = jcr->verify_job->fileset;
136 Dmsg2(100, "ClientId=%u JobLevel=%c\n", jcr->previous_jr.ClientId, jcr->JobLevel);
142 * Do a verification of the specified files against the Catlaog
144 * Returns: false on failure
147 bool do_verify(JCR *jcr)
154 if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) {
155 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db));
159 /* Print Job Start message */
160 Jmsg(jcr, M_INFO, 0, _("Start Verify JobId=%s Level=%s Job=%s\n"),
161 edit_uint64(jcr->JobId, ed1), level_to_str(jcr->JobLevel), jcr->Job);
163 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG) {
165 * Start conversation with Storage daemon
167 set_jcr_job_status(jcr, JS_Blocked);
168 if (!connect_to_storage_daemon(jcr, 10, SDConnectTimeout, 1)) {
172 * Now start a job with the Storage daemon
174 if (!start_storage_daemon_job(jcr, jcr->storage, NULL)) {
177 if (!bnet_fsend(jcr->store_bsock, "run")) {
181 * Now start a Storage daemon message thread
183 if (!start_storage_daemon_message_thread(jcr)) {
186 Dmsg0(50, "Storage daemon connection OK\n");
190 * OK, now connect to the File daemon
191 * and ask him for the files.
193 set_jcr_job_status(jcr, JS_Blocked);
194 if (!connect_to_file_daemon(jcr, 10, FDConnectTimeout, 1)) {
198 set_jcr_job_status(jcr, JS_Running);
199 fd = jcr->file_bsock;
202 Dmsg0(30, ">filed: Send include list\n");
203 if (!send_include_list(jcr)) {
207 Dmsg0(30, ">filed: Send exclude list\n");
208 if (!send_exclude_list(jcr)) {
213 * Send Level command to File daemon, as well
214 * as the Storage address if appropriate.
216 switch (jcr->JobLevel) {
220 case L_VERIFY_CATALOG:
223 case L_VERIFY_VOLUME_TO_CATALOG:
225 * send Storage daemon address to the File daemon
227 if (jcr->store->SDDport == 0) {
228 jcr->store->SDDport = jcr->store->SDport;
230 bnet_fsend(fd, storaddr, jcr->store->address, jcr->store->SDDport);
231 if (!response(jcr, fd, OKstore, "Storage", DISPLAY_ERROR)) {
236 * Send the bootstrap file -- what Volumes/files to restore
238 if (!send_bootstrap_file(jcr, fd) ||
239 !response(jcr, fd, OKbootstrap, "Bootstrap", DISPLAY_ERROR)) {
243 if (!jcr->RestoreBootstrap) {
244 Jmsg0(jcr, M_FATAL, 0, _("Deprecated feature ... use bootstrap.\n"));
253 case L_VERIFY_DISK_TO_CATALOG:
254 level="disk_to_catalog";
257 Jmsg2(jcr, M_FATAL, 0, _("Unimplemented Verify level %d(%c)\n"), jcr->JobLevel,
262 if (!send_runscripts_commands(jcr)) {
267 * Send verify command/level to File daemon
269 bnet_fsend(fd, verifycmd, level);
270 if (!response(jcr, fd, OKverify, "Verify", DISPLAY_ERROR)) {
275 * Now get data back from File daemon and
276 * compare it to the catalog or store it in the
277 * catalog depending on the run type.
279 /* Compare to catalog */
280 switch (jcr->JobLevel) {
281 case L_VERIFY_CATALOG:
282 Dmsg0(10, "Verify level=catalog\n");
283 jcr->sd_msg_thread_done = true; /* no SD msg thread, so it is done */
284 jcr->SDJobStatus = JS_Terminated;
285 get_attributes_and_compare_to_catalog(jcr, jcr->previous_jr.JobId);
288 case L_VERIFY_VOLUME_TO_CATALOG:
289 Dmsg0(10, "Verify level=volume\n");
290 get_attributes_and_compare_to_catalog(jcr, jcr->previous_jr.JobId);
293 case L_VERIFY_DISK_TO_CATALOG:
294 Dmsg0(10, "Verify level=disk_to_catalog\n");
295 jcr->sd_msg_thread_done = true; /* no SD msg thread, so it is done */
296 jcr->SDJobStatus = JS_Terminated;
297 get_attributes_and_compare_to_catalog(jcr, jcr->previous_jr.JobId);
302 Dmsg0(10, "Verify level=init\n");
303 jcr->sd_msg_thread_done = true; /* no SD msg thread, so it is done */
304 jcr->SDJobStatus = JS_Terminated;
305 get_attributes_and_put_in_catalog(jcr);
309 Jmsg1(jcr, M_FATAL, 0, _("Unimplemented verify level %d\n"), jcr->JobLevel);
313 stat = wait_for_job_termination(jcr);
314 if (stat == JS_Terminated) {
315 verify_cleanup(jcr, stat);
323 * Release resources allocated during backup.
326 void verify_cleanup(JCR *jcr, int TermCode)
328 char sdt[50], edt[50];
329 char ec1[30], ec2[30];
330 char term_code[100], fd_term_msg[100], sd_term_msg[100];
331 const char *term_msg;
336 // Dmsg1(100, "Enter verify_cleanup() TermCod=%d\n", TermCode);
337 dequeue_messages(jcr); /* display any queued messages */
339 Dmsg3(900, "JobLevel=%c Expected=%u JobFiles=%u\n", jcr->JobLevel,
340 jcr->ExpectedFiles, jcr->JobFiles);
341 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG &&
342 jcr->ExpectedFiles != jcr->JobFiles) {
343 TermCode = JS_ErrorTerminated;
346 /* If no files were expected, there can be no error */
347 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG &&
348 jcr->ExpectedFiles == 0) {
349 TermCode = JS_Terminated;
352 JobId = jcr->jr.JobId;
353 set_jcr_job_status(jcr, TermCode);
355 update_job_end_record(jcr);
356 if (jcr->unlink_bsr && jcr->RestoreBootstrap) {
357 unlink(jcr->RestoreBootstrap);
358 jcr->unlink_bsr = false;
361 msg_type = M_INFO; /* by default INFO message */
364 term_msg = _("Verify OK");
367 case JS_ErrorTerminated:
368 term_msg = _("*** Verify Error ***");
369 msg_type = M_ERROR; /* Generate error message */
372 term_msg = _("Verify warnings");
375 term_msg = _("Verify Canceled");
378 term_msg = _("Verify Differences");
381 term_msg = term_code;
382 bsnprintf(term_code, sizeof(term_code),
383 _("Inappropriate term code: %d %c\n"), TermCode, TermCode);
386 bstrftimes(sdt, sizeof(sdt), jcr->jr.StartTime);
387 bstrftimes(edt, sizeof(edt), jcr->jr.EndTime);
388 if (jcr->verify_job) {
389 Name = jcr->verify_job->hdr.name;
394 jobstatus_to_ascii(jcr->FDJobStatus, fd_term_msg, sizeof(fd_term_msg));
395 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG) {
396 jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg));
397 Jmsg(jcr, msg_type, 0, _("Bacula %s (%s): %s\n"
401 " Verify Level: %s\n"
403 " Verify JobId: %d\n"
407 " Files Expected: %s\n"
408 " Files Examined: %s\n"
409 " Non-fatal FD errors: %d\n"
410 " FD termination status: %s\n"
411 " SD termination status: %s\n"
412 " Termination: %s\n\n"),
418 jcr->fileset->hdr.name,
419 level_to_str(jcr->JobLevel),
420 jcr->client->hdr.name,
421 jcr->previous_jr.JobId,
425 edit_uint64_with_commas(jcr->ExpectedFiles, ec1),
426 edit_uint64_with_commas(jcr->JobFiles, ec2),
432 Jmsg(jcr, msg_type, 0, _("Bacula %s (%s): %s\n"
436 " Verify Level: %s\n"
438 " Verify JobId: %d\n"
442 " Files Examined: %s\n"
443 " Non-fatal FD errors: %d\n"
444 " FD termination status: %s\n"
445 " Termination: %s\n\n"),
451 jcr->fileset->hdr.name,
452 level_to_str(jcr->JobLevel),
453 jcr->client->hdr.name,
454 jcr->previous_jr.JobId,
458 edit_uint64_with_commas(jcr->JobFiles, ec1),
463 Dmsg0(100, "Leave verify_cleanup()\n");
467 * This routine is called only during a Verify
469 int get_attributes_and_compare_to_catalog(JCR *jcr, JobId_t JobId)
474 struct stat statf; /* file stat */
475 struct stat statc; /* catalog stat */
476 int stat = JS_Terminated;
478 POOLMEM *fname = get_pool_memory(PM_MESSAGE);
479 int do_Digest = CRYPTO_DIGEST_NONE;
480 int32_t file_index = 0;
482 memset(&fdbr, 0, sizeof(FILE_DBR));
483 fd = jcr->file_bsock;
487 Dmsg0(20, "bdird: waiting to receive file attributes\n");
489 * Get Attributes and Signature from File daemon
493 * Options or Digest (MD5/SHA1)
498 while ((n=bget_dirmsg(fd)) >= 0 && !job_canceled(jcr)) {
501 char Opts_Digest[MAXSTRING]; /* Verify Opts or MD5/SHA1 digest */
503 fname = check_pool_memory_size(fname, fd->msglen);
504 jcr->fname = check_pool_memory_size(jcr->fname, fd->msglen);
505 Dmsg1(200, "Atts+Digest=%s\n", fd->msg);
506 if ((len = sscanf(fd->msg, "%ld %d %100s", &file_index, &stream,
508 Jmsg3(jcr, M_FATAL, 0, _("bird<filed: bad attributes, expected 3 fields got %d\n"
509 " mslen=%d msg=%s\n"), len, fd->msglen, fd->msg);
513 * We read the Options or Signature into fname
514 * to prevent overrun, now copy it to proper location.
516 bstrncpy(Opts_Digest, fname, sizeof(Opts_Digest));
518 skip_nonspaces(&p); /* skip FileIndex */
520 skip_nonspaces(&p); /* skip Stream */
522 skip_nonspaces(&p); /* skip Opts_Digest */
523 p++; /* skip space */
526 *fn++ = *p++; /* copy filename */
528 *fn = *p++; /* term filename and point to attribs */
531 * Got attributes stream, decode it
533 if (stream == STREAM_UNIX_ATTRIBUTES || stream == STREAM_UNIX_ATTRIBUTES_EX) {
534 int32_t LinkFIf, LinkFIc;
535 Dmsg2(400, "file_index=%d attr=%s\n", file_index, attr);
537 jcr->FileIndex = file_index; /* remember attribute file_index */
538 decode_stat(attr, &statf, &LinkFIf); /* decode file stat packet */
539 do_Digest = CRYPTO_DIGEST_NONE;
540 jcr->fn_printed = false;
541 pm_strcpy(jcr->fname, fname); /* move filename into JCR */
543 Dmsg2(040, "dird<filed: stream=%d %s\n", stream, jcr->fname);
544 Dmsg1(020, "dird<filed: attr=%s\n", attr);
547 * Find equivalent record in the database
550 if (!db_get_file_attributes_record(jcr, jcr->db, jcr->fname,
551 &jcr->previous_jr, &fdbr)) {
552 Jmsg(jcr, M_INFO, 0, _("New file: %s\n"), jcr->fname);
553 Dmsg1(020, _("File not in catalog: %s\n"), jcr->fname);
554 stat = JS_Differences;
558 * mark file record as visited by stuffing the
559 * current JobId, which is unique, into the MarkId field.
561 db_mark_file_record(jcr, jcr->db, fdbr.FileId, jcr->JobId);
564 Dmsg3(400, "Found %s in catalog. inx=%d Opts=%s\n", jcr->fname,
565 file_index, Opts_Digest);
566 decode_stat(fdbr.LStat, &statc, &LinkFIc); /* decode catalog stat */
568 * Loop over options supplied by user and verify the
569 * fields he requests.
571 for (p=Opts_Digest; *p; p++) {
572 char ed1[30], ed2[30];
574 case 'i': /* compare INODEs */
575 if (statc.st_ino != statf.st_ino) {
577 Jmsg(jcr, M_INFO, 0, _(" st_ino differ. Cat: %s File: %s\n"),
578 edit_uint64((uint64_t)statc.st_ino, ed1),
579 edit_uint64((uint64_t)statf.st_ino, ed2));
580 stat = JS_Differences;
583 case 'p': /* permissions bits */
584 if (statc.st_mode != statf.st_mode) {
586 Jmsg(jcr, M_INFO, 0, _(" st_mode differ. Cat: %x File: %x\n"),
587 (uint32_t)statc.st_mode, (uint32_t)statf.st_mode);
588 stat = JS_Differences;
591 case 'n': /* number of links */
592 if (statc.st_nlink != statf.st_nlink) {
594 Jmsg(jcr, M_INFO, 0, _(" st_nlink differ. Cat: %d File: %d\n"),
595 (uint32_t)statc.st_nlink, (uint32_t)statf.st_nlink);
596 stat = JS_Differences;
599 case 'u': /* user id */
600 if (statc.st_uid != statf.st_uid) {
602 Jmsg(jcr, M_INFO, 0, _(" st_uid differ. Cat: %u File: %u\n"),
603 (uint32_t)statc.st_uid, (uint32_t)statf.st_uid);
604 stat = JS_Differences;
607 case 'g': /* group id */
608 if (statc.st_gid != statf.st_gid) {
610 Jmsg(jcr, M_INFO, 0, _(" st_gid differ. Cat: %u File: %u\n"),
611 (uint32_t)statc.st_gid, (uint32_t)statf.st_gid);
612 stat = JS_Differences;
616 if (statc.st_size != statf.st_size) {
618 Jmsg(jcr, M_INFO, 0, _(" st_size differ. Cat: %s File: %s\n"),
619 edit_uint64((uint64_t)statc.st_size, ed1),
620 edit_uint64((uint64_t)statf.st_size, ed2));
621 stat = JS_Differences;
624 case 'a': /* access time */
625 if (statc.st_atime != statf.st_atime) {
627 Jmsg(jcr, M_INFO, 0, _(" st_atime differs\n"));
628 stat = JS_Differences;
632 if (statc.st_mtime != statf.st_mtime) {
634 Jmsg(jcr, M_INFO, 0, _(" st_mtime differs\n"));
635 stat = JS_Differences;
638 case 'c': /* ctime */
639 if (statc.st_ctime != statf.st_ctime) {
641 Jmsg(jcr, M_INFO, 0, _(" st_ctime differs\n"));
642 stat = JS_Differences;
645 case 'd': /* file size decrease */
646 if (statc.st_size > statf.st_size) {
648 Jmsg(jcr, M_INFO, 0, _(" st_size decrease. Cat: %s File: %s\n"),
649 edit_uint64((uint64_t)statc.st_size, ed1),
650 edit_uint64((uint64_t)statf.st_size, ed2));
651 stat = JS_Differences;
654 case '5': /* compare MD5 */
655 Dmsg1(500, "set Do_MD5 for %s\n", jcr->fname);
656 do_Digest = CRYPTO_DIGEST_MD5;
658 case '1': /* compare SHA1 */
659 do_Digest = CRYPTO_DIGEST_SHA1;
668 * Got Digest Signature from Storage daemon
669 * It came across in the Opts_Digest field.
671 } else if (crypto_digest_stream_type(stream) != CRYPTO_DIGEST_NONE) {
672 Dmsg2(400, "stream=Digest inx=%d Digest=%s\n", file_index, Opts_Digest);
674 * When ever we get a digest is MUST have been
675 * preceded by an attributes record, which sets attr_file_index
677 if (jcr->FileIndex != (uint32_t)file_index) {
678 Jmsg2(jcr, M_FATAL, 0, _("MD5/SHA1 index %d not same as attributes %d\n"),
679 file_index, jcr->FileIndex);
682 if (do_Digest != CRYPTO_DIGEST_NONE) {
683 db_escape_string(buf, Opts_Digest, strlen(Opts_Digest));
684 if (strcmp(buf, fdbr.Digest) != 0) {
686 if (debug_level >= 10) {
687 Jmsg(jcr, M_INFO, 0, _(" %s not same. File=%s Cat=%s\n"),
688 stream_to_ascii(stream), buf, fdbr.Digest);
690 Jmsg(jcr, M_INFO, 0, _(" %s differs.\n"),
691 stream_to_ascii(stream));
693 stat = JS_Differences;
695 do_Digest = CRYPTO_DIGEST_NONE;
698 jcr->JobFiles = file_index;
700 if (is_bnet_error(fd)) {
702 Jmsg2(jcr, M_FATAL, 0, _("bdird<filed: bad attributes from filed n=%d : %s\n"),
707 /* Now find all the files that are missing -- i.e. all files in
708 * the database where the MarkId != current JobId
710 jcr->fn_printed = false;
711 bsnprintf(buf, sizeof(buf),
712 "SELECT Path.Path,Filename.Name FROM File,Path,Filename "
713 "WHERE File.JobId=%d "
714 "AND File.MarkId!=%d AND File.PathId=Path.PathId "
715 "AND File.FilenameId=Filename.FilenameId",
717 /* missing_handler is called for each file found */
718 db_sql_query(jcr->db, buf, missing_handler, (void *)jcr);
719 if (jcr->fn_printed) {
720 stat = JS_Differences;
722 free_pool_memory(fname);
723 set_jcr_job_status(jcr, stat);
724 return stat == JS_Terminated;
728 * We are called here for each record that matches the above
729 * SQL query -- that is for each file contained in the Catalog
730 * that was not marked earlier. This means that the file in
731 * question is a missing file (in the Catalog but not on Disk).
733 static int missing_handler(void *ctx, int num_fields, char **row)
735 JCR *jcr = (JCR *)ctx;
737 if (!jcr->fn_printed) {
738 Jmsg(jcr, M_INFO, 0, "\n");
739 Jmsg(jcr, M_INFO, 0, _("The following files are missing:\n"));
740 jcr->fn_printed = true;
742 Jmsg(jcr, M_INFO, 0, " %s%s\n", row[0]?row[0]:"", row[1]?row[1]:"");
748 * Print filename for verify
750 static void prt_fname(JCR *jcr)
752 if (!jcr->fn_printed) {
753 Jmsg(jcr, M_INFO, 0, _("File: %s\n"), jcr->fname);
754 jcr->fn_printed = TRUE;