2 Bacula® - The Network Backup Solution
4 Copyright (C) 2000-2007 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of John Walker.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
30 * Bacula Director -- verify.c -- responsible for running file verification
32 * Kern Sibbald, October MM
34 * Basic tasks done here:
36 * Open connection with File daemon and pass him commands
38 * When the File daemon sends the attributes, compare them to
47 #include "findlib/find.h"
49 /* Commands sent to File daemon */
50 static char verifycmd[] = "verify level=%s\n";
51 static char storaddr[] = "storage address=%s port=%d ssl=0\n";
53 /* Responses received from File daemon */
54 static char OKverify[] = "2000 OK verify\n";
55 static char OKstore[] = "2000 OK storage\n";
56 static char OKbootstrap[] = "2000 OK bootstrap\n";
58 /* Forward referenced functions */
59 static void prt_fname(JCR *jcr);
60 static int missing_handler(void *ctx, int num_fields, char **row);
64 * Called here before the job is run to do the job
67 bool do_verify_init(JCR *jcr)
74 * Do a verification of the specified files against the Catlaog
76 * Returns: false on failure
79 bool do_verify(JCR *jcr)
86 JobId_t verify_jobid = 0;
89 free_wstorage(jcr); /* we don't write */
91 memset(&jcr->previous_jr, 0, sizeof(jcr->previous_jr));
94 * Find JobId of last job that ran. Note, we do this when
95 * the job actually starts running, not at schedule time,
96 * so that we find the last job that terminated before
97 * this job runs rather than before it is scheduled. This
98 * permits scheduling a Backup and Verify at the same time,
99 * but with the Verify at a lower priority.
101 * For VERIFY_CATALOG we want the JobId of the last INIT.
102 * For VERIFY_VOLUME_TO_CATALOG, we want the JobId of the
105 if (jcr->JobLevel == L_VERIFY_CATALOG ||
106 jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG ||
107 jcr->JobLevel == L_VERIFY_DISK_TO_CATALOG) {
108 memcpy(&jr, &jcr->jr, sizeof(jr));
109 if (jcr->verify_job &&
110 (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG ||
111 jcr->JobLevel == L_VERIFY_DISK_TO_CATALOG)) {
112 Name = jcr->verify_job->name();
116 Dmsg1(100, "find last jobid for: %s\n", NPRT(Name));
117 if (!db_find_last_jobid(jcr, jcr->db, Name, &jr)) {
118 if (jcr->JobLevel == L_VERIFY_CATALOG) {
119 Jmsg(jcr, M_FATAL, 0, _(
120 "Unable to find JobId of previous InitCatalog Job.\n"
121 "Please run a Verify with Level=InitCatalog before\n"
122 "running the current Job.\n"));
124 Jmsg(jcr, M_FATAL, 0, _(
125 "Unable to find JobId of previous Job for this client.\n"));
129 verify_jobid = jr.JobId;
130 Dmsg1(100, "Last full jobid=%d\n", verify_jobid);
133 * Now get the job record for the previous backup that interests
134 * us. We use the verify_jobid that we found above.
136 if (jcr->JobLevel == L_VERIFY_CATALOG ||
137 jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG ||
138 jcr->JobLevel == L_VERIFY_DISK_TO_CATALOG) {
139 jcr->previous_jr.JobId = verify_jobid;
140 if (!db_get_job_record(jcr, jcr->db, &jcr->previous_jr)) {
141 Jmsg(jcr, M_FATAL, 0, _("Could not get job record for previous Job. ERR=%s"),
142 db_strerror(jcr->db));
145 if (jcr->previous_jr.JobStatus != 'T') {
146 Jmsg(jcr, M_FATAL, 0, _("Last Job %d did not terminate normally. JobStatus=%c\n"),
147 verify_jobid, jcr->previous_jr.JobStatus);
150 Jmsg(jcr, M_INFO, 0, _("Verifying against JobId=%d Job=%s\n"),
151 jcr->previous_jr.JobId, jcr->previous_jr.Job);
155 * If we are verifying a Volume, we need the Storage
156 * daemon, so open a connection, otherwise, just
157 * create a dummy authorization key (passed to
158 * File daemon but not used).
160 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG) {
161 if (!create_restore_bootstrap_file(jcr)) {
165 jcr->sd_auth_key = bstrdup("dummy"); /* dummy Storage daemon key */
168 if (jcr->JobLevel == L_VERIFY_DISK_TO_CATALOG && jcr->verify_job) {
169 jcr->fileset = jcr->verify_job->fileset;
171 Dmsg2(100, "ClientId=%u JobLevel=%c\n", jcr->previous_jr.ClientId, jcr->JobLevel);
173 if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) {
174 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db));
178 /* Print Job Start message */
179 Jmsg(jcr, M_INFO, 0, _("Start Verify JobId=%s Level=%s Job=%s\n"),
180 edit_uint64(jcr->JobId, ed1), level_to_str(jcr->JobLevel), jcr->Job);
182 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG) {
184 * Start conversation with Storage daemon
186 set_jcr_job_status(jcr, JS_Blocked);
187 if (!connect_to_storage_daemon(jcr, 10, SDConnectTimeout, 1)) {
191 * Now start a job with the Storage daemon
193 if (!start_storage_daemon_job(jcr, jcr->rstorage, NULL)) {
196 if (!bnet_fsend(jcr->store_bsock, "run")) {
200 * Now start a Storage daemon message thread
202 if (!start_storage_daemon_message_thread(jcr)) {
205 Dmsg0(50, "Storage daemon connection OK\n");
209 * OK, now connect to the File daemon
210 * and ask him for the files.
212 set_jcr_job_status(jcr, JS_Blocked);
213 if (!connect_to_file_daemon(jcr, 10, FDConnectTimeout, 1)) {
217 set_jcr_job_status(jcr, JS_Running);
218 fd = jcr->file_bsock;
221 Dmsg0(30, ">filed: Send include list\n");
222 if (!send_include_list(jcr)) {
226 Dmsg0(30, ">filed: Send exclude list\n");
227 if (!send_exclude_list(jcr)) {
232 * Send Level command to File daemon, as well
233 * as the Storage address if appropriate.
235 switch (jcr->JobLevel) {
239 case L_VERIFY_CATALOG:
242 case L_VERIFY_VOLUME_TO_CATALOG:
244 * send Storage daemon address to the File daemon
246 if (jcr->rstore->SDDport == 0) {
247 jcr->rstore->SDDport = jcr->rstore->SDport;
249 bnet_fsend(fd, storaddr, jcr->rstore->address, jcr->rstore->SDDport);
250 if (!response(jcr, fd, OKstore, "Storage", DISPLAY_ERROR)) {
255 * Send the bootstrap file -- what Volumes/files to restore
257 if (!send_bootstrap_file(jcr, fd) ||
258 !response(jcr, fd, OKbootstrap, "Bootstrap", DISPLAY_ERROR)) {
262 if (!jcr->RestoreBootstrap) {
263 Jmsg0(jcr, M_FATAL, 0, _("Deprecated feature ... use bootstrap.\n"));
272 case L_VERIFY_DISK_TO_CATALOG:
273 level="disk_to_catalog";
276 Jmsg2(jcr, M_FATAL, 0, _("Unimplemented Verify level %d(%c)\n"), jcr->JobLevel,
281 if (!send_runscripts_commands(jcr)) {
286 * Send verify command/level to File daemon
288 fd->fsend(verifycmd, level);
289 if (!response(jcr, fd, OKverify, "Verify", DISPLAY_ERROR)) {
294 * Now get data back from File daemon and
295 * compare it to the catalog or store it in the
296 * catalog depending on the run type.
298 /* Compare to catalog */
299 switch (jcr->JobLevel) {
300 case L_VERIFY_CATALOG:
301 Dmsg0(10, "Verify level=catalog\n");
302 jcr->sd_msg_thread_done = true; /* no SD msg thread, so it is done */
303 jcr->SDJobStatus = JS_Terminated;
304 get_attributes_and_compare_to_catalog(jcr, jcr->previous_jr.JobId);
307 case L_VERIFY_VOLUME_TO_CATALOG:
308 Dmsg0(10, "Verify level=volume\n");
309 get_attributes_and_compare_to_catalog(jcr, jcr->previous_jr.JobId);
312 case L_VERIFY_DISK_TO_CATALOG:
313 Dmsg0(10, "Verify level=disk_to_catalog\n");
314 jcr->sd_msg_thread_done = true; /* no SD msg thread, so it is done */
315 jcr->SDJobStatus = JS_Terminated;
316 get_attributes_and_compare_to_catalog(jcr, jcr->previous_jr.JobId);
321 Dmsg0(10, "Verify level=init\n");
322 jcr->sd_msg_thread_done = true; /* no SD msg thread, so it is done */
323 jcr->SDJobStatus = JS_Terminated;
324 get_attributes_and_put_in_catalog(jcr);
325 db_end_transaction(jcr, jcr->db); /* terminate any open transaction */
326 db_write_batch_file_records(jcr);
330 Jmsg1(jcr, M_FATAL, 0, _("Unimplemented verify level %d\n"), jcr->JobLevel);
334 stat = wait_for_job_termination(jcr);
335 verify_cleanup(jcr, stat);
344 * Release resources allocated during backup.
347 void verify_cleanup(JCR *jcr, int TermCode)
349 char sdt[50], edt[50];
350 char ec1[30], ec2[30];
351 char term_code[100], fd_term_msg[100], sd_term_msg[100];
352 const char *term_msg;
357 // Dmsg1(100, "Enter verify_cleanup() TermCod=%d\n", TermCode);
359 Dmsg3(900, "JobLevel=%c Expected=%u JobFiles=%u\n", jcr->JobLevel,
360 jcr->ExpectedFiles, jcr->JobFiles);
361 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG &&
362 jcr->ExpectedFiles != jcr->JobFiles) {
363 TermCode = JS_ErrorTerminated;
366 /* If no files were expected, there can be no error */
367 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG &&
368 jcr->ExpectedFiles == 0) {
369 TermCode = JS_Terminated;
372 JobId = jcr->jr.JobId;
374 update_job_end(jcr, TermCode);
376 if (job_canceled(jcr)) {
377 cancel_storage_daemon_job(jcr);
380 if (jcr->unlink_bsr && jcr->RestoreBootstrap) {
381 unlink(jcr->RestoreBootstrap);
382 jcr->unlink_bsr = false;
385 msg_type = M_INFO; /* by default INFO message */
388 term_msg = _("Verify OK");
391 case JS_ErrorTerminated:
392 term_msg = _("*** Verify Error ***");
393 msg_type = M_ERROR; /* Generate error message */
396 term_msg = _("Verify warnings");
399 term_msg = _("Verify Canceled");
402 term_msg = _("Verify Differences");
405 term_msg = term_code;
406 bsnprintf(term_code, sizeof(term_code),
407 _("Inappropriate term code: %d %c\n"), TermCode, TermCode);
410 bstrftimes(sdt, sizeof(sdt), jcr->jr.StartTime);
411 bstrftimes(edt, sizeof(edt), jcr->jr.EndTime);
412 if (jcr->verify_job) {
413 Name = jcr->verify_job->hdr.name;
418 jobstatus_to_ascii(jcr->FDJobStatus, fd_term_msg, sizeof(fd_term_msg));
419 if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG) {
420 jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg));
421 Jmsg(jcr, msg_type, 0, _("Bacula %s %s (%s): %s\n"
422 " Build OS: %s %s %s\n"
426 " Verify Level: %s\n"
428 " Verify JobId: %d\n"
432 " Files Expected: %s\n"
433 " Files Examined: %s\n"
434 " Non-fatal FD errors: %d\n"
435 " FD termination status: %s\n"
436 " SD termination status: %s\n"
437 " Termination: %s\n\n"),
438 my_name, VERSION, LSMDATE, edt,
439 HOST_OS, DISTNAME, DISTVER,
442 jcr->fileset->hdr.name,
443 level_to_str(jcr->JobLevel),
444 jcr->client->hdr.name,
445 jcr->previous_jr.JobId,
449 edit_uint64_with_commas(jcr->ExpectedFiles, ec1),
450 edit_uint64_with_commas(jcr->JobFiles, ec2),
456 Jmsg(jcr, msg_type, 0, _("Bacula %s %s (%s): %s\n"
461 " Verify Level: %s\n"
463 " Verify JobId: %d\n"
467 " Files Examined: %s\n"
468 " Non-fatal FD errors: %d\n"
469 " FD termination status: %s\n"
470 " Termination: %s\n\n"),
471 my_name, VERSION, LSMDATE, edt,
472 HOST_OS, DISTNAME, DISTVER,
475 jcr->fileset->hdr.name,
476 level_to_str(jcr->JobLevel),
478 jcr->previous_jr.JobId,
482 edit_uint64_with_commas(jcr->JobFiles, ec1),
487 Dmsg0(100, "Leave verify_cleanup()\n");
491 * This routine is called only during a Verify
493 int get_attributes_and_compare_to_catalog(JCR *jcr, JobId_t JobId)
498 struct stat statf; /* file stat */
499 struct stat statc; /* catalog stat */
500 int stat = JS_Terminated;
502 POOLMEM *fname = get_pool_memory(PM_MESSAGE);
503 int do_Digest = CRYPTO_DIGEST_NONE;
504 int32_t file_index = 0;
506 memset(&fdbr, 0, sizeof(FILE_DBR));
507 fd = jcr->file_bsock;
511 Dmsg0(20, "bdird: waiting to receive file attributes\n");
513 * Get Attributes and Signature from File daemon
517 * Options or Digest (MD5/SHA1)
522 while ((n=bget_dirmsg(fd)) >= 0 && !job_canceled(jcr)) {
525 char Opts_Digest[MAXSTRING]; /* Verify Opts or MD5/SHA1 digest */
527 if (job_canceled(jcr)) {
530 fname = check_pool_memory_size(fname, fd->msglen);
531 jcr->fname = check_pool_memory_size(jcr->fname, fd->msglen);
532 Dmsg1(200, "Atts+Digest=%s\n", fd->msg);
533 if ((len = sscanf(fd->msg, "%ld %d %100s", &file_index, &stream,
535 Jmsg3(jcr, M_FATAL, 0, _("bird<filed: bad attributes, expected 3 fields got %d\n"
536 " mslen=%d msg=%s\n"), len, fd->msglen, fd->msg);
540 * We read the Options or Signature into fname
541 * to prevent overrun, now copy it to proper location.
543 bstrncpy(Opts_Digest, fname, sizeof(Opts_Digest));
545 skip_nonspaces(&p); /* skip FileIndex */
547 skip_nonspaces(&p); /* skip Stream */
549 skip_nonspaces(&p); /* skip Opts_Digest */
550 p++; /* skip space */
553 *fn++ = *p++; /* copy filename */
555 *fn = *p++; /* term filename and point to attribs */
558 * Got attributes stream, decode it
560 if (stream == STREAM_UNIX_ATTRIBUTES || stream == STREAM_UNIX_ATTRIBUTES_EX) {
561 int32_t LinkFIf, LinkFIc;
562 Dmsg2(400, "file_index=%d attr=%s\n", file_index, attr);
564 jcr->FileIndex = file_index; /* remember attribute file_index */
565 decode_stat(attr, &statf, &LinkFIf); /* decode file stat packet */
566 do_Digest = CRYPTO_DIGEST_NONE;
567 jcr->fn_printed = false;
568 pm_strcpy(jcr->fname, fname); /* move filename into JCR */
570 Dmsg2(040, "dird<filed: stream=%d %s\n", stream, jcr->fname);
571 Dmsg1(020, "dird<filed: attr=%s\n", attr);
574 * Find equivalent record in the database
577 if (!db_get_file_attributes_record(jcr, jcr->db, jcr->fname,
578 &jcr->previous_jr, &fdbr)) {
579 Jmsg(jcr, M_INFO, 0, _("New file: %s\n"), jcr->fname);
580 Dmsg1(020, _("File not in catalog: %s\n"), jcr->fname);
581 stat = JS_Differences;
585 * mark file record as visited by stuffing the
586 * current JobId, which is unique, into the MarkId field.
588 db_mark_file_record(jcr, jcr->db, fdbr.FileId, jcr->JobId);
591 Dmsg3(400, "Found %s in catalog. inx=%d Opts=%s\n", jcr->fname,
592 file_index, Opts_Digest);
593 decode_stat(fdbr.LStat, &statc, &LinkFIc); /* decode catalog stat */
595 * Loop over options supplied by user and verify the
596 * fields he requests.
598 for (p=Opts_Digest; *p; p++) {
599 char ed1[30], ed2[30];
601 case 'i': /* compare INODEs */
602 if (statc.st_ino != statf.st_ino) {
604 Jmsg(jcr, M_INFO, 0, _(" st_ino differ. Cat: %s File: %s\n"),
605 edit_uint64((uint64_t)statc.st_ino, ed1),
606 edit_uint64((uint64_t)statf.st_ino, ed2));
607 stat = JS_Differences;
610 case 'p': /* permissions bits */
611 if (statc.st_mode != statf.st_mode) {
613 Jmsg(jcr, M_INFO, 0, _(" st_mode differ. Cat: %x File: %x\n"),
614 (uint32_t)statc.st_mode, (uint32_t)statf.st_mode);
615 stat = JS_Differences;
618 case 'n': /* number of links */
619 if (statc.st_nlink != statf.st_nlink) {
621 Jmsg(jcr, M_INFO, 0, _(" st_nlink differ. Cat: %d File: %d\n"),
622 (uint32_t)statc.st_nlink, (uint32_t)statf.st_nlink);
623 stat = JS_Differences;
626 case 'u': /* user id */
627 if (statc.st_uid != statf.st_uid) {
629 Jmsg(jcr, M_INFO, 0, _(" st_uid differ. Cat: %u File: %u\n"),
630 (uint32_t)statc.st_uid, (uint32_t)statf.st_uid);
631 stat = JS_Differences;
634 case 'g': /* group id */
635 if (statc.st_gid != statf.st_gid) {
637 Jmsg(jcr, M_INFO, 0, _(" st_gid differ. Cat: %u File: %u\n"),
638 (uint32_t)statc.st_gid, (uint32_t)statf.st_gid);
639 stat = JS_Differences;
643 if (statc.st_size != statf.st_size) {
645 Jmsg(jcr, M_INFO, 0, _(" st_size differ. Cat: %s File: %s\n"),
646 edit_uint64((uint64_t)statc.st_size, ed1),
647 edit_uint64((uint64_t)statf.st_size, ed2));
648 stat = JS_Differences;
651 case 'a': /* access time */
652 if (statc.st_atime != statf.st_atime) {
654 Jmsg(jcr, M_INFO, 0, _(" st_atime differs\n"));
655 stat = JS_Differences;
659 if (statc.st_mtime != statf.st_mtime) {
661 Jmsg(jcr, M_INFO, 0, _(" st_mtime differs\n"));
662 stat = JS_Differences;
665 case 'c': /* ctime */
666 if (statc.st_ctime != statf.st_ctime) {
668 Jmsg(jcr, M_INFO, 0, _(" st_ctime differs\n"));
669 stat = JS_Differences;
672 case 'd': /* file size decrease */
673 if (statc.st_size > statf.st_size) {
675 Jmsg(jcr, M_INFO, 0, _(" st_size decrease. Cat: %s File: %s\n"),
676 edit_uint64((uint64_t)statc.st_size, ed1),
677 edit_uint64((uint64_t)statf.st_size, ed2));
678 stat = JS_Differences;
681 case '5': /* compare MD5 */
682 Dmsg1(500, "set Do_MD5 for %s\n", jcr->fname);
683 do_Digest = CRYPTO_DIGEST_MD5;
685 case '1': /* compare SHA1 */
686 do_Digest = CRYPTO_DIGEST_SHA1;
695 * Got Digest Signature from Storage daemon
696 * It came across in the Opts_Digest field.
698 } else if (crypto_digest_stream_type(stream) != CRYPTO_DIGEST_NONE) {
699 Dmsg2(400, "stream=Digest inx=%d Digest=%s\n", file_index, Opts_Digest);
701 * When ever we get a digest it MUST have been
702 * preceded by an attributes record, which sets attr_file_index
704 if (jcr->FileIndex != (uint32_t)file_index) {
705 Jmsg2(jcr, M_FATAL, 0, _("MD5/SHA1 index %d not same as attributes %d\n"),
706 file_index, jcr->FileIndex);
709 if (do_Digest != CRYPTO_DIGEST_NONE) {
710 db_escape_string(jcr, jcr->db, buf, Opts_Digest, strlen(Opts_Digest));
711 if (strcmp(buf, fdbr.Digest) != 0) {
713 if (debug_level >= 10) {
714 Jmsg(jcr, M_INFO, 0, _(" %s not same. File=%s Cat=%s\n"),
715 stream_to_ascii(stream), buf, fdbr.Digest);
717 Jmsg(jcr, M_INFO, 0, _(" %s differs.\n"),
718 stream_to_ascii(stream));
720 stat = JS_Differences;
722 do_Digest = CRYPTO_DIGEST_NONE;
725 jcr->JobFiles = file_index;
727 if (is_bnet_error(fd)) {
729 Jmsg2(jcr, M_FATAL, 0, _("bdird<filed: bad attributes from filed n=%d : %s\n"),
734 /* Now find all the files that are missing -- i.e. all files in
735 * the database where the MarkId != current JobId
737 jcr->fn_printed = false;
738 bsnprintf(buf, sizeof(buf),
739 "SELECT Path.Path,Filename.Name FROM File,Path,Filename "
740 "WHERE File.JobId=%d "
741 "AND File.MarkId!=%d AND File.PathId=Path.PathId "
742 "AND File.FilenameId=Filename.FilenameId",
744 /* missing_handler is called for each file found */
745 db_sql_query(jcr->db, buf, missing_handler, (void *)jcr);
746 if (jcr->fn_printed) {
747 stat = JS_Differences;
749 free_pool_memory(fname);
750 if (!job_canceled(jcr)) {
751 jcr->JobStatus = stat;
753 return stat == JS_Terminated;
757 * We are called here for each record that matches the above
758 * SQL query -- that is for each file contained in the Catalog
759 * that was not marked earlier. This means that the file in
760 * question is a missing file (in the Catalog but not on Disk).
762 static int missing_handler(void *ctx, int num_fields, char **row)
764 JCR *jcr = (JCR *)ctx;
766 if (job_canceled(jcr)) {
769 if (!jcr->fn_printed) {
770 Jmsg(jcr, M_INFO, 0, _("\nThe following files are in the Catalog but not on %s:\n"),
771 jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG ? "the Volume(s)" : "disk");
772 jcr->fn_printed = true;
774 Jmsg(jcr, M_INFO, 0, " %s%s\n", row[0]?row[0]:"", row[1]?row[1]:"");
780 * Print filename for verify
782 static void prt_fname(JCR *jcr)
784 if (!jcr->fn_printed) {
785 Jmsg(jcr, M_INFO, 0, _("File: %s\n"), jcr->fname);
786 jcr->fn_printed = true;