1 Index: src/dird/backup.c
2 ===================================================================
3 --- src/dird/backup.c (révision 6368)
4 +++ src/dird/backup.c (copie de travail)
9 +#include "findlib/find.h"
11 /* Commands sent to File daemon */
12 static char backupcmd[] = "backup\n";
17 + * We are called here for each record that matches the above
18 + * SQL query -- that is for each file contained in the Catalog
19 + * that was not marked earlier. This means that the file in
20 + * question is a missing file (in the Catalog but not on Disk).
22 +static int missing_handler(void *ctx, int num_fields, char **row)
24 + JCR *jcr = (JCR *)ctx;
26 + if (job_canceled(jcr)) {
30 + /* TODO: return the list to the FD */
31 + Qmsg(jcr, M_INFO, 0, " %s%s\n", row[0]?row[0]:"", row[1]?row[1]:"");
36 + * Accurate backup mode
37 + * 1. Receive the list of all files including those backed up to the Dir
38 + * 2. Dir computes files and deleted files.
39 + * 3. Dir sends list of additional files (new files) to backup, and list of files
42 + * Cleanup attributes (don't use atime, inode etc..)
43 + * Need to insert file and attributes to temp table
44 + * Batch compare files and attributes
48 +bool accurate_compute_files(JCR *jcr)
53 + struct stat statf; /* file stat */
54 + struct stat statc; /* catalog stat */
55 + int stat = JS_Terminated;
56 + char buf[MAXSTRING];
57 + POOLMEM *fname = get_pool_memory(PM_MESSAGE);
58 + int do_Digest = CRYPTO_DIGEST_NONE;
59 + int32_t file_index = 0;
60 + JobId_t JobId=0; /* TODO: compute the job key in new table */
62 + memset(&fdbr, 0, sizeof(FILE_DBR));
63 + fd = jcr->file_bsock;
67 + Dmsg0(20, "bdird: waiting to receive file attributes\n");
69 + * Get Attributes and Signature from File daemon
73 + * Options or Digest (MD5/SHA1)
78 + while ((n=bget_dirmsg(fd)) >= 0 && !job_canceled(jcr)) {
80 + char *attr, *p, *fn;
81 + char Opts_Digest[MAXSTRING]; /* Verify Opts or MD5/SHA1 digest */
83 + if (job_canceled(jcr)) {
86 + fname = check_pool_memory_size(fname, fd->msglen);
87 + jcr->fname = check_pool_memory_size(jcr->fname, fd->msglen);
88 + Dmsg1(20, "Atts+Digest=%s\n", fd->msg);
89 + if ((len = sscanf(fd->msg, "%ld %d %100s", &file_index, &stream,
91 + Jmsg3(jcr, M_FATAL, 0, _("bird<filed: bad attributes, expected 3 fields got %d\n"
92 +" mslen=%d msg=%s\n"), len, fd->msglen, fd->msg);
96 + * We read the Options or Signature into fname
97 + * to prevent overrun, now copy it to proper location.
99 + bstrncpy(Opts_Digest, fname, sizeof(Opts_Digest));
101 + skip_nonspaces(&p); /* skip FileIndex */
103 + skip_nonspaces(&p); /* skip Stream */
105 + skip_nonspaces(&p); /* skip Opts_Digest */
106 + p++; /* skip space */
109 + *fn++ = *p++; /* copy filename */
111 + *fn = *p++; /* term filename and point to attribs */
114 + * Got attributes stream, decode it
116 + if (stream == STREAM_UNIX_ATTRIBUTES || stream == STREAM_UNIX_ATTRIBUTES_EX) {
117 + int32_t LinkFIf, LinkFIc;
118 + Dmsg2(400, "file_index=%d attr=%s\n", file_index, attr);
120 + jcr->FileIndex = file_index; /* remember attribute file_index */
121 + decode_stat(attr, &statf, &LinkFIf); /* decode file stat packet */
122 + do_Digest = CRYPTO_DIGEST_NONE;
123 + pm_strcpy(jcr->fname, fname); /* move filename into JCR */
125 + Dmsg3(040, "dird<filed: stream=%d %s %s\n", stream, jcr->fname, attr);
128 + * Find equivalent record in the database
131 +// if (!db_get_file_attributes_record(jcr, jcr->db, jcr->fname,
132 +// &jcr->previous_jr, &fdbr)) {
134 + Jmsg(jcr, M_INFO, 0, _("New file: %s\n"), jcr->fname);
135 + Dmsg1(020, _("File not in catalog: %s\n"), jcr->fname);
139 + * mark file record as visited by stuffing the
140 + * current JobId, which is unique, into the MarkId field.
142 + db_mark_file_record(jcr, jcr->db, fdbr.FileId, jcr->JobId);
145 + Dmsg3(400, "Found %s in catalog. inx=%d Opts=%s\n", jcr->fname,
146 + file_index, Opts_Digest);
147 + decode_stat(fdbr.LStat, &statc, &LinkFIc); /* decode catalog stat */
149 + // TODO: for each JS_Differences, send it to FD for backup
151 + * Loop over options supplied by user and verify the
152 + * fields he requests.
154 + for (p=Opts_Digest; *p; p++) {
155 + char ed1[30], ed2[30];
157 + case 'i': /* compare INODEs */
158 + if (statc.st_ino != statf.st_ino) {
159 + Jmsg(jcr, M_INFO, 0, _(" st_ino differ. Cat: %s File: %s\n"),
160 + edit_uint64((uint64_t)statc.st_ino, ed1),
161 + edit_uint64((uint64_t)statf.st_ino, ed2));
162 + stat = JS_Differences;
165 + case 'p': /* permissions bits */
166 + if (statc.st_mode != statf.st_mode) {
167 + Jmsg(jcr, M_INFO, 0, _(" st_mode differ. Cat: %x File: %x\n"),
168 + (uint32_t)statc.st_mode, (uint32_t)statf.st_mode);
169 + stat = JS_Differences;
172 + case 'n': /* number of links */
173 + if (statc.st_nlink != statf.st_nlink) {
174 + Jmsg(jcr, M_INFO, 0, _(" st_nlink differ. Cat: %d File: %d\n"),
175 + (uint32_t)statc.st_nlink, (uint32_t)statf.st_nlink);
176 + stat = JS_Differences;
179 + case 'u': /* user id */
180 + if (statc.st_uid != statf.st_uid) {
181 + Jmsg(jcr, M_INFO, 0, _(" st_uid differ. Cat: %u File: %u\n"),
182 + (uint32_t)statc.st_uid, (uint32_t)statf.st_uid);
183 + stat = JS_Differences;
186 + case 'g': /* group id */
187 + if (statc.st_gid != statf.st_gid) {
188 + Jmsg(jcr, M_INFO, 0, _(" st_gid differ. Cat: %u File: %u\n"),
189 + (uint32_t)statc.st_gid, (uint32_t)statf.st_gid);
190 + stat = JS_Differences;
193 + case 's': /* size */
194 + if (statc.st_size != statf.st_size) {
195 + Jmsg(jcr, M_INFO, 0, _(" st_size differ. Cat: %s File: %s\n"),
196 + edit_uint64((uint64_t)statc.st_size, ed1),
197 + edit_uint64((uint64_t)statf.st_size, ed2));
198 + stat = JS_Differences;
201 + case 'a': /* access time */
202 + if (statc.st_atime != statf.st_atime) {
203 + Jmsg(jcr, M_INFO, 0, _(" st_atime differs\n"));
204 + stat = JS_Differences;
208 + if (statc.st_mtime != statf.st_mtime) {
209 + Jmsg(jcr, M_INFO, 0, _(" st_mtime differs\n"));
210 + stat = JS_Differences;
213 + case 'c': /* ctime */
214 + if (statc.st_ctime != statf.st_ctime) {
215 + Jmsg(jcr, M_INFO, 0, _(" st_ctime differs\n"));
216 + stat = JS_Differences;
219 + case 'd': /* file size decrease */
220 + if (statc.st_size > statf.st_size) {
221 + Jmsg(jcr, M_INFO, 0, _(" st_size decrease. Cat: %s File: %s\n"),
222 + edit_uint64((uint64_t)statc.st_size, ed1),
223 + edit_uint64((uint64_t)statf.st_size, ed2));
224 + stat = JS_Differences;
227 + case '5': /* compare MD5 */
228 + Dmsg1(500, "set Do_MD5 for %s\n", jcr->fname);
229 + do_Digest = CRYPTO_DIGEST_MD5;
231 + case '1': /* compare SHA1 */
232 + do_Digest = CRYPTO_DIGEST_SHA1;
241 + * Got Digest Signature from Storage daemon
242 + * It came across in the Opts_Digest field.
244 + } else if (crypto_digest_stream_type(stream) != CRYPTO_DIGEST_NONE) {
245 + Dmsg2(400, "stream=Digest inx=%d Digest=%s\n", file_index, Opts_Digest);
247 + * When ever we get a digest it MUST have been
248 + * preceded by an attributes record, which sets attr_file_index
250 + if (jcr->FileIndex != (uint32_t)file_index) {
251 + Jmsg2(jcr, M_FATAL, 0, _("MD5/SHA1 index %d not same as attributes %d\n"),
252 + file_index, jcr->FileIndex);
255 + if (do_Digest != CRYPTO_DIGEST_NONE) {
256 + db_escape_string(jcr, jcr->db, buf, Opts_Digest, strlen(Opts_Digest));
257 + if (strcmp(buf, fdbr.Digest) != 0) {
258 + if (debug_level >= 10) {
259 + Jmsg(jcr, M_INFO, 0, _(" %d not same. File=%s Cat=%s\n"),
260 + stream, buf, fdbr.Digest);
262 + Jmsg(jcr, M_INFO, 0, _(" %d differs.\n"),
265 + stat = JS_Differences;
267 + do_Digest = CRYPTO_DIGEST_NONE;
270 +// jcr->JobFiles = file_index;
272 + if (is_bnet_error(fd)) {
274 + Jmsg2(jcr, M_FATAL, 0, _("bdird<filed: bad attributes from filed n=%d : %s\n"),
275 + n, be.bstrerror());
279 + /* Now find all the files that are missing -- i.e. all files in
280 + * the database where the MarkId != current JobId
282 + bsnprintf(buf, sizeof(buf),
283 + "SELECT Path.Path,Filename.Name FROM File,Path,Filename "
284 + "WHERE File.JobId=%d "
285 + "AND File.MarkId!=%d AND File.PathId=Path.PathId "
286 + "AND File.FilenameId=Filename.FilenameId",
287 + JobId, jcr->JobId);
288 + /* missing_handler is called for each file found */
289 + db_sql_query(jcr->db, buf, missing_handler, (void *)jcr);
291 + free_pool_memory(fname);
297 * Do a backup of the specified FileSet
299 * Returns: false on failure
305 + * If backup is in accurate mode, FD will send the list of
306 + * all files. We have to store it, and compute witch files
307 + * have been deleted and witch files have to be backuped.
309 + accurate_compute_files(jcr);
311 /* Pickup Job termination data */
312 stat = wait_for_job_termination(jcr);
313 db_write_batch_file_records(jcr); /* used by bulk batch file insert */
314 Index: src/dird/inc_conf.c
315 ===================================================================
316 --- src/dird/inc_conf.c (révision 6368)
317 +++ src/dird/inc_conf.c (copie de travail)
319 * Items that are valid in an Options resource
321 static RES_ITEM options_items[] = {
322 + {"accurate", store_opts, {0}, 0, 0, 0},
323 {"compression", store_opts, {0}, 0, 0, 0},
324 {"signature", store_opts, {0}, 0, 0, 0},
325 {"verify", store_opts, {0}, 0, 0, 0},
337 * options given above.
339 static struct s_kw FS_option_kw[] = {
340 + {"accurate", INC_KW_ACCURATE},
341 {"compression", INC_KW_COMPRESSION},
342 {"signature", INC_KW_DIGEST},
343 {"encryption", INC_KW_ENCRYPTION},
345 {"no", INC_KW_ENHANCEDWILD, "0"},
346 {"yes", INC_KW_CHKCHANGES, "c"},
347 {"no", INC_KW_CHKCHANGES, "0"},
348 + {"yes", INC_KW_ACCURATE, "C"},
349 + {"no", INC_KW_ACCURATE, "0"},
353 Index: src/dird/dird_conf.c
354 ===================================================================
355 --- src/dird/dird_conf.c (révision 6368)
356 +++ src/dird/dird_conf.c (copie de travail)
358 {"selectionpattern", store_str, ITEM(res_job.selection_pattern), 0, 0, 0},
359 {"runscript", store_runscript, ITEM(res_job.RunScripts), 0, ITEM_NO_EQUALS, 0},
360 {"selectiontype", store_migtype, ITEM(res_job.selection_type), 0, 0, 0},
361 + {"accuratebackup", store_bool, ITEM(res_job.accurate), 0,0,0},
362 {NULL, NULL, {0}, 0, 0, 0}
366 if (res->res_job.spool_size) {
367 sendit(sock, _(" SpoolSize=%s\n"), edit_uint64(res->res_job.spool_size, ed1));
369 + if (res->res_job.JobType == JT_BACKUP) {
370 + sendit(sock, _(" Accurate=%d\n"), res->res_job.accurate);
372 if (res->res_job.JobType == JT_MIGRATE) {
373 sendit(sock, _(" SelectionType=%d\n"), res->res_job.selection_type);
375 Index: src/dird/dird_conf.h
376 ===================================================================
377 --- src/dird/dird_conf.h (révision 6368)
378 +++ src/dird/dird_conf.h (copie de travail)
380 bool write_part_after_job; /* Set to write part after job in SD */
381 bool enabled; /* Set if job enabled */
382 bool OptimizeJobScheduling; /* Set if we should optimize Job scheduling */
383 + bool accurate; /* Set if it is an accurate backup job */
385 MSGS *messages; /* How and where to send messages */
386 SCHED *schedule; /* When -- Automatic schedule */
387 Index: src/filed/backup.c
388 ===================================================================
389 --- src/filed/backup.c (révision 6368)
390 +++ src/filed/backup.c (copie de travail)
392 static void crypto_session_end(JCR *jcr);
393 static bool crypto_session_send(JCR *jcr, BSOCK *sd);
395 +#define backup_stat(x,y,z) (x.z = y.z ; y.z = 0)
398 + * Called by save_file when accept/discard file for backup
399 + * TODO: we could add MD5/SHAX digest, but we have to compute it
402 +static bool accurate_add_file(JCR *jcr, FF_PKT *ff_pkt, char *stats)
405 + char attribs[MAXSTRING];
406 + uint32_t file_index=jcr->JobFiles;
407 + BSOCK *dir = jcr->dir_bsock;
410 + if (jcr->accurate == false) {
416 + encode_stat(attribs, ff_pkt, 0);
420 + if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) {
421 + stat = dir->fsend("%d %d %s %s%c%s%c%s%c", file_index,
422 + STREAM_UNIX_ATTRIBUTES, ff_pkt->VerifyOpts, ff_pkt->fname,
423 + 0, a, 0, ff_pkt->link, 0);
424 + } else if (ff_pkt->type == FT_DIREND || ff_pkt->type == FT_REPARSE) {
425 + /* Here link is the canonical filename (i.e. with trailing slash) */
426 + stat = dir->fsend("%d %d %s %s%c%s%c%c", file_index,
427 + STREAM_UNIX_ATTRIBUTES, ff_pkt->VerifyOpts, ff_pkt->link,
430 + stat = dir->fsend("%d %d %s %s%c%s%c%c", file_index,
431 + STREAM_UNIX_ATTRIBUTES, ff_pkt->VerifyOpts, ff_pkt->fname,
436 + Jmsg(jcr, M_FATAL, 0, _("Network error in send to Director: ERR=%s\n"), bnet_strerror(dir));
443 +/* build a fileset with new files from director */
444 +static bool accurate_get_new_and_deleted_file_list(JCR *jcr)
446 + if (jcr->accurate == false || job_canceled(jcr)) {
452 +/* send deleted file list to stored */
453 +static bool accurate_send_deleted_list(JCR *jcr)
455 + if (jcr->accurate == false || job_canceled(jcr)) {
461 +static bool accurate_send_file_list(JCR *jcr)
463 + if (jcr->accurate == false || job_canceled(jcr)) {
466 + Dmsg0(1, "Sending BNET_EOD\n");
467 + jcr->dir_bsock->signal(BNET_EOD); /* end of sending data */
473 * Find all the requested files and send them
474 * to the Storage daemon.
479 // TODO landonf: Allow user to specify encryption algorithm
480 + jcr->accurate=true; /* TODO: remove that */
482 sd = jcr->store_bsock;
485 set_jcr_job_status(jcr, JS_ErrorTerminated);
488 + /* start accurate stuffs */
489 + if (jcr->accurate) {
490 + /* TODO: test job_canceled() */
491 + accurate_send_file_list(jcr); /* send all files to DIR */
492 + accurate_get_new_and_deleted_file_list(jcr); /* get a new incr fileset from DIR */
493 +// set_find_options((FF_PKT *)jcr->ff, 0, 0); /* we backup all that director wants */
494 +// if (!find_files(jcr, (FF_PKT *)jcr->ff, save_file, (void *)jcr)) {
495 +// ok = false; /* error */
496 +// set_jcr_job_status(jcr, JS_ErrorTerminated);
498 +// accurate_send_file_list(jcr); /* send all new files to DIR */
499 + accurate_send_deleted_list(jcr); /* send deleted list to SD */
502 free_pool_memory(jcr->acl_text);
504 stop_heartbeat_monitor(jcr);
508 Jmsg(jcr, M_SKIPPED, 1, _(" Unchanged file skipped: %s\n"), ff_pkt->fname);
509 + accurate_add_file(jcr, ff_pkt, NULL); /* list skipped files */
512 Jmsg(jcr, M_NOTSAVED, 0, _(" Archive file not saved: %s\n"), ff_pkt->fname);
513 + accurate_add_file(jcr, ff_pkt, NULL); /* list skipped files */
517 @@ -1109,6 +1203,9 @@
519 unstrip_path(ff_pkt);
521 + /* list backuped files */
522 + accurate_add_file(jcr, ff_pkt, attribs);
524 Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg);
526 Jmsg1(jcr, M_FATAL, 0, _("Network send error to SD. ERR=%s\n"),
527 Index: src/filed/job.c
528 ===================================================================
529 --- src/filed/job.c (révision 6368)
530 +++ src/filed/job.c (copie de travail)
531 @@ -1087,6 +1087,9 @@
533 fo->flags |= FO_CHKCHANGES;
536 + fo->flags |= FO_ACCURATE;
539 Emsg1(M_ERROR, 0, _("Unknown include/exclude option: %c\n"), *p);
541 Index: src/cats/sql_create.c
542 ===================================================================
543 --- src/cats/sql_create.c (révision 6368)
544 +++ src/cats/sql_create.c (copie de travail)
549 +bool db_accurate_insert(JCR *jcr, B_DB *mdb, bool saved, const char *fname, struct stat *stat)
552 + split_path_and_file(jcr, mdb, fname);
553 + /* make like in Verify code */
558 * Create File record in B_DB
561 ===================================================================
562 --- src/jcr.h (révision 6368)
563 +++ src/jcr.h (copie de travail)
565 B_DB *db_batch; /* database pointer for batch insert */
566 ATTR_DBR *ar; /* DB attribute record */
567 guid_list *id_list; /* User/group id to name list */
568 + bool accurate; /* true if job is accurate */
572 Index: src/findlib/find.h
573 ===================================================================
574 --- src/findlib/find.h (révision 6368)
575 +++ src/findlib/find.h (copie de travail)
577 #define FO_ENHANCEDWILD (1<<23) /* Enhanced wild card processing */
578 #define FO_CHKCHANGES (1<<24) /* Check if file have been modified during backup */
579 #define FO_STRIPPATH (1<<25) /* Check for stripping path */
580 +#define FO_ACCURATE (1<<26) /* Accurate mode */
582 struct s_included_file {
583 struct s_included_file *next;