From: Eric Bollengier Date: Mon, 25 Feb 2008 10:29:28 +0000 (+0000) Subject: ebl Commit accurate patch project. X-Git-Tag: Release-3.0.0~1803 X-Git-Url: https://git.sur5r.net/?a=commitdiff_plain;h=ab26b486ce077fe2a52e6afd7f8d7a3b4fb4442d;p=bacula%2Fbacula ebl Commit accurate patch project. git-svn-id: https://bacula.svn.sourceforge.net/svnroot/bacula/trunk@6486 91ce42f0-d328-0410-95d8-f526ca767f89 --- diff --git a/bacula/src/baconfig.h b/bacula/src/baconfig.h index 0891c40e53..49c8624028 100644 --- a/bacula/src/baconfig.h +++ b/bacula/src/baconfig.h @@ -277,6 +277,7 @@ void InitWinAPIWrapper(); #define FT_INVALIDDT 20 /* Drive type not allowed for */ #define FT_REPARSE 21 /* Win NTFS reparse point */ #define FT_PLUGIN 22 /* Plugin generated filename */ +#define FT_DELETED 23 /* Deleted file entry */ /* Definitions for upper part of type word (see above). */ #define AR_DATA_STREAM (1<<16) /* Data stream id present */ diff --git a/bacula/src/cats/protos.h b/bacula/src/cats/protos.h index de80996149..c13883494b 100644 --- a/bacula/src/cats/protos.h +++ b/bacula/src/cats/protos.h @@ -102,6 +102,9 @@ int db_get_job_volume_parameters(JCR *jcr, B_DB *mdb, JobId_t JobId, VOL_PARAMS int db_get_client_record(JCR *jcr, B_DB *mdb, CLIENT_DBR *cdbr); int db_get_counter_record(JCR *jcr, B_DB *mdb, COUNTER_DBR *cr); bool db_get_query_dbids(JCR *jcr, B_DB *mdb, POOL_MEM &query, dbid_list &ids); +bool db_get_file_list(JCR *jcr, B_DB *mdb, char *jobids, DB_RESULT_HANDLER *result_handler, void *ctx); +bool db_accurate_get_jobids(JCR *jcr, B_DB *mdb, JOB_DBR *jr, POOLMEM *jobids); +int db_get_int_handler(void *ctx, int num_fields, char **row); /* sql_list.c */ diff --git a/bacula/src/cats/sql_get.c b/bacula/src/cats/sql_get.c index f53257436d..1e9e347537 100644 --- a/bacula/src/cats/sql_get.c +++ b/bacula/src/cats/sql_get.c @@ -898,8 +898,6 @@ bool db_get_query_dbids(JCR *jcr, B_DB *mdb, POOL_MEM &query, dbid_list &ids) return ok; } - - /* Get Media Record * * Returns: false: on failure @@ -1018,5 +1016,141 @@ bool db_get_media_record(JCR *jcr, B_DB *mdb, MEDIA_DBR *mr) return ok; } +/* + * Find the last "accurate" backup state (that can take deleted files in account) + * 1) Get all files with jobid in list (F subquery) + * 2) Take only the last version of each file (Temp subquery) => accurate list is ok + * 3) Join the result to file table to get fileindex, jobid and lstat information + * + * TODO: On postgresql, this is done with +SELECT DISTINCT ON (PathId, FilenameId) FileIndex, Path, Name, LStat + FROM File JOIN Filename USING (FilenameId) JOIN Path USING (PathId) WHERE JobId IN (40341) + ORDER BY PathId, FilenameId, JobId DESC + */ +bool db_get_file_list(JCR *jcr, B_DB *mdb, char *jobids, + DB_RESULT_HANDLER *result_handler, void *ctx) +{ + if (!*jobids) { + db_lock(mdb); + Mmsg(mdb->errmsg, _("ERR=JobIds are empty\n")); + db_unlock(mdb); + return false; + } + + POOL_MEM buf (PM_MESSAGE); + + Mmsg(buf, + "SELECT Path.Path, Filename.Name, File.FileIndex, File.JobId, File.LStat " + "FROM ( " + "SELECT max(FileId) as FileId, PathId, FilenameId " + "FROM (SELECT FileId, PathId, FilenameId FROM File WHERE JobId IN (%s)) AS F " + "GROUP BY PathId, FilenameId " + ") AS Temp " + "JOIN Filename ON (Filename.FilenameId = Temp.FilenameId) " + "JOIN Path ON (Path.PathId = Temp.PathId) " + "JOIN File ON (File.FileId = Temp.FileId) " + "WHERE File.FileIndex > 0 ", + jobids); + + return db_sql_query(mdb, buf.c_str(), result_handler, ctx); +} + + +/* Full : do nothing + * Differential : get the last full id + * Incremental : get the last full + last diff + last incr(s) ids + * + * TODO: look and merge from ua_restore.c + */ +bool db_accurate_get_jobids(JCR *jcr, B_DB *mdb, + JOB_DBR *jr, POOLMEM *jobids) +{ + char clientid[50], jobid[50], filesetid[50]; + char date[MAX_TIME_LENGTH]; + + POOL_MEM query (PM_FNAME); + bstrutime(date, sizeof(date), time(NULL) + 1); + jobids[0]='\0'; + + /* First, find the last good Full backup for this job/client/fileset */ + Mmsg(query, +"CREATE TEMPORARY TABLE btemp3%s AS " + "SELECT JobId, StartTime, EndTime, JobTDate, PurgedFiles " + "FROM Job JOIN FileSet USING (FileSetId) " + "WHERE ClientId = %s " + "AND Level='F' AND JobStatus='T' AND Type='B' " + "AND StartTime<'%s' " + "AND FileSet.FileSet=(SELECT FileSet FROM FileSet WHERE FileSetId = %s) " + "ORDER BY Job.JobTDate DESC LIMIT 1", + edit_uint64(jcr->JobId, jobid), + edit_uint64(jr->ClientId, clientid), + date, + edit_uint64(jr->FileSetId, filesetid)); + + if (!db_sql_query(mdb, query.c_str(), NULL, NULL)) { + return false; + } + + if (jr->JobLevel == L_INCREMENTAL) { + + /* Now, find the last differential backup after the last full */ + Mmsg(query, +"INSERT INTO btemp3%s (JobId, StartTime, EndTime, JobTDate, PurgedFiles) " + "SELECT JobId, StartTime, EndTime, JobTDate, PurgedFiles " + "FROM Job JOIN FileSet USING (FileSetId) " + "WHERE ClientId = %s " + "AND Level='D' AND JobStatus='T' AND Type='B' " + "AND StartTime > (SELECT EndTime FROM btemp3%s ORDER BY EndTime DESC LIMIT 1) " + "AND FileSet.FileSet= (SELECT FileSet FROM FileSet WHERE FileSetId = %s) " + "ORDER BY Job.JobTDate DESC LIMIT 1 ", + jobid, + clientid, + jobid, + filesetid); + + db_sql_query(mdb, query.c_str(), NULL, NULL); + + /* We just have to take all incremental after the last Full/Diff */ + Mmsg(query, +"INSERT INTO btemp3%s (JobId, StartTime, EndTime, JobTDate, PurgedFiles) " + "SELECT JobId, StartTime, EndTime, JobTDate, PurgedFiles " + "FROM Job JOIN FileSet USING (FileSetId) " + "WHERE ClientId = %s " + "AND Level='I' AND JobStatus='T' AND Type='B' " + "AND StartTime > (SELECT EndTime FROM btemp3%s ORDER BY EndTime DESC LIMIT 1) " + "AND FileSet.FileSet= (SELECT FileSet FROM FileSet WHERE FileSetId = %s) " + "ORDER BY Job.JobTDate DESC ", + jobid, + clientid, + jobid, + filesetid); + db_sql_query(mdb, query.c_str(), NULL, NULL); + } + + /* build a jobid list ie: 1,2,3,4 */ + Mmsg(query, "SELECT JobId FROM btemp3%s", jobid); + db_sql_query(mdb, query.c_str(), db_get_int_handler, jobids); + Dmsg1(1, "db_accurate_get_jobids=%s\n", jobids); + + Mmsg(query, "DROP TABLE btemp3%s", jobid); + db_sql_query(mdb, query.c_str(), NULL, NULL); + + return true; +} + +/* + * Use to build a string of int list from a query. "10,20,30" + */ +int db_get_int_handler(void *ctx, int num_fields, char **row) +{ + POOLMEM *ret = (POOLMEM *)ctx; + if (num_fields == 1) { + if (ret[0]) { + pm_strcat(ret, ","); + } + pm_strcat(ret, row[0]); + } + return 0; +} #endif /* HAVE_SQLITE3 || HAVE_MYSQL || HAVE_SQLITE || HAVE_POSTGRESQL || HAVE_DBI */ diff --git a/bacula/src/dird/backup.c b/bacula/src/dird/backup.c index 57f6f07f40..a8aa1d4f87 100644 --- a/bacula/src/dird/backup.c +++ b/bacula/src/dird/backup.c @@ -96,6 +96,65 @@ bool do_backup_init(JCR *jcr) return true; } +/* + * Foreach files in currrent list, send "/path/fname\0LStat" to FD + */ +static int accurate_list_handler(void *ctx, int num_fields, char **row) +{ + JCR *jcr = (JCR *)ctx; + + if (job_canceled(jcr)) { + return 1; + } + + if (row[2] > 0) { /* discard when file_index == 0 */ + jcr->file_bsock->fsend("%s%s%c%s", row[0], row[1], 0, row[4]); + } + return 0; +} + +/* + * Send current file list to FD + * DIR -> FD : accurate files=xxxx + * DIR -> FD : /path/to/file\0Lstat + * DIR -> FD : /path/to/dir/\0Lstat + * ... + * DIR -> FD : EOD + */ +bool send_accurate_current_files(JCR *jcr) +{ + POOL_MEM buf; + + if (jcr->accurate==false || job_canceled(jcr) || jcr->JobLevel==L_FULL) { + return true; + } + POOLMEM *jobids = get_pool_memory(PM_FNAME); + db_accurate_get_jobids(jcr, jcr->db, &jcr->jr, jobids); + + if (*jobids == 0) { + free_pool_memory(jobids); + Jmsg(jcr, M_FATAL, 0, _("Cannot find previous jobids.\n")); + return false; + } + Jmsg(jcr, M_INFO, 0, _("Sending Accurate information.\n")); + + /* to be able to allocate the right size for htable */ + POOLMEM *nb = get_pool_memory(PM_FNAME); + Mmsg(buf, "SELECT sum(JobFiles) FROM Job WHERE JobId IN (%s)",jobids); + db_sql_query(jcr->db, buf.c_str(), db_get_int_handler, nb); + jcr->file_bsock->fsend("accurate files=%s\n", nb); + + db_get_file_list(jcr, jcr->db, jobids, accurate_list_handler, (void *)jcr); + + free_pool_memory(jobids); + free_pool_memory(nb); + + jcr->file_bsock->signal(BNET_EOD); + /* TODO: use response() ? */ + + return true; +} + /* * Do a backup of the specified FileSet * @@ -225,6 +284,14 @@ bool do_backup(JCR *jcr) Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db)); } + /* + * If backup is in accurate mode, we send the list of + * all files to FD. + */ + if (!send_accurate_current_files(jcr)) { + goto bail_out; + } + /* Send backup command */ fd->fsend(backupcmd); if (!response(jcr, fd, OKbackup, "backup", DISPLAY_ERROR)) { @@ -476,6 +543,7 @@ void backup_cleanup(JCR *jcr, int TermCode) " Software Compression: %s\n" " VSS: %s\n" " Encryption: %s\n" +" Accurate: %s\n" " Volume name(s): %s\n" " Volume Session Id: %d\n" " Volume Session Time: %d\n" @@ -508,8 +576,9 @@ void backup_cleanup(JCR *jcr, int TermCode) edit_uint64_with_suffix(jcr->SDJobBytes, ec6), kbps, compress, - jcr->VSS?"yes":"no", - jcr->Encrypt?"yes":"no", + jcr->VSS?_("yes"):_("no"), + jcr->Encrypt?_("yes"):_("no"), + jcr->accurate?_("yes"):_("no"), jcr->VolumeName, jcr->VolSessionId, jcr->VolSessionTime, diff --git a/bacula/src/dird/catreq.c b/bacula/src/dird/catreq.c index 8aa7b550b2..8fb433ab43 100644 --- a/bacula/src/dird/catreq.c +++ b/bacula/src/dird/catreq.c @@ -346,8 +346,8 @@ void catalog_request(JCR *jcr, BSOCK *bs) * Update File Attributes in the catalog with data * sent by the Storage daemon. Note, we receive the whole * attribute record, but we select out only the stat packet, - * VolSessionId, VolSessionTime, FileIndex, and file name - * to store in the catalog. + * VolSessionId, VolSessionTime, FileIndex, file type, and + * file name to store in the catalog. */ void catalog_update(JCR *jcr, BSOCK *bs) { @@ -357,6 +357,7 @@ void catalog_update(JCR *jcr, BSOCK *bs) uint32_t FileIndex; uint32_t data_len; char *p; + int filetype; int len; char *fname, *attr; ATTR_DBR *ar = NULL; @@ -415,6 +416,7 @@ void catalog_update(JCR *jcr, BSOCK *bs) p = jcr->attr - bs->msg + p; /* point p into jcr->attr */ skip_nonspaces(&p); /* skip FileIndex */ skip_spaces(&p); + filetype = str_to_int32(p); /* TODO: choose between unserialize and str_to_int32 */ skip_nonspaces(&p); /* skip FileType */ skip_spaces(&p); fname = p; @@ -425,7 +427,11 @@ void catalog_update(JCR *jcr, BSOCK *bs) Dmsg1(400, "dirdattr = attr; ar->fname = fname; - ar->FileIndex = FileIndex; + if (filetype == FT_DELETED) { + ar->FileIndex = 0; /* special value */ + } else { + ar->FileIndex = FileIndex; + } ar->Stream = Stream; ar->link = NULL; if (jcr->mig_jcr) { diff --git a/bacula/src/dird/fd_cmds.c b/bacula/src/dird/fd_cmds.c index cdb1632562..f5ca007fc2 100644 --- a/bacula/src/dird/fd_cmds.c +++ b/bacula/src/dird/fd_cmds.c @@ -50,7 +50,7 @@ const int dbglvl = 400; static char filesetcmd[] = "fileset%s\n"; /* set full fileset */ static char jobcmd[] = "JobId=%s Job=%s SDid=%u SDtime=%u Authorization=%s\n"; /* Note, mtime_only is not used here -- implemented as file option */ -static char levelcmd[] = "level = %s%s mtime_only=%d\n"; +static char levelcmd[] = "level = %s%s%s mtime_only=%d\n"; static char runscript[] = "Run OnSuccess=%u OnFailure=%u AbortOnError=%u When=%u Command=%s\n"; static char runbeforenow[]= "RunBeforeNow\n"; @@ -226,13 +226,12 @@ static void send_since_time(JCR *jcr) char ed1[50]; stime = str_to_utime(jcr->stime); - fd->fsend(levelcmd, NT_("since_utime "), edit_uint64(stime, ed1), 0); + fd->fsend(levelcmd, "", NT_("since_utime "), edit_uint64(stime, ed1), 0); while (bget_dirmsg(fd) >= 0) { /* allow him to poll us to sync clocks */ Jmsg(jcr, M_INFO, 0, "%s\n", fd->msg); } } - /* * Send level command to FD. * Used for backup jobs and estimate command. @@ -240,24 +239,26 @@ static void send_since_time(JCR *jcr) bool send_level_command(JCR *jcr) { BSOCK *fd = jcr->file_bsock; + const char *accurate=jcr->job->accurate?"accurate_":""; + const char *not_accurate=""; /* * Send Level command to File daemon */ switch (jcr->JobLevel) { case L_BASE: - fd->fsend(levelcmd, "base", " ", 0); + fd->fsend(levelcmd, not_accurate, "base", " ", 0); break; /* L_NONE is the console, sending something off to the FD */ case L_NONE: case L_FULL: - fd->fsend(levelcmd, "full", " ", 0); + fd->fsend(levelcmd, not_accurate, "full", " ", 0); break; case L_DIFFERENTIAL: - fd->fsend(levelcmd, "differential", " ", 0); + fd->fsend(levelcmd, accurate, "differential", " ", 0); send_since_time(jcr); break; case L_INCREMENTAL: - fd->fsend(levelcmd, "incremental", " ", 0); + fd->fsend(levelcmd, accurate, "incremental", " ", 0); send_since_time(jcr); break; case L_SINCE: diff --git a/bacula/src/dird/inc_conf.c b/bacula/src/dird/inc_conf.c index 50fd28e777..b7a52e1acc 100644 --- a/bacula/src/dird/inc_conf.c +++ b/bacula/src/dird/inc_conf.c @@ -96,6 +96,7 @@ static RES_ITEM newinc_items[] = { static RES_ITEM options_items[] = { {"compression", store_opts, {0}, 0, 0, 0}, {"signature", store_opts, {0}, 0, 0, 0}, + {"accurate", store_opts, {0}, 0, 0, 0}, {"verify", store_opts, {0}, 0, 0, 0}, {"onefs", store_opts, {0}, 0, 0, 0}, {"recurse", store_opts, {0}, 0, 0, 0}, @@ -137,6 +138,7 @@ enum { INC_KW_DIGEST, INC_KW_ENCRYPTION, INC_KW_VERIFY, + INC_KW_ACCURATE, INC_KW_ONEFS, INC_KW_RECURSE, INC_KW_SPARSE, @@ -167,6 +169,7 @@ static struct s_kw FS_option_kw[] = { {"signature", INC_KW_DIGEST}, {"encryption", INC_KW_ENCRYPTION}, {"verify", INC_KW_VERIFY}, + {"accurate", INC_KW_ACCURATE}, {"onefs", INC_KW_ONEFS}, {"recurse", INC_KW_RECURSE}, {"sparse", INC_KW_SPARSE}, @@ -278,6 +281,12 @@ static void scan_include_options(LEX *lc, int keyword, char *opts, int optlen) bstrncat(opts, lc->str, optlen); bstrncat(opts, ":", optlen); /* terminate it */ Dmsg3(900, "Catopts=%s option=%s optlen=%d\n", opts, option,optlen); + } else if (keyword == INC_KW_ACCURATE) { /* special case */ + /* ***FIXME**** ensure these are in permitted set */ + bstrncat(opts, "C", optlen); /* indicate Accurate */ + bstrncat(opts, lc->str, optlen); + bstrncat(opts, ":", optlen); /* terminate it */ + Dmsg3(900, "Catopts=%s option=%s optlen=%d\n", opts, option,optlen); } else if (keyword == INC_KW_STRIPPATH) { /* another special case */ if (!is_an_integer(lc->str)) { scan_err1(lc, _("Expected a strip path positive integer, got:%s:"), lc->str); diff --git a/bacula/src/dird/ua_restore.c b/bacula/src/dird/ua_restore.c index 9d9263b70a..e0a364fe3d 100644 --- a/bacula/src/dird/ua_restore.c +++ b/bacula/src/dird/ua_restore.c @@ -1005,7 +1005,6 @@ static bool build_directory_tree(UAContext *ua, RESTORE_CTX *rx) * For display purposes, the same JobId, with different volumes may * appear more than once, however, we only insert it once. */ - int items = 0; p = rx->JobIds; tree.FileEstimate = 0; if (get_next_jobid_from_list(&p, &JobId) > 0) { @@ -1020,23 +1019,12 @@ static bool build_directory_tree(UAContext *ua, RESTORE_CTX *rx) tree.DeltaCount = rx->JobId/50; /* print 50 ticks */ } } - for (p=rx->JobIds; get_next_jobid_from_list(&p, &JobId) > 0; ) { - char ed1[50]; - if (JobId == last_JobId) { - continue; /* eliminate duplicate JobIds */ - } - last_JobId = JobId; - ua->info_msg(_("\nBuilding directory tree for JobId %s ... "), - edit_int64(JobId, ed1)); - items++; - /* - * Find files for this JobId and insert them in the tree - */ - Mmsg(rx->query, uar_sel_files, edit_int64(JobId, ed1)); - if (!db_sql_query(ua->db, rx->query, insert_tree_handler, (void *)&tree)) { - ua->error_msg("%s", db_strerror(ua->db)); - } + ua->info_msg(_("\nBuilding directory tree for JobId(s) %s ... "), + rx->JobIds); + + if (!db_get_file_list(ua->jcr, ua->db, rx->JobIds, insert_tree_handler, (void *)&tree)) { + ua->error_msg("%s", db_strerror(ua->db)); } if (tree.FileCount == 0) { ua->send_msg(_("\nThere were no files inserted into the tree, so file selection\n" @@ -1055,25 +1043,12 @@ static bool build_directory_tree(UAContext *ua, RESTORE_CTX *rx) } } else { char ec1[50]; - if (items==1) { - if (tree.all) { - ua->info_msg(_("\n1 Job, %s files inserted into the tree and marked for extraction.\n"), - edit_uint64_with_commas(tree.FileCount, ec1)); - } - else { - ua->info_msg(_("\n1 Job, %s files inserted into the tree.\n"), - edit_uint64_with_commas(tree.FileCount, ec1)); - } - } - else { - if (tree.all) { - ua->info_msg(_("\n%d Jobs, %s files inserted into the tree and marked for extraction.\n"), - items, edit_uint64_with_commas(tree.FileCount, ec1)); - } - else { - ua->info_msg(_("\n%d Jobs, %s files inserted into the tree.\n"), - items, edit_uint64_with_commas(tree.FileCount, ec1)); - } + if (tree.all) { + ua->info_msg(_("\n%s files inserted into the tree and marked for extraction.\n"), + edit_uint64_with_commas(tree.FileCount, ec1)); + } else { + ua->info_msg(_("\n%s files inserted into the tree.\n"), + edit_uint64_with_commas(tree.FileCount, ec1)); } if (find_arg(ua, NT_("done")) < 0) { diff --git a/bacula/src/filed/backup.c b/bacula/src/filed/backup.c index 6bad93b1d7..0f86e2247c 100644 --- a/bacula/src/filed/backup.c +++ b/bacula/src/filed/backup.c @@ -37,6 +37,7 @@ #include "bacula.h" #include "filed.h" +#include "lib/htable.h" /* Forward referenced functions */ int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level); @@ -49,6 +50,254 @@ static bool crypto_session_start(JCR *jcr); static void crypto_session_end(JCR *jcr); static bool crypto_session_send(JCR *jcr, BSOCK *sd); +typedef struct CurFile { + hlink link; + char *fname; + char *lstat; + bool seen; +} CurFile; + +#define accurate_mark_file_as_seen(elt) ((elt)->seen = 1) +#define accurate_file_has_been_seen(elt) ((elt)->seen) + +/* + * This function is called for each file seen in fileset. + * We check in file_list hash if fname have been backuped + * the last time. After we can compare Lstat field. + * + */ +/* TODO: tweak verify code to use the same function ?? */ +bool accurate_check_file(JCR *jcr, FF_PKT *ff_pkt) +{ + char *p; + int stat=false; + struct stat statc; /* catalog stat */ + char *Opts_Digest; + char *fname; + CurFile *elt; + + int32_t LinkFIc; + + if (*ff_pkt->VerifyOpts) { /* use mtime + ctime checks by default */ + Opts_Digest = ff_pkt->VerifyOpts; + } else { + Opts_Digest = "cm"; + } + + if (jcr->accurate == false || jcr->JobLevel == L_FULL) { + return true; + } + + strip_path(ff_pkt); + + if (S_ISDIR(ff_pkt->statp.st_mode)) { + fname = ff_pkt->link; + } else { + fname = ff_pkt->fname; + } + + elt = (CurFile *) jcr->file_list->lookup(fname); + + if (!elt) { + Dmsg1(500, "accurate %s = yes (not found)\n", fname); + stat=true; + goto bail_out; + } + + if (accurate_file_has_been_seen(elt)) { + Dmsg1(500, "accurate %s = no (already seen)\n", fname); + stat=false; + goto bail_out; + } + + decode_stat(elt->lstat, &statc, &LinkFIc); /* decode catalog stat */ +// *do_Digest = CRYPTO_DIGEST_NONE; + + for (p=Opts_Digest; *p; p++) { + char ed1[30], ed2[30]; + switch (*p) { + case 'i': /* compare INODEs */ + if (statc.st_ino != ff_pkt->statp.st_ino) { + Jmsg(jcr, M_SAVED, 0, _("%s st_ino differ. Cat: %s File: %s\n"), fname, + edit_uint64((uint64_t)statc.st_ino, ed1), + edit_uint64((uint64_t)ff_pkt->statp.st_ino, ed2)); + stat = true; + } + break; + case 'p': /* permissions bits */ + if (statc.st_mode != ff_pkt->statp.st_mode) { + Jmsg(jcr, M_SAVED, 0, _("%s st_mode differ. Cat: %x File: %x\n"), fname, + (uint32_t)statc.st_mode, (uint32_t)ff_pkt->statp.st_mode); + stat = true; + } + break; +// case 'n': /* number of links */ +// if (statc.st_nlink != ff_pkt->statp.st_nlink) { +// Jmsg(jcr, M_SAVED, 0, _("%s st_nlink differ. Cat: %d File: %d\n"), fname, +// (uint32_t)statc.st_nlink, (uint32_t)ff_pkt->statp.st_nlink); +// stat = true; +// } +// break; + case 'u': /* user id */ + if (statc.st_uid != ff_pkt->statp.st_uid) { + Jmsg(jcr, M_SAVED, 0, _("%s st_uid differ. Cat: %u File: %u\n"), fname, + (uint32_t)statc.st_uid, (uint32_t)ff_pkt->statp.st_uid); + stat = true; + } + break; + case 'g': /* group id */ + if (statc.st_gid != ff_pkt->statp.st_gid) { + Jmsg(jcr, M_SAVED, 0, _("%s st_gid differ. Cat: %u File: %u\n"), fname, + (uint32_t)statc.st_gid, (uint32_t)ff_pkt->statp.st_gid); + stat = true; + } + break; + case 's': /* size */ + if (statc.st_size != ff_pkt->statp.st_size) { + Jmsg(jcr, M_SAVED, 0, _("%s st_size differ. Cat: %s File: %s\n"), fname, + edit_uint64((uint64_t)statc.st_size, ed1), + edit_uint64((uint64_t)ff_pkt->statp.st_size, ed2)); + stat = true; + } + break; +// case 'a': /* access time */ +// if (statc.st_atime != ff_pkt->statp.st_atime) { +// Jmsg(jcr, M_SAVED, 0, _("%s st_atime differs\n"), fname); +// stat = true; +// } +// break; + case 'm': + if (statc.st_mtime != ff_pkt->statp.st_mtime) { + Jmsg(jcr, M_SAVED, 0, _("%s st_mtime differs\n"), fname); + stat = true; + } + break; + case 'c': /* ctime */ + if (statc.st_ctime != ff_pkt->statp.st_ctime) { + Jmsg(jcr, M_SAVED, 0, _("%s st_ctime differs\n"), fname); + stat = true; + } + break; + case 'd': /* file size decrease */ + if (statc.st_size > ff_pkt->statp.st_size) { + Jmsg(jcr, M_SAVED, 0, _("%s st_size decrease. Cat: %s File: %s\n"), fname, + edit_uint64((uint64_t)statc.st_size, ed1), + edit_uint64((uint64_t)ff_pkt->statp.st_size, ed2)); + stat = true; + } + break; + case '5': /* compare MD5 */ + Dmsg1(500, "set Do_MD5 for %s\n", ff_pkt->fname); +// *do_Digest = CRYPTO_DIGEST_MD5; + break; + case '1': /* compare SHA1 */ +// *do_Digest = CRYPTO_DIGEST_SHA1; + break; + case ':': + case 'V': + default: + break; + } + } + accurate_mark_file_as_seen(elt); + Dmsg2(500, "accurate %s = %i\n", fname, stat); + +bail_out: + unstrip_path(ff_pkt); + return stat; +} + +/* + * This function doesn't work very well with smartalloc + * TODO: use bigbuffer from htable + */ +int accurate_cmd(JCR *jcr) +{ + BSOCK *dir = jcr->dir_bsock; + int len; + uint64_t nb; + CurFile *elt=NULL; + + if (jcr->accurate==false || job_canceled(jcr) || jcr->JobLevel==L_FULL) { + return true; + } + + if (sscanf(dir->msg, "accurate files=%ld", &nb) != 1) { + dir->fsend(_("2991 Bad accurate command\n")); + return false; + } + + jcr->file_list = (htable *)malloc(sizeof(htable)); + jcr->file_list->init(elt, &elt->link, nb); + + /* + * buffer = sizeof(CurFile) + dirmsg + * dirmsg = fname + lstat + */ + /* get current files */ + while (dir->recv() >= 0) { + len = strlen(dir->msg); + if ((len+1) < dir->msglen) { +// elt = (CurFile *)malloc(sizeof(CurFile)); +// elt->fname = (char *) malloc(dir->msglen+1); + + /* we store CurFile, fname and lstat in the same chunk */ + elt = (CurFile *)malloc(sizeof(CurFile)+dir->msglen+1); + elt->fname = (char *) elt+sizeof(CurFile); + memcpy(elt->fname, dir->msg, dir->msglen); + elt->fname[dir->msglen]='\0'; + elt->lstat = elt->fname + len + 1; + elt->seen=0; + jcr->file_list->insert(elt->fname, elt); + Dmsg2(500, "add fname=%s lstat=%s\n", elt->fname, elt->lstat); + } + } + +// jcr->file_list->stats(); + /* TODO: send a EOM ? + dir->fsend("2000 OK accurate\n"); + */ + return true; +} + +bool accurate_send_deleted_list(JCR *jcr) +{ + CurFile *elt; + FF_PKT *ff_pkt; + + int stream = STREAM_UNIX_ATTRIBUTES; + + if (jcr->accurate == false || jcr->JobLevel == L_FULL) { + goto bail_out; + } + + if (jcr->file_list == NULL) { + goto bail_out; + } + + ff_pkt = init_find_files(); + ff_pkt->type = FT_DELETED; + + foreach_htable (elt, jcr->file_list) { + if (!accurate_file_has_been_seen(elt)) { /* already seen */ + Dmsg3(500, "deleted fname=%s lstat=%s seen=%i\n", elt->fname, elt->lstat, elt->seen); + ff_pkt->fname = elt->fname; + decode_stat(elt->lstat, &ff_pkt->statp, &ff_pkt->LinkFI); /* decode catalog stat */ + encode_and_send_attributes(jcr, ff_pkt, stream); + } +// Free(elt->fname); + } + term_find_files(ff_pkt); +bail_out: + /* TODO: clean htable when this function is not reached ? */ + if (jcr->file_list) { + jcr->file_list->destroy(); + free(jcr->file_list); + jcr->file_list = NULL; + } + return true; +} + /* * Find all the requested files and send them * to the Storage daemon. @@ -100,7 +349,7 @@ bool blast_data_to_storage_daemon(JCR *jcr, char *addr) */ jcr->compress_buf_size = jcr->buf_size + ((jcr->buf_size+999) / 1000) + 30; jcr->compress_buf = get_memory(jcr->compress_buf_size); - + #ifdef HAVE_LIBZ z_stream *pZlibStream = (z_stream*)malloc(sizeof(z_stream)); if (pZlibStream) { @@ -121,10 +370,13 @@ bool blast_data_to_storage_daemon(JCR *jcr, char *addr) return false; } - Dmsg1(300, "set_find_options ff=%p\n", jcr->ff); set_find_options((FF_PKT *)jcr->ff, jcr->incremental, jcr->mtime); - Dmsg0(300, "start find files\n"); + /* in accurate mode, we overwrite the find_one check function */ + if (jcr->accurate) { + set_find_changed_function((FF_PKT *)jcr->ff, accurate_check_file); + } + start_heartbeat_monitor(jcr); jcr->acl_text = get_pool_memory(PM_MESSAGE); @@ -135,6 +387,8 @@ bool blast_data_to_storage_daemon(JCR *jcr, char *addr) set_jcr_job_status(jcr, JS_ErrorTerminated); } + accurate_send_deleted_list(jcr); /* send deleted list to SD */ + free_pool_memory(jcr->acl_text); stop_heartbeat_monitor(jcr); @@ -1102,7 +1356,9 @@ static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_strea * For a directory, link is the same as fname, but with trailing * slash. For a linked file, link is the link. */ - strip_path(ff_pkt); + if (ff_pkt->type != FT_DELETED) { /* already stripped */ + strip_path(ff_pkt); + } if (ff_pkt->type == FT_LNK || ff_pkt->type == FT_LNKSAVED) { Dmsg2(300, "Link %s to %s\n", ff_pkt->fname, ff_pkt->link); stat = sd->fsend("%ld %d %s%c%s%c%s%c%s%c", jcr->JobFiles, @@ -1116,7 +1372,9 @@ static bool encode_and_send_attributes(JCR *jcr, FF_PKT *ff_pkt, int &data_strea stat = sd->fsend("%ld %d %s%c%s%c%c%s%c", jcr->JobFiles, ff_pkt->type, ff_pkt->fname, 0, attribs, 0, 0, attribsEx, 0); } - unstrip_path(ff_pkt); + if (ff_pkt->type != FT_DELETED) { + unstrip_path(ff_pkt); + } Dmsg2(300, ">stored: attr len=%d: %s\n", sd->msglen, sd->msg); if (!stat) { diff --git a/bacula/src/filed/job.c b/bacula/src/filed/job.c index 23921df0c8..9e1f8d2de4 100644 --- a/bacula/src/filed/job.c +++ b/bacula/src/filed/job.c @@ -49,6 +49,7 @@ extern CLIENT *me; /* our client resource */ /* Imported functions */ extern int status_cmd(JCR *jcr); extern int qstatus_cmd(JCR *jcr); +extern int accurate_cmd(JCR *jcr); /* Forward referenced functions */ static int backup_cmd(JCR *jcr); @@ -106,6 +107,7 @@ static struct s_cmds cmds[] = { {"RunBeforeJob", runbefore_cmd, 0}, {"RunAfterJob", runafter_cmd, 0}, {"Run", runscript_cmd, 0}, + {"accurate", accurate_cmd, 0}, {NULL, NULL} /* list terminator */ }; @@ -1057,6 +1059,16 @@ static void set_options(findFOPTS *fo, const char *opts) } fo->VerifyOpts[j] = 0; break; + case 'C': /* accurate options */ + /* Copy Accurate Options */ + for (j=0; *p && *p != ':'; p++) { + fo->AccurateOpts[j] = *p; + if (j < (int)sizeof(fo->AccurateOpts) - 1) { + j++; + } + } + fo->AccurateOpts[j] = 0; + break; case 'P': /* strip path */ /* Get integer */ p++; /* skip P */ @@ -1195,6 +1207,9 @@ static int level_cmd(JCR *jcr) level = get_memory(dir->msglen+1); Dmsg1(110, "level_cmd: %s", dir->msg); + if (strstr(dir->msg, "accurate")) { + jcr->accurate = true; + } if (sscanf(dir->msg, "level = %s ", level) != 1) { goto bail_out; } @@ -1204,14 +1219,14 @@ static int level_cmd(JCR *jcr) /* Full backup requested? */ } else if (strcmp(level, "full") == 0) { jcr->JobLevel = L_FULL; - } else if (strcmp(level, "differential") == 0) { + } else if (strstr(level, "differential")) { jcr->JobLevel = L_DIFFERENTIAL; free_memory(level); return 1; - } else if (strcmp(level, "incremental") == 0) { + } else if (strstr(level, "incremental")) { jcr->JobLevel = L_INCREMENTAL; free_memory(level); - return 1; + return 1; /* * We get his UTC since time, then sync the clocks and correct it * to agree with our clock. diff --git a/bacula/src/filed/restore.c b/bacula/src/filed/restore.c index 33c02c1c39..0a0aa6a25e 100644 --- a/bacula/src/filed/restore.c +++ b/bacula/src/filed/restore.c @@ -320,6 +320,11 @@ void do_restore(JCR *jcr) bclose(&rctx.bfd); } + /* TODO: manage deleted files */ + if (rctx.type == FT_DELETED) { /* deleted file */ + continue; + } + /* * Unpack attributes and do sanity check them */ diff --git a/bacula/src/findlib/create_file.c b/bacula/src/findlib/create_file.c index ea329d8d87..09a645c5bf 100644 --- a/bacula/src/findlib/create_file.c +++ b/bacula/src/findlib/create_file.c @@ -389,6 +389,9 @@ int create_file(JCR *jcr, ATTR *attr, BFILE *bfd, int replace) return CF_CREATED; } + case FT_DELETED: + Qmsg2(jcr, M_INFO, 0, _("Original file %s have been deleted: type=%d\n"), attr->fname, attr->type); + break; /* The following should not occur */ case FT_NOACCESS: case FT_NOFOLLOW: diff --git a/bacula/src/findlib/find.c b/bacula/src/findlib/find.c index c231115a6a..88b0bd08d4 100644 --- a/bacula/src/findlib/find.c +++ b/bacula/src/findlib/find.c @@ -96,6 +96,13 @@ set_find_options(FF_PKT *ff, int incremental, time_t save_time) Dmsg0(100, "Leave set_find_options()\n"); } +void +set_find_changed_function(FF_PKT *ff, bool check_fct(JCR *jcr, FF_PKT *ff)) +{ + Dmsg0(1, "Enter set_find_changed_function()\n"); + ff->check_fct = check_fct; +} + /* * For VSS we need to know which windows drives * are used, because we create a snapshot of all used diff --git a/bacula/src/findlib/find.h b/bacula/src/findlib/find.h index 20f44aa33a..7a00819ee0 100644 --- a/bacula/src/findlib/find.h +++ b/bacula/src/findlib/find.h @@ -146,6 +146,7 @@ struct findFOPTS { int GZIP_level; /* GZIP level */ int strip_path; /* strip path count */ char VerifyOpts[MAX_FOPTS]; /* verify options */ + char AccurateOpts[MAX_FOPTS]; /* accurate mode options */ alist regex; /* regex string(s) */ alist regexdir; /* regex string(s) for directories */ alist regexfile; /* regex string(s) for files */ @@ -215,6 +216,7 @@ struct FF_PKT { findFILESET *fileset; int (*file_save)(JCR *, FF_PKT *, bool); /* User's callback */ int (*plugin_save)(JCR *, FF_PKT *, bool); /* User's callback */ + bool (*check_fct)(JCR *, FF_PKT *); /* optionnal user fct to check file changes */ /* Values set by accept_file while processing Options */ uint32_t flags; /* backup options */ diff --git a/bacula/src/findlib/find_one.c b/bacula/src/findlib/find_one.c index 1e508fa6b1..a1e8b9075c 100644 --- a/bacula/src/findlib/find_one.c +++ b/bacula/src/findlib/find_one.c @@ -257,6 +257,33 @@ bool has_file_changed(JCR *jcr, FF_PKT *ff_pkt) return false; } +/* + * In incremental/diffential or accurate backup, we + * say if the current file has changed. + */ +static bool check_changes(JCR *jcr, FF_PKT *ff_pkt) +{ + /* in special mode (like accurate backup), user can + * choose his comparison function. + */ + if (ff_pkt->check_fct) { + return ff_pkt->check_fct(jcr, ff_pkt); + } + + /* in normal modes (incr/diff), we use this default + * behaviour + */ + if (ff_pkt->incremental && + (ff_pkt->statp.st_mtime < ff_pkt->save_time && + ((ff_pkt->flags & FO_MTIMEONLY) || + ff_pkt->statp.st_ctime < ff_pkt->save_time))) + { + return false; + } + + return true; +} + /* * Find a single file. * handle_file is the callback for handling the file. @@ -327,22 +354,18 @@ find_one_file(JCR *jcr, FF_PKT *ff_pkt, } ff_pkt->volhas_attrlist = volume_has_attrlist(fname); } - /* * If this is an Incremental backup, see if file was modified * since our last "save_time", presumably the last Full save * or Incremental. */ - if (ff_pkt->incremental && !S_ISDIR(ff_pkt->statp.st_mode)) { - Dmsg1(300, "Non-directory incremental: %s\n", ff_pkt->fname); - /* Not a directory */ - if (ff_pkt->statp.st_mtime < ff_pkt->save_time - && ((ff_pkt->flags & FO_MTIMEONLY) || - ff_pkt->statp.st_ctime < ff_pkt->save_time)) { - /* Incremental option, file not changed */ - ff_pkt->type = FT_NOCHG; - return handle_file(jcr, ff_pkt, top_level); - } + if ( ff_pkt->incremental + && !S_ISDIR(ff_pkt->statp.st_mode) + && !check_changes(jcr, ff_pkt)) + { + Dmsg1(500, "Non-directory incremental: %s\n", ff_pkt->fname); + ff_pkt->type = FT_NOCHG; + return handle_file(jcr, ff_pkt, top_level); } #ifdef HAVE_DARWIN_OS @@ -502,10 +525,7 @@ find_one_file(JCR *jcr, FF_PKT *ff_pkt, link[len] = 0; ff_pkt->link = link; - if (ff_pkt->incremental && - (ff_pkt->statp.st_mtime < ff_pkt->save_time && - ((ff_pkt->flags & FO_MTIMEONLY) || - ff_pkt->statp.st_ctime < ff_pkt->save_time))) { + if (ff_pkt->incremental && !check_changes(jcr, ff_pkt)) { /* Incremental option, directory entry not changed */ ff_pkt->type = FT_DIRNOCHG; } else { diff --git a/bacula/src/findlib/protos.h b/bacula/src/findlib/protos.h index c50a2db5d4..5c6fd4599a 100644 --- a/bacula/src/findlib/protos.h +++ b/bacula/src/findlib/protos.h @@ -45,6 +45,7 @@ int create_file (JCR *jcr, ATTR *attr, BFILE *ofd, int replace); /* From find.c */ FF_PKT *init_find_files(); void set_find_options(FF_PKT *ff, int incremental, time_t mtime); +void set_find_changed_function(FF_PKT *ff, bool check_fct(JCR *jcr, FF_PKT *ff)); int find_files(JCR *jcr, FF_PKT *ff, int file_sub(JCR *, FF_PKT *ff_pkt, bool), int plugin_sub(JCR *, FF_PKT *ff_pkt, bool)); int match_files(JCR *jcr, FF_PKT *ff, int sub(JCR *, FF_PKT *ff_pkt, bool)); diff --git a/bacula/src/jcr.h b/bacula/src/jcr.h index 60c9dcba1d..7571b76c75 100644 --- a/bacula/src/jcr.h +++ b/bacula/src/jcr.h @@ -119,6 +119,7 @@ enum { /* Forward referenced structures */ class JCR; +class htable; struct FF_PKT; struct B_DB; struct ATTR_DBR; @@ -319,6 +320,7 @@ public: CRYPTO_CTX crypto; /* Crypto ctx */ DIRRES* director; /* Director resource */ bool VSS; /* VSS used by FD */ + htable *file_list; /* Previous file list (accurate mode) */ #endif /* FILE_DAEMON */ diff --git a/bacula/src/lib/Makefile.in b/bacula/src/lib/Makefile.in index dc819f8276..fa07fdd52f 100644 --- a/bacula/src/lib/Makefile.in +++ b/bacula/src/lib/Makefile.in @@ -29,7 +29,7 @@ LIBSRCS = attr.c base64.c berrno.c bsys.c bget_msg.c \ res.c rwlock.c scan.c serial.c sha1.c \ signal.c smartall.c rblist.c tls.c tree.c \ util.c var.c watchdog.c workq.c btimers.c \ - address_conf.c pythonlib.c breg.c + address_conf.c pythonlib.c breg.c htable.c LIBOBJS = attr.o base64.o berrno.o bsys.o bget_msg.o \ @@ -42,7 +42,7 @@ LIBOBJS = attr.o base64.o berrno.o bsys.o bget_msg.o \ res.o rwlock.o scan.o serial.o sha1.o \ signal.o smartall.o rblist.o tls.o tree.o \ util.o var.o watchdog.o workq.o btimers.o \ - address_conf.o pythonlib.o breg.o + address_conf.o pythonlib.o breg.o htable.o EXTRAOBJS = @OBJLIST@ diff --git a/bacula/src/lib/attr.c b/bacula/src/lib/attr.c index 7057c2f349..5e5210e4af 100644 --- a/bacula/src/lib/attr.c +++ b/bacula/src/lib/attr.c @@ -242,6 +242,14 @@ void print_ls_output(JCR *jcr, ATTR *attr) char *p, *f; guid_list *guid; + if (attr->type == FT_DELETED) { /* TODO: change this to get last seen values */ + bsnprintf(buf, sizeof(buf), + "---------- - - - - ---------- -------- %s\n", attr->ofname); + Dmsg1(20, "%s", buf); + Jmsg(jcr, M_RESTORED, 1, "%s", buf); + return; + } + if (!jcr->id_list) { jcr->id_list = new_guid_list(); } diff --git a/bacula/src/stored/bextract.c b/bacula/src/stored/bextract.c index 53eb01b8f1..7679da70af 100644 --- a/bacula/src/stored/bextract.c +++ b/bacula/src/stored/bextract.c @@ -343,6 +343,12 @@ static bool record_cb(DCR *dcr, DEV_RECORD *rec) build_attr_output_fnames(jcr, attr); + if (attr->type == FT_DELETED) { /* TODO: choose the right fname/ofname */ + Jmsg(jcr, M_INFO, 0, _("%s was deleted.\n"), attr->fname); + extract = false; + return true; + } + extract = false; stat = create_file(jcr, attr, &bfd, REPLACE_ALWAYS); switch (stat) { diff --git a/bacula/src/stored/bscan.c b/bacula/src/stored/bscan.c index c65348f6c8..25b7d702d2 100644 --- a/bacula/src/stored/bscan.c +++ b/bacula/src/stored/bscan.c @@ -845,7 +845,11 @@ static int create_file_attributes_record(B_DB *db, JCR *mjcr, ar.ClientId = mjcr->ClientId; ar.JobId = mjcr->JobId; ar.Stream = rec->Stream; - ar.FileIndex = rec->FileIndex; + if (type == FT_DELETED) { + ar.FileIndex = 0; + } else { + ar.FileIndex = rec->FileIndex; + } ar.attr = ap; if (dcr->VolFirstIndex == 0) { dcr->VolFirstIndex = rec->FileIndex;