#define FT_REPARSE 21 /* Win NTFS reparse point */
#define FT_PLUGIN 22 /* Plugin generated filename */
#define FT_DELETED 23 /* Deleted file entry */
+#define FT_BASE 24 /* Duplicate base file entry */
/* Definitions for upper part of type word (see above). */
#define AR_DATA_STREAM (1<<16) /* Data stream id present */
char *attr; /* attributes statp */
uint32_t FileIndex;
uint32_t Stream;
+ uint32_t FileType;
JobId_t JobId;
DBId_t ClientId;
DBId_t PathId;
int db_get_counter_record(JCR *jcr, B_DB *mdb, COUNTER_DBR *cr);
bool db_get_query_dbids(JCR *jcr, B_DB *mdb, POOL_MEM &query, dbid_list &ids);
bool db_get_file_list(JCR *jcr, B_DB *mdb, char *jobids, DB_RESULT_HANDLER *result_handler, void *ctx);
+bool db_get_base_jobid(JCR *jcr, B_DB *mdb, JOB_DBR *jr, JobId_t *jobid);
bool db_accurate_get_jobids(JCR *jcr, B_DB *mdb, JOB_DBR *jr, POOLMEM *jobids);
int db_get_int_handler(void *ctx, int num_fields, char **row);
int db_list_sql_query(JCR *jcr, B_DB *mdb, const char *query, DB_LIST_HANDLER *sendit, void *ctx, int verbose, e_list_type type);
void db_list_client_records(JCR *jcr, B_DB *mdb, DB_LIST_HANDLER *sendit, void *ctx, e_list_type type);
void db_list_copies_records(JCR *jcr, B_DB *mdb, uint32_t limit, char *jobids, DB_LIST_HANDLER *sendit, void *ctx, e_list_type type);
+void
+db_list_base_files_for_job(JCR *jcr, B_DB *mdb, JobId_t jobid, DB_LIST_HANDLER *sendit, void *ctx);
+
/* sql_update.c */
bool db_update_job_start_record(JCR *jcr, B_DB *db, JOB_DBR *jr);
void db_make_inchanger_unique(JCR *jcr, B_DB *mdb, MEDIA_DBR *mr);
int db_update_stats(JCR *jcr, B_DB *mdb, utime_t age);
+
+bool db_get_used_base_jobids(JCR *jcr, B_DB *mdb, POOLMEM *jobids, POOLMEM *result);
+bool db_create_attributes_record(JCR *jcr, B_DB *mdb, ATTR_DBR *ar);
+bool db_create_base_file_attributes_record(JCR *jcr, B_DB *mdb, ATTR_DBR *ar);
+bool db_commit_base_file_attributes_record(JCR *jcr, B_DB *mdb);
+bool db_create_base_file_list(JCR *jcr, B_DB *mdb, char *jobids);
+bool db_get_base_file_list(JCR *jcr, B_DB *mdb, DB_RESULT_HANDLER *result_handler,
+ void *ctx);
+
+
#endif /* __SQL_PROTOS_H */
if (jcr && jcr->cached_attribute) {
Dmsg0(400, "Flush last cached attribute.\n");
- if (!db_create_file_attributes_record(jcr, mdb, jcr->ar)) {
+ if (!db_create_attributes_record(jcr, mdb, jcr->ar)) {
Jmsg1(jcr, M_FATAL, 0, _("Attribute create error. %s"), db_strerror(jcr->db));
}
jcr->cached_attribute = false;
"FROM Client,Job WHERE Client.ClientId=Job.ClientId AND JobStatus IN ('T','W') "
"AND Type='B' ORDER BY StartTime DESC LIMIT 20";
+const char *uar_print_jobs =
+ "SELECT DISTINCT JobId,Level,JobFiles,JobBytes,StartTime,VolumeName"
+ " FROM Job JOIN JobMedia USING (JobId) JOIN Media USING (MediaId) "
+ " WHERE JobId IN (%s) "
+ " ORDER BY StartTime ASC";
+
/*
* Find all files for a particular JobId and insert them into
* the tree during a restore.
extern const char CATS_IMP_EXP *cleanup_created_job;
extern const char CATS_IMP_EXP *cleanup_running_job;
extern const char CATS_IMP_EXP *uar_list_jobs;
+extern const char CATS_IMP_EXP *uar_print_jobs;
extern const char CATS_IMP_EXP *uar_count_files;
extern const char CATS_IMP_EXP *uar_sel_files;
extern const char CATS_IMP_EXP *uar_del_temp;
#include "bacula.h"
#include "cats.h"
-static const int dbglevel = 500;
+static const int dbglevel = 10;
#if HAVE_SQLITE3 || HAVE_MYSQL || HAVE_SQLITE || HAVE_POSTGRESQL || HAVE_DBI
*/
bool db_create_file_attributes_record(JCR *jcr, B_DB *mdb, ATTR_DBR *ar)
{
+ ASSERT(ar->FileType != FT_BASE);
+
Dmsg1(dbglevel, "Fname=%s\n", ar->fname);
Dmsg0(dbglevel, "put_file_into_catalog\n");
#endif /* ! HAVE_BATCH_FILE_INSERT */
+
+/* List of SQL commands to create temp table and indicies */
+const char *create_temp_basefile[4] = {
+ /* MySQL */
+ "CREATE TEMPORARY TABLE basefile%lld ("
+ "Path BLOB NOT NULL,"
+ "Name BLOB NOT NULL)",
+
+ /* Postgresql */
+ "CREATE TEMPORARY TABLE basefile%lld ("
+// "CREATE TABLE basefile%lld ("
+ "Path TEXT,"
+ "Name TEXT)",
+
+ /* SQLite */
+ "CREATE TEMPORARY TABLE basefile%lld ("
+ "Path TEXT,"
+ "Name TEXT)",
+
+ /* SQLite3 */
+ "CREATE TEMPORARY TABLE basefile%lld ("
+ "Path TEXT,"
+ "Name TEXT)"
+};
+
+/*
+ * Create file attributes record, or base file attributes record
+ */
+bool db_create_attributes_record(JCR *jcr, B_DB *mdb, ATTR_DBR *ar)
+{
+ bool ret;
+ if (ar->FileType != FT_BASE) {
+ ret = db_create_file_attributes_record(jcr, mdb, ar);
+
+ } else if (jcr->HasBase) {
+ ret = db_create_base_file_attributes_record(jcr, mdb, ar);
+
+ } else {
+ Jmsg0(jcr, M_FATAL, 0, _("Can't Copy/Migrate job using BaseJob"));
+ ret = true; /* in copy/migration what do we do ? */
+ }
+
+ return ret;
+}
+
+/*
+ * Create Base File record in B_DB
+ *
+ */
+bool db_create_base_file_attributes_record(JCR *jcr, B_DB *mdb, ATTR_DBR *ar)
+{
+ bool ret;
+ Dmsg1(dbglevel, "create_base_file Fname=%s\n", ar->fname);
+ Dmsg0(dbglevel, "put_base_file_into_catalog\n");
+
+ /*
+ * Make sure we have an acceptable attributes record.
+ */
+ if (!(ar->Stream == STREAM_UNIX_ATTRIBUTES ||
+ ar->Stream == STREAM_UNIX_ATTRIBUTES_EX)) {
+ Mmsg1(&mdb->errmsg, _("Attempt to put non-attributes into catalog. Stream=%d\n"),
+ ar->Stream);
+ Jmsg(jcr, M_FATAL, 0, "%s", mdb->errmsg);
+ return false;
+ }
+
+ db_lock(mdb);
+ split_path_and_file(jcr, mdb, ar->fname);
+
+ mdb->esc_name = check_pool_memory_size(mdb->esc_name, mdb->fnl*2+1);
+ db_escape_string(jcr, mdb, mdb->esc_name, mdb->fname, mdb->fnl);
+
+ mdb->esc_path = check_pool_memory_size(mdb->esc_path, mdb->pnl*2+1);
+ db_escape_string(jcr, mdb, mdb->esc_path, mdb->path, mdb->pnl);
+
+ Mmsg(mdb->cmd, "INSERT INTO basefile%lld (Path, Name) VALUES ('%s','%s')",
+ (uint64_t)jcr->JobId, mdb->esc_path, mdb->esc_name);
+
+ ret = INSERT_DB(jcr, mdb, mdb->cmd);
+ db_unlock(mdb);
+
+ return ret;
+}
+
+/*
+ * Cleanup the base file temporary tables
+ */
+static void db_cleanup_base_file(JCR *jcr, B_DB *mdb)
+{
+ POOL_MEM buf(PM_MESSAGE);
+ Mmsg(buf, "DROP TABLE new_basefile%lld", (uint64_t) jcr->JobId);
+ db_sql_query(mdb, buf.c_str(), NULL, NULL);
+
+ Mmsg(buf, "DROP TABLE basefile%lld", (uint64_t) jcr->JobId);
+ db_sql_query(mdb, buf.c_str(), NULL, NULL);
+}
+
+/*
+ * Put all base file seen in the backup to the BaseFile table
+ * and cleanup temporary tables
+ */
+bool db_commit_base_file_attributes_record(JCR *jcr, B_DB *mdb)
+{
+ bool ret;
+ char ed1[50];
+
+ db_lock(mdb);
+
+ Mmsg(mdb->cmd,
+ "INSERT INTO BaseFiles (BaseJobId, JobId, FileId, FileIndex) ( "
+ "SELECT B.JobId AS BaseJobId, %s AS JobId, "
+ "B.FileId, B.FileIndex "
+ "FROM basefile%s AS A, new_basefile%s AS B "
+ "WHERE A.Path = B.Path "
+ "AND A.Name = B.Name "
+ "ORDER BY B.FileId)",
+ edit_uint64(jcr->JobId, ed1), ed1, ed1);
+ ret = QUERY_DB(jcr, mdb, mdb->cmd);
+ jcr->nb_base_files_used = sql_affected_rows(mdb);
+ db_cleanup_base_file(jcr, mdb);
+
+ db_unlock(mdb);
+ return ret;
+}
+
+/*
+ * Find the last "accurate" backup state with Base jobs
+ * 1) Get all files with jobid in list (F subquery)
+ * 2) Take only the last version of each file (Temp subquery) => accurate list is ok
+ * 3) Put the result in a temporary table for the end of job
+ *
+ */
+bool db_create_base_file_list(JCR *jcr, B_DB *mdb, char *jobids)
+{
+ bool ret=false;
+
+ db_lock(mdb);
+
+ if (!*jobids) {
+ Mmsg(mdb->errmsg, _("ERR=JobIds are empty\n"));
+ goto bail_out;
+ }
+
+ Mmsg(mdb->cmd, create_temp_basefile[db_type], (uint64_t) jcr->JobId);
+ if (!QUERY_DB(jcr, mdb, mdb->cmd)) {
+ goto bail_out;
+ }
+
+ Mmsg(mdb->cmd,
+"CREATE TEMPORARY TABLE new_basefile%lld AS ( "
+//"CREATE TABLE new_basefile%lld AS ( "
+ "SELECT Path.Path AS Path, Filename.Name AS Name, File.FileIndex AS FileIndex,"
+ "File.JobId AS JobId, File.LStat AS LStat, File.FileId AS FileId "
+ "FROM ( "
+ "SELECT max(FileId) as FileId, PathId, FilenameId "
+ "FROM (SELECT FileId, PathId, FilenameId FROM File WHERE JobId IN (%s)) AS F "
+ "GROUP BY PathId, FilenameId "
+ ") AS Temp "
+ "JOIN Filename ON (Filename.FilenameId = Temp.FilenameId) "
+ "JOIN Path ON (Path.PathId = Temp.PathId) "
+ "JOIN File ON (File.FileId = Temp.FileId) "
+ "WHERE File.FileIndex > 0)",
+ (uint64_t)jcr->JobId, jobids);
+ ret = QUERY_DB(jcr, mdb, mdb->cmd);
+bail_out:
+ db_unlock(mdb);
+ return ret;
+}
+
#endif /* HAVE_SQLITE3 || HAVE_MYSQL || HAVE_SQLITE || HAVE_POSTGRESQL || HAVE_DBI */
Mmsg(mdb->cmd, "SELECT VolSessionId,VolSessionTime,"
"PoolId,StartTime,EndTime,JobFiles,JobBytes,JobTDate,Job,JobStatus,"
"Type,Level,ClientId,Name,PriorJobId,RealEndTime,JobId,FileSetId,"
-"SchedTime,RealEndTime,ReadBytes "
+"SchedTime,RealEndTime,ReadBytes,HasBase "
"FROM Job WHERE Job='%s'", jr->Job);
} else {
Mmsg(mdb->cmd, "SELECT VolSessionId,VolSessionTime,"
"PoolId,StartTime,EndTime,JobFiles,JobBytes,JobTDate,Job,JobStatus,"
"Type,Level,ClientId,Name,PriorJobId,RealEndTime,JobId,FileSetId,"
-"SchedTime,RealEndTime,ReadBytes "
+"SchedTime,RealEndTime,ReadBytes,HasBase "
"FROM Job WHERE JobId=%s",
edit_int64(jr->JobId, ed1));
}
jr->SchedTime = str_to_utime(jr->cSchedTime);
jr->EndTime = str_to_utime(jr->cEndTime);
jr->RealEndTime = str_to_utime(jr->cRealEndTime);
+ jr->HasBase = str_to_int64(row[21]);
sql_free_result(mdb);
db_unlock(mdb);
/*
* Find the last "accurate" backup state (that can take deleted files in account)
- * 1) Get all files with jobid in list (F subquery)
+ * 1) Get all files with jobid in list (F subquery)
+ * Get all files in BaseFiles with jobid in list
* 2) Take only the last version of each file (Temp subquery) => accurate list is ok
* 3) Join the result to file table to get fileindex, jobid and lstat information
*
#define new_db_get_file_list
#ifdef new_db_get_file_list
- /* This is broken, at least if called from ua_restore.c */
Mmsg(buf,
"SELECT Path.Path, Filename.Name, File.FileIndex, File.JobId, File.LStat "
"FROM ( "
"SELECT max(FileId) as FileId, PathId, FilenameId "
- "FROM (SELECT FileId, PathId, FilenameId FROM File WHERE JobId IN (%s)) AS F "
+ "FROM (SELECT FileId, PathId, FilenameId FROM File WHERE JobId IN (%s) "
+ "UNION ALL " /* we already sort after */
+ "SELECT File.FileId, PathId, FilenameId "
+ "FROM BaseFiles JOIN File USING (FileId) "
+ "WHERE BaseFiles.JobId IN (%s) "
+ ") AS F "
"GROUP BY PathId, FilenameId "
") AS Temp "
"JOIN Filename ON (Filename.FilenameId = Temp.FilenameId) "
"JOIN Path ON (Path.PathId = Temp.PathId) "
"JOIN File ON (File.FileId = Temp.FileId) "
-"WHERE File.FileIndex > 0 ORDER BY JobId, FileIndex ASC", /* Return sorted by JobId, */
- /* FileIndex for restore code */
- jobids);
+"WHERE File.FileIndex > 0 ORDER BY JobId, FileIndex ASC",/* Return sorted by JobId, */
+ /* FileIndex for restore code */
+ jobids, jobids);
#else
/*
* I am not sure that this works the same as the code in ua_restore.c
return db_sql_query(mdb, buf.c_str(), result_handler, ctx);
}
+/*
+ * This procedure gets the base jobid list used by jobids,
+ * You can specify jobids == result to concat base jobids to current jobids
+ */
+bool db_get_used_base_jobids(JCR *jcr, B_DB *mdb, POOLMEM *jobids, POOLMEM *result)
+{
+ POOL_MEM buf;
+ Mmsg(buf,
+ "SELECT DISTINCT BaseJobId "
+ " FROM Job JOIN BaseFiles USING (JobId) "
+ " WHERE Job.HasBase = 1 "
+ " AND JobId IN (%s) ", jobids);
+ return db_sql_query(mdb, buf.c_str(), db_get_int_handler, result);
+}
+
/* The decision do change an incr/diff was done before
* Full : do nothing
* Differential : get the last full id
/* build a jobid list ie: 1,2,3,4 */
Mmsg(query, "SELECT JobId FROM btemp3%s ORDER by JobTDate", jobid);
db_sql_query(mdb, query.c_str(), db_get_int_handler, jobids);
- Dmsg1(1, "db_accurate_get_jobids=%s\n", jobids);
+ Dmsg1(10, "db_accurate_get_jobids=%s\n", jobids);
ret = true;
bail_out:
return ret;
}
+bool db_get_base_file_list(JCR *jcr, B_DB *mdb,
+ DB_RESULT_HANDLER *result_handler, void *ctx)
+{
+ POOL_MEM buf(PM_MESSAGE);
+
+ Mmsg(buf,
+ "SELECT Path, Name, FileIndex, JobId, LStat "
+ "FROM new_basefile%lld ORDER BY JobId, FileIndex ASC",
+ (uint64_t) jcr->JobId);
+
+ return db_sql_query(mdb, buf.c_str(), result_handler, ctx);
+}
+bool db_get_base_jobid(JCR *jcr, B_DB *mdb, JOB_DBR *jr, JobId_t *jobid)
+{
+ char date[MAX_TIME_LENGTH];
+ int64_t id = *jobid = 0;
+ POOL_MEM query(PM_FNAME);
+
+// char clientid[50], filesetid[50];
+
+ utime_t StartTime = (jr->StartTime)?jr->StartTime:time(NULL);
+ bstrutime(date, sizeof(date), StartTime + 1);
+
+ /* we can take also client name, fileset, etc... */
+
+ Mmsg(query,
+ "SELECT JobId, Job, StartTime, EndTime, JobTDate, PurgedFiles "
+ "FROM Job "
+// "JOIN FileSet USING (FileSetId) JOIN Client USING (ClientId) "
+ "WHERE Job.Name = '%s' "
+ "AND Level='B' AND JobStatus IN ('T','W') AND Type='B' "
+// "AND FileSet.FileSet= '%s' "
+// "AND Client.Name = '%s' "
+ "AND StartTime<'%s' "
+ "ORDER BY Job.JobTDate DESC LIMIT 1",
+ jr->Name,
+// edit_uint64(jr->ClientId, clientid),
+// edit_uint64(jr->FileSetId, filesetid));
+ date);
+
+ Dmsg1(10, "db_get_base_jobid q=%s\n", query.c_str());
+ if (!db_sql_query(mdb, query.c_str(), db_int64_handler, &id)) {
+ goto bail_out;
+ }
+ *jobid = (JobId_t) id;
+
+ Dmsg1(10, "db_get_base_jobid=%lld\n", id);
+ return true;
+
+bail_out:
+ return false;
+}
+
/*
* Use to build a string of int list from a query. "10,20,30"
*/
* Stupid MySQL is NON-STANDARD !
*/
if (db_type == SQL_TYPE_MYSQL) {
- Mmsg(mdb->cmd, "SELECT CONCAT(Path.Path,Filename.Name) AS Filename FROM File,"
- "Filename,Path WHERE File.JobId=%s AND Filename.FilenameId=File.FilenameId "
- "AND Path.PathId=File.PathId",
- edit_int64(jobid, ed1));
+ Mmsg(mdb->cmd, "SELECT CONCAT(Path.Path,Filename.Name) AS Filename "
+ "FROM (SELECT PathId, FilenameId FROM File WHERE JobId=%s "
+ "UNION ALL "
+ "SELECT PathId, FilenameId "
+ "FROM BaseFiles JOIN File "
+ "ON (BaseFiles.FileId = File.FileId) "
+ "WHERE BaseFiles.JobId = %s"
+ ") AS F, Filename,Path "
+ "WHERE Filename.FilenameId=F.FilenameId "
+ "AND Path.PathId=F.PathId",
+ edit_int64(jobid, ed1), ed1);
} else {
- Mmsg(mdb->cmd, "SELECT Path.Path||Filename.Name AS Filename FROM File,"
- "Filename,Path WHERE File.JobId=%s AND Filename.FilenameId=File.FilenameId "
- "AND Path.PathId=File.PathId",
+ Mmsg(mdb->cmd, "SELECT Path.Path||Filename.Name AS Filename "
+ "FROM (SELECT PathId, FilenameId FROM File WHERE JobId=%s "
+ "UNION ALL "
+ "SELECT PathId, FilenameId "
+ "FROM BaseFiles JOIN File "
+ "ON (BaseFiles.FileId = File.FileId) "
+ "WHERE BaseFiles.JobId = %s"
+ ") AS F, Filename,Path "
+ "WHERE Filename.FilenameId=F.FilenameId "
+ "AND Path.PathId=F.PathId",
+ edit_int64(jobid, ed1), ed1);
+ }
+
+ if (!QUERY_DB(jcr, mdb, mdb->cmd)) {
+ db_unlock(mdb);
+ return;
+ }
+
+ list_result(jcr, mdb, sendit, ctx, HORZ_LIST);
+
+ sql_free_result(mdb);
+ db_unlock(mdb);
+}
+
+void
+db_list_base_files_for_job(JCR *jcr, B_DB *mdb, JobId_t jobid, DB_LIST_HANDLER *sendit, void *ctx)
+{
+ char ed1[50];
+ db_lock(mdb);
+
+ /*
+ * Stupid MySQL is NON-STANDARD !
+ */
+ if (db_type == SQL_TYPE_MYSQL) {
+ Mmsg(mdb->cmd, "SELECT CONCAT(Path.Path,Filename.Name) AS Filename "
+ "FROM BaseFiles, File, Filename, Path "
+ "WHERE BaseFiles.JobId=%s AND BaseFiles.BaseJobId = File.JobId "
+ "AND BaseFiles.FileId = File.FileId "
+ "AND Filename.FilenameId=File.FilenameId "
+ "AND Path.PathId=File.PathId",
edit_int64(jobid, ed1));
+ } else {
+ Mmsg(mdb->cmd, "SELECT Path.Path||Filename.Name AS Filename "
+ "FROM BaseFiles, File, Filename, Path "
+ "WHERE BaseFiles.JobId=%s AND BaseFiles.BaseJobId = File.JobId "
+ "AND BaseFiles.FileId = File.FileId "
+ "AND Filename.FilenameId=File.FilenameId "
+ "AND Path.PathId=File.PathId",
+ edit_int64(jobid, ed1));
}
if (!QUERY_DB(jcr, mdb, mdb->cmd)) {
"UPDATE Job SET JobStatus='%c',EndTime='%s',"
"ClientId=%u,JobBytes=%s,ReadBytes=%s,JobFiles=%u,JobErrors=%u,VolSessionId=%u,"
"VolSessionTime=%u,PoolId=%u,FileSetId=%u,JobTDate=%s,"
-"RealEndTime='%s',PriorJobId=%s WHERE JobId=%s",
+"RealEndTime='%s',PriorJobId=%s,HasBase=%u WHERE JobId=%s",
(char)(jr->JobStatus), dt, jr->ClientId, edit_uint64(jr->JobBytes, ed1),
edit_uint64(jr->ReadBytes, ed4),
jr->JobFiles, jr->JobErrors, jr->VolSessionId, jr->VolSessionTime,
jr->PoolId, jr->FileSetId, edit_uint64(JobTDate, ed2),
rdt,
PriorJobId,
+ jr->HasBase,
edit_int64(jr->JobId, ed3));
stat = UPDATE_DB(jcr, mdb, mdb->cmd);
return true;
}
+/* Take all base jobs from job resource and find the
+ * last L_BASE jobid.
+ */
+static bool get_base_jobids(JCR *jcr, POOLMEM *jobids)
+{
+ JOB_DBR jr;
+ JOB *job;
+ JobId_t id;
+ char str_jobid[50];
+
+ if (!jcr->job->base) {
+ return false; /* no base job, stop accurate */
+ }
+
+ memset(&jr, 0, sizeof(JOB_DBR));
+ jr.StartTime = jcr->jr.StartTime;
+
+ foreach_alist(job, jcr->job->base) {
+ bstrncpy(jr.Name, job->name(), sizeof(jr.Name));
+ db_get_base_jobid(jcr, jcr->db, &jr, &id);
+
+ if (id) {
+ if (jobids[0]) {
+ pm_strcat(jobids, ",");
+ }
+ pm_strcat(jobids, edit_uint64(id, str_jobid));
+ }
+ }
+
+ return *jobids != '\0';
+}
+
/*
* Foreach files in currrent list, send "/path/fname\0LStat" to FD
*/
bool send_accurate_current_files(JCR *jcr)
{
POOL_MEM buf;
+ bool ret=true;
- if (!jcr->accurate || job_canceled(jcr) || jcr->get_JobLevel()==L_FULL) {
+ if (!jcr->accurate || job_canceled(jcr)) {
+ return true;
+ }
+ /* In base level, no previous job is used */
+ if (jcr->get_JobLevel() == L_BASE) {
return true;
}
+
+ POOLMEM *nb = get_pool_memory(PM_FNAME);
POOLMEM *jobids = get_pool_memory(PM_FNAME);
+ nb[0] = jobids[0] = '\0';
- db_accurate_get_jobids(jcr, jcr->db, &jcr->jr, jobids);
+ if (jcr->get_JobLevel() == L_FULL) {
+ /* On Full mode, if no previous base job, no accurate things */
+ if (!get_base_jobids(jcr, jobids)) {
+ goto bail_out;
+ }
+ jcr->HasBase = true;
+ Jmsg(jcr, M_INFO, 0, _("Using BaseJobId(s): %s\n"), jobids);
- if (*jobids == 0) {
- free_pool_memory(jobids);
- Jmsg(jcr, M_FATAL, 0, _("Cannot find previous jobids.\n"));
- return false;
+ } else {
+ /* For Incr/Diff level, we search for older jobs */
+ db_accurate_get_jobids(jcr, jcr->db, &jcr->jr, jobids);
+
+ /* We are in Incr/Diff, but no Full to build the accurate list... */
+ if (*jobids == 0) {
+ ret=false;
+ Jmsg(jcr, M_FATAL, 0, _("Cannot find previous jobids.\n"));
+ goto bail_out;
+ }
}
+
if (jcr->JobId) { /* display the message only for real jobs */
Jmsg(jcr, M_INFO, 0, _("Sending Accurate information.\n"));
}
+
/* to be able to allocate the right size for htable */
- POOLMEM *nb = get_pool_memory(PM_FNAME);
- *nb = 0; /* clear buffer */
Mmsg(buf, "SELECT sum(JobFiles) FROM Job WHERE JobId IN (%s)",jobids);
db_sql_query(jcr->db, buf.c_str(), db_get_int_handler, nb);
- Dmsg2(200, "jobids=%s nb=%s\n", jobids, nb);
+ Dmsg2(0, "jobids=%s nb=%s\n", jobids, nb);
jcr->file_bsock->fsend("accurate files=%s\n", nb);
if (!db_open_batch_connexion(jcr, jcr->db)) {
+ ret = false;
Jmsg0(jcr, M_FATAL, 0, "Can't get dedicate sql connexion");
- return false;
+ goto bail_out;
}
+
+ if (jcr->HasBase) {
+ jcr->nb_base_files = str_to_int64(nb);
+ db_create_base_file_list(jcr, jcr->db, jobids);
+ db_get_base_file_list(jcr, jcr->db,
+ accurate_list_handler, (void *)jcr);
- db_get_file_list(jcr, jcr->db_batch, jobids, accurate_list_handler, (void *)jcr);
+ } else {
+ db_get_file_list(jcr, jcr->db_batch, jobids,
+ accurate_list_handler, (void *)jcr);
+ }
/* TODO: close the batch connexion ? (can be used very soon) */
+ jcr->file_bsock->signal(BNET_EOD);
+
+bail_out:
free_pool_memory(jobids);
free_pool_memory(nb);
- jcr->file_bsock->signal(BNET_EOD);
-
- return true;
+ return ret;
}
/*
/* Pickup Job termination data */
stat = wait_for_job_termination(jcr);
db_write_batch_file_records(jcr); /* used by bulk batch file insert */
+
+ if (jcr->HasBase &&
+ !db_commit_base_file_attributes_record(jcr, jcr->db))
+ {
+ Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db));
+ }
+
if (stat == JS_Terminated) {
backup_cleanup(jcr, stat);
return true;
jobstatus_to_ascii(jcr->FDJobStatus, fd_term_msg, sizeof(fd_term_msg));
jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg));
+ if (jcr->HasBase) {
+ Dmsg3(0, "Base files/Used files %lld/%lld=%.2f%%\n", jcr->nb_base_files,
+ jcr->nb_base_files_used,
+ jcr->nb_base_files_used*100.0/jcr->nb_base_files);
+ }
// bmicrosleep(15, 0); /* for debugging SIGHUP */
Jmsg(jcr, msg_type, 0, _("%s %s %s (%s): %s\n"
uint32_t FileIndex;
uint32_t data_len;
char *p;
- int filetype;
int len;
char *fname, *attr;
ATTR_DBR *ar = NULL;
if (Stream == STREAM_UNIX_ATTRIBUTES || Stream == STREAM_UNIX_ATTRIBUTES_EX) {
if (jcr->cached_attribute) {
Dmsg2(400, "Cached attr. Stream=%d fname=%s\n", ar->Stream, ar->fname);
- if (!db_create_file_attributes_record(jcr, jcr->db, ar)) {
+ if (!db_create_attributes_record(jcr, jcr->db, ar)) {
Jmsg1(jcr, M_FATAL, 0, _("Attribute create error. %s"), db_strerror(jcr->db));
}
}
p = jcr->attr - msg + p; /* point p into jcr->attr */
skip_nonspaces(&p); /* skip FileIndex */
skip_spaces(&p);
- filetype = str_to_int32(p); /* TODO: choose between unserialize and str_to_int32 */
+ ar->FileType = str_to_int32(p); /* TODO: choose between unserialize and str_to_int32 */
skip_nonspaces(&p); /* skip FileType */
skip_spaces(&p);
fname = p;
Dmsg1(400, "dird<stored: attr=%s\n", attr);
ar->attr = attr;
ar->fname = fname;
- if (filetype == FT_DELETED) {
+ if (ar->FileType == FT_DELETED) {
ar->FileIndex = 0; /* special value */
} else {
ar->FileIndex = FileIndex;
}
bin_to_base64(digestbuf, sizeof(digestbuf), fname, len, true);
- Dmsg3(400, "DigestLen=%d Digest=%s type=%d\n", strlen(digestbuf), digestbuf, Stream);
+ Dmsg3(400, "DigestLen=%d Digest=%s type=%d\n", strlen(digestbuf),
+ digestbuf, Stream);
if (jcr->cached_attribute) {
ar->Digest = digestbuf;
ar->DigestType = type;
- Dmsg2(400, "Cached attr with digest. Stream=%d fname=%s\n", ar->Stream, ar->fname);
- if (!db_create_file_attributes_record(jcr, jcr->db, ar)) {
- Jmsg1(jcr, M_FATAL, 0, _("Attribute create error. %s"), db_strerror(jcr->db));
+ Dmsg2(400, "Cached attr with digest. Stream=%d fname=%s\n",
+ ar->Stream, ar->fname);
+
+ /* Update BaseFile table */
+ if (!db_create_attributes_record(jcr, jcr->db, ar)) {
+ Jmsg1(jcr, M_FATAL, 0, _("attribute create error. %s"),
+ db_strerror(jcr->db));
}
jcr->cached_attribute = false;
} else {
if (mode == UPDATE_AND_FIX) {
db_sql_query(db, cleanup_created_job, NULL, NULL);
db_sql_query(db, cleanup_running_job, NULL, NULL);
+ db_sql_query(db, "CREATE INDEX basefiles_jobid_idx ON BaseFiles ( JobId )" , NULL, NULL);
}
db_close_database(NULL, db);
{"cancelqueuedduplicates", store_bool, ITEM(res_job.CancelQueuedDuplicates), 0, ITEM_DEFAULT, false},
{"cancelrunningduplicates", store_bool, ITEM(res_job.CancelRunningDuplicates), 0, ITEM_DEFAULT, false},
{"pluginoptions", store_str, ITEM(res_job.PluginOptions), 0, 0, 0},
+ {"base", store_alist_res, ITEM(res_job.base), R_JOB, 0, 0},
{NULL, NULL, {0}, 0, 0, 0}
};
dump_resource(-R_STORAGE, (RES *)store, sendit, sock);
}
}
+ if (res->res_job.base) {
+ JOB *job;
+ foreach_alist(job, res->res_job.base) {
+ sendit(sock, _(" --> Base %s\n"), job->name());
+ }
+ }
if (res->res_job.RunScripts) {
RUNSCRIPT *script;
foreach_alist(script, res->res_job.RunScripts) {
if (res->res_job.storage) {
delete res->res_job.storage;
}
+ if (res->res_job.base) {
+ delete res->res_job.base;
+ }
if (res->res_job.RunScripts) {
free_runscripts(res->res_job.RunScripts);
delete res->res_job.RunScripts;
res->res_job.client = res_all.res_job.client;
res->res_job.fileset = res_all.res_job.fileset;
res->res_job.storage = res_all.res_job.storage;
+ res->res_job.base = res_all.res_job.base;
res->res_job.pool = res_all.res_job.pool;
res->res_job.full_pool = res_all.res_job.full_pool;
res->res_job.inc_pool = res_all.res_job.inc_pool;
bool AllowHigherDuplicates; /* Permit Higher Level */
bool CancelQueuedDuplicates; /* Cancel queued jobs */
bool CancelRunningDuplicates; /* Cancel Running jobs */
-
+ alist *base; /* Base jobs */
/* Methods */
char *name() const;
jcr->jr.VolSessionId = jcr->VolSessionId;
jcr->jr.VolSessionTime = jcr->VolSessionTime;
jcr->jr.JobErrors = jcr->JobErrors;
+ jcr->jr.HasBase = jcr->HasBase;
if (!db_update_job_end_record(jcr, jcr->db, &jcr->jr)) {
Jmsg(jcr, M_WARNING, 0, _("Error updating job record. %s"),
db_strerror(jcr->db));
char RestoreClientName[MAX_NAME_LENGTH]; /* restore client */
char last_jobid[20];
POOLMEM *JobIds; /* User entered string of JobIds */
+ POOLMEM *BaseJobIds; /* Base jobids */
STORE *store;
JOB *restore_job;
POOL *pool;
jr.JobId = 0;
db_list_job_records(ua->jcr, ua->db, &jr, prtit, ua, llist);
+ /* List Base files */
+ } else if (strcasecmp(ua->argk[i], NT_("basefiles")) == 0) {
+ /* TODO: cleanup this block */
+ for (j=i+1; j<ua->argc; j++) {
+ if (strcasecmp(ua->argk[j], NT_("ujobid")) == 0 && ua->argv[j]) {
+ bstrncpy(jr.Job, ua->argv[j], MAX_NAME_LENGTH);
+ jr.JobId = 0;
+ db_get_job_record(ua->jcr, ua->db, &jr);
+ jobid = jr.JobId;
+ } else if (strcasecmp(ua->argk[j], NT_("jobid")) == 0 && ua->argv[j]) {
+ jobid = str_to_int64(ua->argv[j]);
+ } else {
+ continue;
+ }
+ if (jobid > 0) {
+ db_list_base_files_for_job(ua->jcr, ua->db, jobid, prtit, ua);
+ }
+ }
+
/* List FILES */
} else if (strcasecmp(ua->argk[i], NT_("files")) == 0) {
db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
Dmsg1(050, "Delete File sql=%s\n", query.c_str());
+ Mmsg(query, "DELETE FROM BaseFiles WHERE JobId IN (%s)", jobs);
+ db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL);
+ Dmsg1(050, "Delete BaseFiles sql=%s\n", query.c_str());
+
/*
* Now mark Job as having files purged. This is necessary to
* avoid having too many Jobs to process in future prunings. If
static int get_date(UAContext *ua, char *date, int date_len);
static int restore_count_handler(void *ctx, int num_fields, char **row);
static bool insert_table_into_findex_list(UAContext *ua, RESTORE_CTX *rx, char *table);
+static void get_and_display_basejobs(UAContext *ua, RESTORE_CTX *rx);
/*
* Restore files
rx.path = get_pool_memory(PM_FNAME);
rx.fname = get_pool_memory(PM_FNAME);
rx.JobIds = get_pool_memory(PM_FNAME);
+ rx.BaseJobIds = get_pool_memory(PM_FNAME);
rx.query = get_pool_memory(PM_FNAME);
rx.bsr = new_bsr();
case 0: /* error */
goto bail_out;
case 1: /* selected by jobid */
+ get_and_display_basejobs(ua, &rx);
if (!build_directory_tree(ua, &rx)) {
ua->send_msg(_("Restore not done.\n"));
goto bail_out;
}
+/*
+ * Fill the rx->BaseJobIds and display the list
+ */
+static void get_and_display_basejobs(UAContext *ua, RESTORE_CTX *rx)
+{
+ rx->BaseJobIds[0] = '\0';
+
+ if (!db_get_used_base_jobids(ua->jcr, ua->db, rx->JobIds, rx->BaseJobIds)) {
+ ua->warning_msg("%s", db_strerror(ua->db));
+ }
+
+ if (*rx->BaseJobIds) {
+ POOL_MEM q;
+ Mmsg(q, uar_print_jobs, rx->BaseJobIds);
+ ua->send_msg(_("The restore will use the following job(s) as Base\n"));
+ db_list_sql_query(ua->jcr, ua->db, q.c_str(), prtit, ua, 1, HORZ_LIST);
+ }
+}
+
static void free_rx(RESTORE_CTX *rx)
{
free_bsr(rx->bsr);
rx->bsr = NULL;
- if (rx->JobIds) {
- free_pool_memory(rx->JobIds);
- rx->JobIds = NULL;
- }
- if (rx->fname) {
- free_pool_memory(rx->fname);
- rx->fname = NULL;
- }
- if (rx->path) {
- free_pool_memory(rx->path);
- rx->path = NULL;
- }
- if (rx->query) {
- free_pool_memory(rx->query);
- rx->query = NULL;
- }
+ free_and_null_pool_memory(rx->JobIds);
+ free_and_null_pool_memory(rx->BaseJobIds);
+ free_and_null_pool_memory(rx->fname);
+ free_and_null_pool_memory(rx->path);
+ free_and_null_pool_memory(rx->query);
free_name_list(&rx->name_list);
}
NULL
};
- *rx->JobIds = 0;
+ rx->JobIds[0] = 0;
for (i=1; i<ua->argc; i++) { /* loop through arguments */
bool found_kw = false;
ua->warning_msg(_("No Jobs selected.\n"));
return 0;
}
+
if (strchr(rx->JobIds,',')) {
ua->info_msg(_("You have selected the following JobIds: %s\n"), rx->JobIds);
} else {
if (!db_get_file_list(ua->jcr, ua->db, rx->JobIds, insert_tree_handler, (void *)&tree)) {
ua->error_msg("%s", db_strerror(ua->db));
}
+ if (*rx->BaseJobIds) {
+ pm_strcat(rx->JobIds, ",");
+ pm_strcat(rx->JobIds, rx->BaseJobIds);
+ }
#else
for (p=rx->JobIds; get_next_jobid_from_list(&p, &JobId) > 0; ) {
char ed1[50];
for (TREE_NODE *node=first_tree_node(tree.root); node; node=next_tree_node(node)) {
Dmsg2(400, "FI=%d node=0x%x\n", node->FileIndex, node);
if (node->extract || node->extract_dir) {
- Dmsg2(400, "type=%d FI=%d\n", node->type, node->FileIndex);
+ Dmsg3(400, "JobId=%lld type=%d FI=%d\n", (uint64_t)node->JobId, node->type, node->FileIndex);
add_findex(rx->bsr, node->JobId, node->FileIndex);
if (node->extract && node->type != TN_NEWDIR) {
rx->selected_files++; /* count only saved files */
}
/* Get the JobIds from that list */
- rx->JobIds[0] = 0;
- rx->last_jobid[0] = 0;
+ rx->last_jobid[0] = rx->JobIds[0] = 0;
+
if (!db_sql_query(ua->db, uar_sel_jobid_temp, jobid_handler, (void *)rx)) {
ua->warning_msg("%s\n", db_strerror(ua->db));
}
prtit, ua, HORZ_LIST);
}
/* Display a list of Jobs selected for this restore */
- db_list_sql_query(ua->jcr, ua->db, uar_list_temp, prtit, ua, 1, HORZ_LIST);
+ db_list_sql_query(ua->jcr, ua->db, uar_list_temp, prtit, ua, 1,HORZ_LIST);
ok = true;
+
} else {
ua->warning_msg(_("No jobs found.\n"));
}
for (int i=0; i < name_list->num_ids; i++) {
free(name_list->name[i]);
}
- if (name_list->name) {
- free(name_list->name);
- name_list->name = NULL;
- }
+ bfree_and_null(name_list->name);
name_list->max_ids = 0;
name_list->num_ids = 0;
}
node = insert_tree_node(row[0], row[1], type, tree->root, NULL);
JobId = str_to_int64(row[3]);
FileIndex = str_to_int64(row[2]);
+ Dmsg2(400, "JobId=%s FileIndex=%s\n", row[3], row[2]);
/*
* - The first time we see a file (node->inserted==true), we accept it.
* - In the same JobId, we accept only the first copy of a
#include "bacula.h"
#include "filed.h"
-static int dbglvl=200;
+static int dbglvl=100;
typedef struct PrivateCurFile {
hlink link;
char *fname;
- utime_t ctime;
- utime_t mtime;
+ char *lstat;
bool seen;
} CurFile;
return true;
}
+static bool accurate_send_base_file_list(JCR *jcr)
+{
+ CurFile *elt;
+ struct stat statc;
+ int32_t LinkFIc;
+ FF_PKT *ff_pkt;
+ int stream = STREAM_UNIX_ATTRIBUTES;
+
+ if (!jcr->accurate || jcr->get_JobLevel() != L_FULL) {
+ return true;
+ }
+
+ if (jcr->file_list == NULL) {
+ return true;
+ }
+
+ ff_pkt = init_find_files();
+ ff_pkt->type = FT_BASE;
+
+ foreach_htable(elt, jcr->file_list) {
+ if (elt->seen) {
+ Dmsg2(dbglvl, "base file fname=%s seen=%i\n", elt->fname, elt->seen);
+ /* TODO: skip the decode and use directly the lstat field */
+ decode_stat(elt->lstat, &statc, &LinkFIc); /* decode catalog stat */
+ ff_pkt->fname = elt->fname;
+ ff_pkt->statp = statc;
+ encode_and_send_attributes(jcr, ff_pkt, stream);
+// free(elt->fname);
+ }
+ }
+
+ term_find_files(ff_pkt);
+ return true;
+}
+
+
/* This function is called at the end of backup
* We walk over all hash disk element, and we check
* for elt.seen.
*/
-bool accurate_send_deleted_list(JCR *jcr)
+static bool accurate_send_deleted_list(JCR *jcr)
{
CurFile *elt;
+ struct stat statc;
+ int32_t LinkFIc;
FF_PKT *ff_pkt;
int stream = STREAM_UNIX_ATTRIBUTES;
- if (!jcr->accurate || jcr->get_JobLevel() == L_FULL) {
- goto bail_out;
+ if (!jcr->accurate) {
+ return true;
}
if (jcr->file_list == NULL) {
- goto bail_out;
+ return true;
}
ff_pkt = init_find_files();
continue;
}
Dmsg2(dbglvl, "deleted fname=%s seen=%i\n", elt->fname, elt->seen);
+ /* TODO: skip the decode and use directly the lstat field */
+ decode_stat(elt->lstat, &statc, &LinkFIc); /* decode catalog stat */
ff_pkt->fname = elt->fname;
- ff_pkt->statp.st_mtime = elt->mtime;
- ff_pkt->statp.st_ctime = elt->ctime;
+ ff_pkt->statp.st_mtime = statc.st_mtime;
+ ff_pkt->statp.st_ctime = statc.st_ctime;
encode_and_send_attributes(jcr, ff_pkt, stream);
// free(elt->fname);
}
term_find_files(ff_pkt);
-bail_out:
- /* TODO: clean htable when this function is not reached ? */
- accurate_free(jcr);
return true;
}
}
}
+/* Send the deleted or the base file list and cleanup */
+bool accurate_finish(JCR *jcr)
+{
+ bool ret=true;
+ if (jcr->accurate) {
+ if (jcr->get_JobLevel() == L_FULL) {
+ ret = accurate_send_base_file_list(jcr);
+ } else {
+ ret = accurate_send_deleted_list(jcr);
+ }
+
+ accurate_free(jcr);
+ if (jcr->get_JobLevel() == L_FULL) {
+ Dmsg1(0, "Space saved with Base jobs: %lld MB\n",
+ jcr->base_size/(1024*1024));
+ }
+ }
+ return ret;
+}
+
static bool accurate_add_file(JCR *jcr, char *fname, char *lstat)
{
bool ret = true;
CurFile elt;
- struct stat statp;
- int32_t LinkFIc;
- decode_stat(lstat, &statp, &LinkFIc); /* decode catalog stat */
- elt.ctime = statp.st_ctime;
- elt.mtime = statp.st_mtime;
elt.seen = 0;
CurFile *item;
/* we store CurFile, fname and ctime/mtime in the same chunk */
- item = (CurFile *)jcr->file_list->hash_malloc(sizeof(CurFile)+strlen(fname)+1);
+ item = (CurFile *)jcr->file_list->hash_malloc(sizeof(CurFile)+strlen(fname)+strlen(lstat)+2);
memcpy(item, &elt, sizeof(CurFile));
item->fname = (char *)item+sizeof(CurFile);
strcpy(item->fname, fname);
+ item->lstat = item->fname+strlen(item->fname)+1;
+ strcpy(item->lstat, lstat);
jcr->file_list->insert(item->fname, item);
Dmsg2(dbglvl, "add fname=<%s> lstat=%s\n", fname, lstat);
*/
bool accurate_check_file(JCR *jcr, FF_PKT *ff_pkt)
{
+ struct stat statc;
+ int32_t LinkFIc;
bool stat = false;
char *fname;
CurFile elt;
- if (!jcr->accurate || jcr->get_JobLevel() == L_FULL) {
+ if (!jcr->accurate) {
return true;
}
goto bail_out;
}
+ decode_stat(elt.lstat, &statc, &LinkFIc); /* decode catalog stat */
+
+//#if 0
+ /*
+ * Loop over options supplied by user and verify the
+ * fields he requests.
+ */
+ for (char *p=ff_pkt->AccurateOpts; *p; p++) {
+ char ed1[30], ed2[30];
+ switch (*p) {
+ case 'i': /* compare INODEs */
+ if (statc.st_ino != ff_pkt->statp.st_ino) {
+ Dmsg3(dbglvl-1, "%s st_ino differ. Cat: %s File: %s\n",
+ fname,
+ edit_uint64((uint64_t)statc.st_ino, ed1),
+ edit_uint64((uint64_t)ff_pkt->statp.st_ino, ed2));
+ stat = true;
+ }
+ break;
+ case 'p': /* permissions bits */
+ if (statc.st_mode != ff_pkt->statp.st_mode) {
+ Dmsg3(dbglvl-1, "%s st_mode differ. Cat: %x File: %x\n",
+ fname,
+ (uint32_t)statc.st_mode, (uint32_t)ff_pkt->statp.st_mode);
+ stat = true;
+ }
+ break;
+ case 'n': /* number of links */
+ if (statc.st_nlink != ff_pkt->statp.st_nlink) {
+ Dmsg3(dbglvl-1, "%s st_nlink differ. Cat: %d File: %d\n",
+ fname,
+ (uint32_t)statc.st_nlink, (uint32_t)ff_pkt->statp.st_nlink);
+ stat = true;
+ }
+ break;
+ case 'u': /* user id */
+ if (statc.st_uid != ff_pkt->statp.st_uid) {
+ Dmsg3(dbglvl-1, "%s st_uid differ. Cat: %u File: %u\n",
+ fname,
+ (uint32_t)statc.st_uid, (uint32_t)ff_pkt->statp.st_uid);
+ stat = true;
+ }
+ break;
+ case 'g': /* group id */
+ if (statc.st_gid != ff_pkt->statp.st_gid) {
+ Dmsg3(dbglvl-1, "%s st_gid differ. Cat: %u File: %u\n",
+ fname,
+ (uint32_t)statc.st_gid, (uint32_t)ff_pkt->statp.st_gid);
+ stat = true;
+ }
+ break;
+ case 's': /* size */
+ if (statc.st_size != ff_pkt->statp.st_size) {
+ Dmsg3(dbglvl-1, "%s st_size differ. Cat: %s File: %s\n",
+ fname,
+ edit_uint64((uint64_t)statc.st_size, ed1),
+ edit_uint64((uint64_t)ff_pkt->statp.st_size, ed2));
+ stat = true;
+ }
+ break;
+ case 'a': /* access time */
+ if (statc.st_atime != ff_pkt->statp.st_atime) {
+ Dmsg1(dbglvl-1, "%s st_atime differs\n", fname);
+ stat = true;
+ }
+ break;
+ case 'm':
+ if (statc.st_mtime != ff_pkt->statp.st_mtime) {
+ Dmsg1(dbglvl-1, "%s st_mtime differs\n", fname);
+ stat = true;
+ }
+ break;
+ case 'c': /* ctime */
+ if (statc.st_ctime != ff_pkt->statp.st_ctime) {
+ Dmsg1(dbglvl-1, " st_ctime differs\n", fname);
+ stat = true;
+ }
+ break;
+ case 'd': /* file size decrease */
+ if (statc.st_size > ff_pkt->statp.st_size) {
+ Dmsg3(dbglvl-1, "%s st_size decrease. Cat: %s File: %s\n",
+ fname,
+ edit_uint64((uint64_t)statc.st_size, ed1),
+ edit_uint64((uint64_t)ff_pkt->statp.st_size, ed2));
+ stat = true;
+ }
+ break;
+ case '5': /* compare MD5 */
+ break;
+ case '1': /* compare SHA1 */
+ break;
+ case ':':
+ case 'C':
+ default:
+ break;
+ }
+ }
+//#endif
+#if 0
/*
* We check only mtime/ctime like with the normal
* incremental/differential mode
*/
- if (elt.mtime != ff_pkt->statp.st_mtime) {
+ if (statc.st_mtime != ff_pkt->statp.st_mtime) {
// Jmsg(jcr, M_SAVED, 0, _("%s st_mtime differs\n"), fname);
Dmsg3(dbglvl, "%s st_mtime differs (%lld!=%lld)\n",
- fname, elt.mtime, (utime_t)ff_pkt->statp.st_mtime);
+ fname, statc.st_mtime, (utime_t)ff_pkt->statp.st_mtime);
stat = true;
} else if (!(ff_pkt->flags & FO_MTIMEONLY)
- && (elt.ctime != ff_pkt->statp.st_ctime)) {
+ && (statc.st_ctime != ff_pkt->statp.st_ctime)) {
// Jmsg(jcr, M_SAVED, 0, _("%s st_ctime differs\n"), fname);
- Dmsg3(dbglvl, "%s st_ctime differs\n",
- fname, elt.ctime, ff_pkt->statp.st_ctime);
- stat = true;
+ Dmsg1(dbglvl, "%s st_ctime differs\n", fname);
+ stat = true;
+
+ } else if (statc.st_size != ff_pkt->statp.st_size) {
+// Jmsg(jcr, M_SAVED, 0, _("%s st_size differs\n"), fname);
+ Dmsg1(dbglvl, "%s st_size differs\n", fname);
+ stat = true;
}
+#endif
- accurate_mark_file_as_seen(jcr, &elt);
-// Dmsg2(dbglvl, "accurate %s = %d\n", fname, stat);
+ /* In Incr/Diff accurate mode, we mark all files as seen
+ * When in Full+Base mode, we mark only if the file match exactly
+ */
+ if (jcr->get_JobLevel() == L_FULL) {
+ if (!stat) {
+ /* compute space saved with basefile */
+ jcr->base_size += ff_pkt->statp.st_size;
+ accurate_mark_file_as_seen(jcr, &elt);
+ }
+ } else {
+ accurate_mark_file_as_seen(jcr, &elt);
+ }
bail_out:
unstrip_path(ff_pkt);
int len;
int32_t nb;
- if (!jcr->accurate || job_canceled(jcr) || jcr->get_JobLevel()==L_FULL) {
+ if (job_canceled(jcr)) {
return true;
}
if (sscanf(dir->msg, "accurate files=%ld", &nb) != 1) {
return false;
}
+ jcr->accurate = true;
+
accurate_init(jcr, nb);
/*
set_jcr_job_status(jcr, JS_ErrorTerminated);
}
- accurate_send_deleted_list(jcr); /* send deleted list to SD */
+ accurate_finish(jcr); /* send deleted or base file list to SD */
stop_heartbeat_monitor(jcr);
level = get_memory(dir->msglen+1);
Dmsg1(100, "level_cmd: %s", dir->msg);
+
+ /* keep compatibility with older directors */
if (strstr(dir->msg, "accurate")) {
jcr->accurate = true;
}
bool parse_acl_stream(JCR *jcr, int stream);
/* from accurate.c */
-bool accurate_send_deleted_list(JCR *jcr);
+bool accurate_finish(JCR *jcr);
bool accurate_check_file(JCR *jcr, FF_PKT *ff_pkt);
bool accurate_mark_file_as_seen(JCR *jcr, char *fname);
void accurate_free(JCR *jcr);
ff->flags = 0;
ff->VerifyOpts[0] = 'V';
ff->VerifyOpts[1] = 0;
+ strcpy(ff->AccurateOpts, "C:mc"); /* mtime+ctime by default */
for (i=0; i<fileset->include_list.size(); i++) {
findINCEXE *incexe = (findINCEXE *)fileset->include_list.get(i);
fileset->incexe = incexe;
ff->fstypes = fo->fstype;
ff->drivetypes = fo->drivetype;
bstrncat(ff->VerifyOpts, fo->VerifyOpts, sizeof(ff->VerifyOpts));
+ bstrncat(ff->AccurateOpts, fo->AccurateOpts, sizeof(ff->AccurateOpts));
}
dlistString *node;
foreach_dlist(node, &incexe->name_list) {
bool null_output_device; /* using null output device */
bool incremental; /* incremental save */
char VerifyOpts[20];
+ char AccurateOpts[20];
struct s_included_file *included_files_list;
struct s_excluded_file *excluded_files_list;
struct s_excluded_file *excluded_paths_list;
* since our last "save_time", presumably the last Full save
* or Incremental.
*/
- if ( ff_pkt->incremental
- && !S_ISDIR(ff_pkt->statp.st_mode)
+ if ( !S_ISDIR(ff_pkt->statp.st_mode)
&& !check_changes(jcr, ff_pkt))
{
Dmsg1(500, "Non-directory incremental: %s\n", ff_pkt->fname);
link[len] = 0;
ff_pkt->link = link;
- if (ff_pkt->incremental && !check_changes(jcr, ff_pkt)) {
- /* Incremental option, directory entry not changed */
+ if (!check_changes(jcr, ff_pkt)) {
+ /* Incremental/Full+Base option, directory entry not changed */
ff_pkt->type = FT_DIRNOCHG;
} else {
ff_pkt->type = FT_DIRBEGIN;
B_DB *db; /* database pointer */
B_DB *db_batch; /* database pointer for batch and accurate */
bool batch_started; /* is batch mode already started ? */
+ bool HasBase; /* True if job use base jobs */
+ uint64_t nb_base_files; /* Number of base files */
+ uint64_t nb_base_files_used; /* Number of useful files in base */
+
ATTR_DBR *ar; /* DB attribute record */
guid_list *id_list; /* User/group id to name list */
bool accurate; /* true if job is accurate */
bool VSS; /* VSS used by FD */
bool multi_restore; /* Dir can do multiple storage restore */
htable *file_list; /* Previous file list (accurate mode) */
+ uint64_t base_size; /* compute space saved with base job */
#endif /* FILE_DAEMON */
--- /dev/null
+#!/bin/sh
+#
+# Run a basejob backup of the Bacula build directory
+# then restore it.
+#
+
+TestName="base-job-test"
+JobName=backup
+. scripts/functions
+$rscripts/cleanup
+
+copy_test_confs
+sed 's/backup_advance/base_backup/' $rscripts/bacula-dir.conf.accurate > $tmp/1
+sed 's/Name = backup/Name = backup; Base = base_backup, backup/' $tmp/1 > $conf/bacula-dir.conf
+sed s/all,/all,saved,/ $conf/bacula-fd.conf > tmp/1
+cp tmp/1 $conf/bacula-fd.conf
+
+change_jobname BackupClient1 $JobName
+
+p() {
+ echo "##############################################" >> ${cwd}/tmp/log1.out
+ echo "$*" >> ${cwd}/tmp/log1.out
+ echo "##############################################" >> ${cwd}/tmp/log2.out
+ echo "$*" >> ${cwd}/tmp/log2.out
+}
+
+# cleanup
+rm -rf ${cwd}/build/accurate.new
+rm -rf ${cwd}/build/accurate
+
+
+# add extra files
+mkdir ${cwd}/build/accurate
+mkdir ${cwd}/build/accurate/dirtest
+echo "test test" > ${cwd}/build/accurate/dirtest/hello
+echo "test test" > ${cwd}/build/accurate/xxx
+echo "test test" > ${cwd}/build/accurate/yyy
+echo "test test" > ${cwd}/build/accurate/zzz
+echo "test test" > ${cwd}/build/accurate/zzzzzz
+echo "test test" > ${cwd}/build/accurate/xxxxxx
+echo "test test" > ${cwd}/build/accurate/yyyyyy
+echo "test test" > ${cwd}/build/accurate/xxxxxxxxx
+echo "test test" > ${cwd}/build/accurate/yyyyyyyyy
+echo "test test" > ${cwd}/build/accurate/zzzzzzzzz
+echo ${cwd}/build > ${cwd}/tmp/file-list
+
+start_test
+
+cat <<END_OF_DATA >${cwd}/tmp/bconcmds
+@output /dev/null
+messages
+label volume=TestVolume001 storage=File pool=Default
+messages
+END_OF_DATA
+
+run_bacula
+
+################################################################
+p Now do a backup using base backup
+################################################################
+
+echo ${cwd}/bin >> ${cwd}/tmp/file-list
+
+cat <<END_OF_DATA >${cwd}/tmp/bconcmds
+@$out ${cwd}/tmp/log1.out
+run job=base_backup level=base yes
+wait
+messages
+update volume=TestVolume001 volstatus=Used
+END_OF_DATA
+
+run_bconsole
+
+echo ${cwd}/build > ${cwd}/tmp/file-list
+
+cat <<END_OF_DATA >${cwd}/tmp/bconcmds
+@$out ${cwd}/tmp/log4.out
+label volume=TestVolume002 storage=File pool=Default
+run job=backup level=full yes
+wait
+messages
+@#
+@# now do a restore
+@#
+@$out ${cwd}/tmp/log2.out
+restore fileset=FS_TESTJOB where=${cwd}/tmp/bacula-restores select all done
+yes
+wait
+messages
+END_OF_DATA
+
+
+run_bconsole
+check_for_zombie_jobs storage=File
+
+check_two_logs
+check_restore_diff
+
+rm -rf ${cwd}/tmp/bacula-restores
+
+grep -e 'FD Bytes Written: *0' ${cwd}/tmp/log4.out > /dev/null
+if [ $? -ne 0 ]; then
+ print_debug "The first full job should have 0 byte in log4.out"
+ bstat=2
+fi
+
+################################################################
+p Now do a backup after making few changes
+################################################################
+cat <<END_OF_DATA >${cwd}/tmp/bconcmds
+@$out ${cwd}/tmp/log1.out
+update volume=TestVolume002 volstatus=Used
+label volume=TestVolume003 storage=File pool=Default
+run job=backup level=incremental yes
+wait
+messages
+@#
+@# now do a restore
+@#
+@$out ${cwd}/tmp/log2.out
+restore fileset=FS_TESTJOB where=${cwd}/tmp/bacula-restores select all done
+yes
+wait
+messages
+END_OF_DATA
+
+rm ${cwd}/build/accurate/yyyyyy # delete a file
+rm -rf ${cwd}/build/accurate/dirtest
+
+
+run_bconsole
+check_for_zombie_jobs storage=File
+
+check_two_logs
+check_restore_diff
+check_files_written ${cwd}/tmp/log1.out 4
+
+rm -rf ${cwd}/tmp/bacula-restores
+
+################################################################
+p Test the job purge
+################################################################
+cat <<END_OF_DATA >${cwd}/tmp/bconcmds
+@$out ${cwd}/tmp/log3.out
+sql
+SELECT count(*) FROM BaseFiles;
+
+purge volume=TestVolume002
+messages
+sql
+SELECT count(*) FROM BaseFiles;
+
+END_OF_DATA
+
+run_bconsole
+
+grep -e ' 0 *|' ${cwd}/tmp/log3.out > /dev/null
+if [ $? -ne 0 ]; then
+ print_debug "Can't purge the base job"
+ dstat=2
+fi
+
+
+################################################################
+p Test list commands
+################################################################
+
+touch ${cwd}/build/po/fr.po
+
+cat <<END_OF_DATA >${cwd}/tmp/bconcmds
+run level=full job=backup yes
+wait
+messages
+@$out ${cwd}/tmp/log5.out
+list basefiles jobid=6
+@$out ${cwd}/tmp/log6.out
+list files jobid=6
+messages
+END_OF_DATA
+
+run_bconsole
+
+grep po/fr.po ${cwd}/tmp/log5.out > /dev/null
+if [ $? -eq 0 ]; then
+ print_debug "Should not display fr.po as basefile"
+ bstat=2
+fi
+
+export bstat dstat
+
+stop_bacula
+end_test
projects / bacula / bacula / commitdiff