From 895be3269c70926d7200ee7fba8b6cb64b269512 Mon Sep 17 00:00:00 2001 From: Eric Bollengier Date: Wed, 29 Jul 2009 18:09:33 +0200 Subject: [PATCH] make sql part --- bacula/src/baconfig.h | 1 + bacula/src/cats/cats.h | 1 + bacula/src/cats/sql_create.c | 135 +++++++++++++++++++++++++++++++++++ bacula/src/dird/catreq.c | 27 +++++-- bacula/src/filed/accurate.c | 61 ++++++++++++++-- bacula/src/filed/backup.c | 2 +- bacula/src/filed/protos.h | 2 +- 7 files changed, 213 insertions(+), 16 deletions(-) diff --git a/bacula/src/baconfig.h b/bacula/src/baconfig.h index 24301083f3..37e787f0e3 100644 --- a/bacula/src/baconfig.h +++ b/bacula/src/baconfig.h @@ -332,6 +332,7 @@ void InitWinAPIWrapper(); #define FT_REPARSE 21 /* Win NTFS reparse point */ #define FT_PLUGIN 22 /* Plugin generated filename */ #define FT_DELETED 23 /* Deleted file entry */ +#define FT_BASE 24 /* Duplicate base file entry */ /* Definitions for upper part of type word (see above). */ #define AR_DATA_STREAM (1<<16) /* Data stream id present */ diff --git a/bacula/src/cats/cats.h b/bacula/src/cats/cats.h index 2ff803fbd2..ae57dc1009 100644 --- a/bacula/src/cats/cats.h +++ b/bacula/src/cats/cats.h @@ -859,6 +859,7 @@ struct ATTR_DBR { char *attr; /* attributes statp */ uint32_t FileIndex; uint32_t Stream; + uint32_t FileType; JobId_t JobId; DBId_t ClientId; DBId_t PathId; diff --git a/bacula/src/cats/sql_create.c b/bacula/src/cats/sql_create.c index 422a0a6b42..cc8d6e7433 100644 --- a/bacula/src/cats/sql_create.c +++ b/bacula/src/cats/sql_create.c @@ -1108,4 +1108,139 @@ bool db_write_batch_file_records(JCR *jcr) #endif /* ! HAVE_BATCH_FILE_INSERT */ + +/* List of SQL commands to create temp table and indicies */ +const char *create_temp_basefile[4] = { + /* MySQL */ + "CREATE TEMPORARY TABLE basefile%lld (" + "Name BLOB NOT NULL," + "FileName BLOB NOT NULL)", + + /* Postgresql */ + "CREATE TEMPORARY TABLE basefile%lld (" + "Name TEXT," + "FileName TEXT)", + + /* SQLite */ + "CREATE TEMPORARY TABLE basefile%lld (" + "Name TEXT," + "FileName TEXT)", + + /* SQLite3 */ + "CREATE TEMPORARY TABLE basefile%lld (" + "Name TEXT," + "FileName TEXT)" +}; + +boot db_init_base_file(JCR *jcr, B_DB *mdb) +{ + POOL_MEM q(PM_MESSAGE); + Mmsg(q, create_temp_basefile[db_type], (uint64_t) jcr->JobId); + return db_sql_query(mdb, q.c_str(), NULL, NULL); +} + +/* + * Create Base File record in B_DB + * + */ +bool db_create_base_file_attributes_record(JCR *jcr, B_DB *mdb, ATTR_DBR *ar) +{ + Dmsg1(dbglevel, "Fname=%s\n", ar->fname); + Dmsg0(dbglevel, "put_file_into_catalog\n"); + + /* + * Make sure we have an acceptable attributes record. + */ + if (!(ar->Stream == STREAM_UNIX_ATTRIBUTES || + ar->Stream == STREAM_UNIX_ATTRIBUTES_EX)) { + Mmsg1(&mdb->errmsg, _("Attempt to put non-attributes into catalog. Stream=%d\n"), + ar->Stream); + Jmsg(jcr, M_FATAL, 0, "%s", mdb->errmsg); + return false; + } + + db_lock(mdb); + split_path_and_file(jcr, bdb, ar->fname); + + mdb->esc_name = check_pool_memory_size(mdb->esc_name, mdb->fnl*2+1); + db_escape_string(jcr, mdb, mdb->esc_name, mdb->fname, mdb->fnl); + + mdb->esc_path = check_pool_memory_size(mdb->esc_path, mdb->pnl*2+1); + db_escape_string(jcr, mdb, mdb->esc_path, mdb->path, mdb->pnl); + + len = Mmsg(mdb->cmd, "INSERT INTO basefile%lld VALUES ('%s','%s')", + (uint64_t)jcr->JobId, mdb->esc_path, mdb->esc_name); + + boot ret = INSERT_DB(jcr, mdb, mdb->cmd); + db_unlock(mdb); + + return ret; +} +/* + * Put all base file seen in the backup to the BaseFile table + */ +bool db_commit_base_file_attributes_record(JCR *jcr, B_DB *mdb) +{ + char ed1[50]; + POOL_MEM buf(PM_MESSAGE); + + Mmsg(buf, + "INSERT INTO BaseFile (BaseJobId, JobId, FileId, FileIndex) ( " + "SELECT A.JobId AS BaseJobId, %s AS JobId, " + "A.FileId, A.FileIndex " + "FROM basefile%s AS A, new_basefile%s AS B " + "WHERE A.Path = B.Path " + "AND A.Filename = B.Filename " + "ORDER BY FileId)", + edit_uint64(ed1, jcr->JobId), ed1, ed1); + + return db_sql_query(mdb, buf.c_str(), NULL, NULL); +} + +/* + * Cleanup the base file temporary tables + */ +void db_cleanup_base_file(JCR *jcr, B_DB *mdb) +{ + Mmsg(buf, "DROP TABLE new_basefile%lld", (uint64_t) jcr->JobId); + db_sql_query(mdb, buf.c_str(), NULL, NULL); + + Mmsg(buf, "DROP TABLE basefile%lld", (uint64_t) jcr->JobId); + db_sql_query(mdb, buf.c_str(), NULL, NULL); +} + +/* + * Find the last "accurate" backup state with Base jobs + * 1) Get all files with jobid in list (F subquery) + * 2) Take only the last version of each file (Temp subquery) => accurate list is ok + * 3) Put the result in a temporary table for the end of job + * + */ +bool db_create_base_file_list(JCR *jcr, B_DB *mdb, char *jobids) +{ + if (!*jobids) { + db_lock(mdb); + Mmsg(mdb->errmsg, _("ERR=JobIds are empty\n")); + db_unlock(mdb); + return false; + } + POOL_MEM buf(PM_MESSAGE); + + Mmsg(buf, + "CREATE TEMPORARY new_basefile%lld AS ( " + "SELECT Path.Path AS Path, Filename.Name AS Filename, File.FileIndex AS FileIndex, " + "File.JobId AS JobId, File.LStat AS LStat, File.FileId AS FileId " + "FROM ( " + "SELECT max(FileId) as FileId, PathId, FilenameId " + "FROM (SELECT FileId, PathId, FilenameId FROM File WHERE JobId IN (%s)) AS F " + "GROUP BY PathId, FilenameId " + ") AS Temp " + "JOIN Filename ON (Filename.FilenameId = Temp.FilenameId) " + "JOIN Path ON (Path.PathId = Temp.PathId) " + "JOIN File ON (File.FileId = Temp.FileId) " + "WHERE File.FileIndex > 0)", + (uint64_t)jcr->JobId, jobids); + return db_sql_query(mdb, buf.c_str(), NULL, NULL); +} + #endif /* HAVE_SQLITE3 || HAVE_MYSQL || HAVE_SQLITE || HAVE_POSTGRESQL || HAVE_DBI */ diff --git a/bacula/src/dird/catreq.c b/bacula/src/dird/catreq.c index aac92cf8fa..c317ad4441 100644 --- a/bacula/src/dird/catreq.c +++ b/bacula/src/dird/catreq.c @@ -370,7 +370,6 @@ static void update_attribute(JCR *jcr, char *msg, int32_t msglen) uint32_t FileIndex; uint32_t data_len; char *p; - int filetype; int len; char *fname, *attr; ATTR_DBR *ar = NULL; @@ -415,7 +414,7 @@ static void update_attribute(JCR *jcr, char *msg, int32_t msglen) p = jcr->attr - msg + p; /* point p into jcr->attr */ skip_nonspaces(&p); /* skip FileIndex */ skip_spaces(&p); - filetype = str_to_int32(p); /* TODO: choose between unserialize and str_to_int32 */ + ar->FileType = str_to_int32(p); /* TODO: choose between unserialize and str_to_int32 */ skip_nonspaces(&p); /* skip FileType */ skip_spaces(&p); fname = p; @@ -426,7 +425,7 @@ static void update_attribute(JCR *jcr, char *msg, int32_t msglen) Dmsg1(400, "dirdattr = attr; ar->fname = fname; - if (filetype == FT_DELETED) { + if (ar->FileType == FT_DELETED) { ar->FileIndex = 0; /* special value */ } else { ar->FileIndex = FileIndex; @@ -479,14 +478,28 @@ static void update_attribute(JCR *jcr, char *msg, int32_t msglen) } bin_to_base64(digestbuf, sizeof(digestbuf), fname, len, true); - Dmsg3(400, "DigestLen=%d Digest=%s type=%d\n", strlen(digestbuf), digestbuf, Stream); + Dmsg3(400, "DigestLen=%d Digest=%s type=%d\n", strlen(digestbuf), + digestbuf, Stream); if (jcr->cached_attribute) { ar->Digest = digestbuf; ar->DigestType = type; - Dmsg2(400, "Cached attr with digest. Stream=%d fname=%s\n", ar->Stream, ar->fname); - if (!db_create_file_attributes_record(jcr, jcr->db, ar)) { - Jmsg1(jcr, M_FATAL, 0, _("Attribute create error. %s"), db_strerror(jcr->db)); + Dmsg2(400, "Cached attr with digest. Stream=%d fname=%s\n", + ar->Stream, ar->fname); + + /* Update BaseFile table */ + if (ar->FileType == FT_BASE) { + if (!db_create_base_file_attributes_record(jcr, jcr->mdb, ar)) { + Jmsg1(jcr, M_FATAL, 0, _("Base attribute create error. %s"), + db_strerror(jcr->db)); + } + + } else { /* Regular files */ + if (!db_create_file_attributes_record(jcr, jcr->db, ar)) { + Jmsg1(jcr, M_FATAL, 0, _("Attribute create error. %s"), + db_strerror(jcr->db)); + } } + jcr->cached_attribute = false; } else { if (!db_add_digest_to_file_record(jcr, jcr->db, ar->FileId, digestbuf, type)) { diff --git a/bacula/src/filed/accurate.c b/bacula/src/filed/accurate.c index 49d78ffb86..c9a69390e6 100644 --- a/bacula/src/filed/accurate.c +++ b/bacula/src/filed/accurate.c @@ -92,22 +92,56 @@ static bool accurate_init(JCR *jcr, int nbfile) return true; } +static bool accurate_send_base_file_list(JCR *jcr) +{ + CurFile *elt; + FF_PKT *ff_pkt; + int stream = STREAM_UNIX_ATTRIBUTES; + + if (!jcr->accurate || jcr->get_JobLevel != L_FULL) { + return true; + } + + if (jcr->file_list == NULL) { + return true; + } + + ff_pkt = init_find_files(); + ff_pkt->type = FT_BASE; + + foreach_htable(elt, jcr->file_list) { + if (!elt->seen || !plugin_check_file(jcr, elt->fname)) { + continue; + } + Dmsg2(dbglvl, "base file fname=%s seen=%i\n", elt->fname, elt->seen); + ff_pkt->fname = elt->fname; + ff_pkt->statp.st_mtime = elt->mtime; + ff_pkt->statp.st_ctime = elt->ctime; + encode_and_send_attributes(jcr, ff_pkt, stream); +// free(elt->fname); + } + + term_find_files(ff_pkt); + return true; +} + + /* This function is called at the end of backup * We walk over all hash disk element, and we check * for elt.seen. */ -bool accurate_send_deleted_list(JCR *jcr) +static bool accurate_send_deleted_list(JCR *jcr) { CurFile *elt; FF_PKT *ff_pkt; int stream = STREAM_UNIX_ATTRIBUTES; if (!jcr->accurate) { - goto bail_out; + return true; } if (jcr->file_list == NULL) { - goto bail_out; + return true; } ff_pkt = init_find_files(); @@ -126,13 +160,10 @@ bool accurate_send_deleted_list(JCR *jcr) } term_find_files(ff_pkt); -bail_out: - /* TODO: clean htable when this function is not reached ? */ - accurate_free(jcr); return true; } -void accurate_free(JCR *jcr) +static void accurate_free(JCR *jcr) { if (jcr->file_list) { jcr->file_list->destroy(); @@ -141,6 +172,22 @@ void accurate_free(JCR *jcr) } } +/* Send the deleted or the base file list and cleanup */ +bool accurate_finish(JCR *jcr) +{ + bool ret=true; + if (jcr->accurate) { + if (jcr->get_JobLevel == L_FULL) { + ret = accurate_send_base_file_list(jcr); + } else { + ret = accurate_send_deleted_list(jcr); + } + + accurate_free(jcr); + } + return ret; +} + static bool accurate_add_file(JCR *jcr, char *fname, char *lstat) { bool ret = true; diff --git a/bacula/src/filed/backup.c b/bacula/src/filed/backup.c index f1818260c8..1721199ea0 100644 --- a/bacula/src/filed/backup.c +++ b/bacula/src/filed/backup.c @@ -153,7 +153,7 @@ bool blast_data_to_storage_daemon(JCR *jcr, char *addr) set_jcr_job_status(jcr, JS_ErrorTerminated); } - accurate_send_deleted_list(jcr); /* send deleted list to SD */ + accurate_finish(jcr); /* send deleted or base file list to SD */ stop_heartbeat_monitor(jcr); diff --git a/bacula/src/filed/protos.h b/bacula/src/filed/protos.h index b5daaa8c19..eb48006283 100644 --- a/bacula/src/filed/protos.h +++ b/bacula/src/filed/protos.h @@ -51,7 +51,7 @@ bool build_acl_streams(JCR *jcr, FF_PKT *ff_pkt); bool parse_acl_stream(JCR *jcr, int stream); /* from accurate.c */ -bool accurate_send_deleted_list(JCR *jcr); +bool accurate_finish(JCR *jcr); bool accurate_check_file(JCR *jcr, FF_PKT *ff_pkt); bool accurate_mark_file_as_seen(JCR *jcr, char *fname); void accurate_free(JCR *jcr); -- 2.39.5