X-Git-Url: https://git.sur5r.net/?a=blobdiff_plain;f=bacula%2Fsrc%2Fcats%2Fbvfs.c;h=c3b8bbd8a28ba94d589990bebc64b3f27931a1c7;hb=c9657772721a3782558c4eb8b99978c7873b45fd;hp=7c4bb2d6d3ce5e6a36962840b3199a25efa51713;hpb=3e192a5112c15535ab7aa1c81a8e9ad1d0318de5;p=bacula%2Fbacula diff --git a/bacula/src/cats/bvfs.c b/bacula/src/cats/bvfs.c index 7c4bb2d6d3..c3b8bbd8a2 100644 --- a/bacula/src/cats/bvfs.c +++ b/bacula/src/cats/bvfs.c @@ -1,12 +1,12 @@ /* Bacula® - The Network Backup Solution - Copyright (C) 2009-2009 Free Software Foundation Europe e.V. + Copyright (C) 2009-2010 Free Software Foundation Europe e.V. The main author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. This program is Free Software; you can redistribute it and/or - modify it under the terms of version two of the GNU General Public + modify it under the terms of version three of the GNU Affero General Public License as published by the Free Software Foundation, which is listed in the file LICENSE. @@ -15,7 +15,7 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - You should have received a copy of the GNU General Public License + You should have received a copy of the GNU Affero General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. @@ -58,11 +58,12 @@ Bvfs::Bvfs(JCR *j, B_DB *mdb) { jcr = j; jcr->inc_use_count(); db = mdb; /* need to inc ref count */ + jobids = get_pool_memory(PM_NAME); prev_dir = get_pool_memory(PM_NAME); pattern = get_pool_memory(PM_NAME); - *prev_dir = *pattern = 0; + *jobids = *prev_dir = *pattern = 0; dir_filenameid = pwd_id = offset = 0; - see_copies = see_all_version = false; + see_copies = see_all_versions = false; limit = 1000; attr = new_attr(jcr); list_entries = result_handler; @@ -70,6 +71,7 @@ Bvfs::Bvfs(JCR *j, B_DB *mdb) { } Bvfs::~Bvfs() { + free_pool_memory(jobids); free_pool_memory(pattern); free_pool_memory(prev_dir); free_attr(attr); @@ -93,6 +95,9 @@ private: hlink *nodes; int nb_node; int max_node; + + alist *table_node; + htable *cache_ppathid; public: @@ -103,14 +108,17 @@ public: max_node = NITEMS; nodes = (hlink *) malloc(max_node * sizeof (hlink)); nb_node = 0; + table_node = New(alist(5, owned_by_alist)); + table_node->append(nodes); } hlink *get_hlink() { - if (nb_node >= max_node) { - max_node *= 2; - nodes = (hlink *)brealloc(nodes, sizeof(hlink) * max_node); + if (++nb_node >= max_node) { + nb_node = 0; + nodes = (hlink *)malloc(max_node * sizeof(hlink)); + table_node->append(nodes); } - return nodes + nb_node++; + return nodes + nb_node; } bool lookup(char *pathid) { @@ -126,7 +134,7 @@ public: ~pathid_cache() { cache_ppathid->destroy(); free(cache_ppathid); - free(nodes); + delete table_node; } private: pathid_cache(const pathid_cache &); /* prohibit pass by value */ @@ -196,7 +204,7 @@ static void build_path_hierarchy(JCR *jcr, B_DB *mdb, /* Does the ppathid exist for this ? we use a memory cache... In order to * avoid the full loop, we consider that if a dir is allready in the - * brestore_pathhierarchy table, then there is no need to calculate all the + * PathHierarchy table, then there is no need to calculate all the * hierarchy */ while (path && *path) @@ -204,7 +212,7 @@ static void build_path_hierarchy(JCR *jcr, B_DB *mdb, if (!ppathid_cache.lookup(pathid)) { Mmsg(mdb->cmd, - "SELECT PPathId FROM brestore_pathhierarchy WHERE PathId = %s", + "SELECT PPathId FROM PathHierarchy WHERE PathId = %s", pathid); QUERY_DB(jcr, mdb, mdb->cmd); @@ -226,7 +234,7 @@ static void build_path_hierarchy(JCR *jcr, B_DB *mdb, ppathid_cache.insert(pathid); Mmsg(mdb->cmd, - "INSERT INTO brestore_pathhierarchy (PathId, PPathId) " + "INSERT INTO PathHierarchy (PathId, PPathId) " "VALUES (%s,%lld)", pathid, (uint64_t) parent.PathId); @@ -265,7 +273,7 @@ static void update_path_hierarchy_cache(JCR *jcr, db_lock(mdb); db_start_transaction(jcr, mdb); - Mmsg(mdb->cmd, "SELECT 1 FROM brestore_knownjobid WHERE JobId = %s", jobid); + Mmsg(mdb->cmd, "SELECT 1 FROM Job WHERE JobId = %s AND HasCache=1", jobid); if (!QUERY_DB(jcr, mdb, mdb->cmd) || sql_num_rows(mdb) > 0) { Dmsg1(dbglevel, "already computed %d\n", (uint32_t)JobId ); @@ -273,24 +281,27 @@ static void update_path_hierarchy_cache(JCR *jcr, } /* Inserting path records for JobId */ - Mmsg(mdb->cmd, "INSERT INTO brestore_pathvisibility (PathId, JobId) " - "SELECT DISTINCT PathId, JobId FROM File WHERE JobId = %s", - jobid); + Mmsg(mdb->cmd, "INSERT INTO PathVisibility (PathId, JobId) " + "SELECT DISTINCT PathId, JobId " + "FROM (SELECT PathId, JobId FROM File WHERE JobId = %s " + "UNION " + "SELECT PathId, BaseFiles.JobId FROM BaseFiles JOIN File AS F USING (FileId) " + "WHERE BaseFiles.JobId = %s) AS B", + jobid, jobid); QUERY_DB(jcr, mdb, mdb->cmd); - /* Now we have to do the directory recursion stuff to determine missing * visibility We try to avoid recursion, to be as fast as possible We also * only work on not allready hierarchised directories... */ Mmsg(mdb->cmd, - "SELECT brestore_pathvisibility.PathId, Path " - "FROM brestore_pathvisibility " - "JOIN Path ON( brestore_pathvisibility.PathId = Path.PathId) " - "LEFT JOIN brestore_pathhierarchy " - "ON (brestore_pathvisibility.PathId = brestore_pathhierarchy.PathId) " - "WHERE brestore_pathvisibility.JobId = %s " - "AND brestore_pathhierarchy.PathId IS NULL " + "SELECT PathVisibility.PathId, Path " + "FROM PathVisibility " + "JOIN Path ON( PathVisibility.PathId = Path.PathId) " + "LEFT JOIN PathHierarchy " + "ON (PathVisibility.PathId = PathHierarchy.PathId) " + "WHERE PathVisibility.JobId = %s " + "AND PathHierarchy.PathId IS NULL " "ORDER BY Path", jobid); Dmsg1(dbglevel_sql, "q=%s\n", mdb->cmd); QUERY_DB(jcr, mdb, mdb->cmd); @@ -319,17 +330,17 @@ static void update_path_hierarchy_cache(JCR *jcr, } free(result); } - + Mmsg(mdb->cmd, - "INSERT INTO brestore_pathvisibility (PathId, JobId) " + "INSERT INTO PathVisibility (PathId, JobId) " "SELECT a.PathId,%s " "FROM ( " "SELECT DISTINCT h.PPathId AS PathId " - "FROM brestore_pathhierarchy AS h " - "JOIN brestore_pathvisibility AS p ON (h.PathId=p.PathId) " + "FROM PathHierarchy AS h " + "JOIN PathVisibility AS p ON (h.PathId=p.PathId) " "WHERE p.JobId=%s) AS a LEFT JOIN " "(SELECT PathId " - "FROM brestore_pathvisibility " + "FROM PathVisibility " "WHERE JobId=%s) AS b ON (a.PathId = b.PathId) " "WHERE b.PathId IS NULL", jobid, jobid, jobid); @@ -337,8 +348,8 @@ static void update_path_hierarchy_cache(JCR *jcr, QUERY_DB(jcr, mdb, mdb->cmd); } while (sql_affected_rows(mdb) > 0); - Mmsg(mdb->cmd, "INSERT INTO brestore_knownjobid (JobId) VALUES (%s)", jobid); - INSERT_DB(jcr, mdb, mdb->cmd); + Mmsg(mdb->cmd, "UPDATE Job SET HasCache=1 WHERE JobId=%s", jobid); + UPDATE_DB(jcr, mdb, mdb->cmd); bail_out: db_end_transaction(jcr, mdb); @@ -364,95 +375,86 @@ DBId_t Bvfs::get_dir_filenameid() void bvfs_update_cache(JCR *jcr, B_DB *mdb) { uint32_t nb=0; - db_list_ctx jobids; + db_list_ctx jobids_list; db_lock(mdb); db_start_transaction(jcr, mdb); - Mmsg(mdb->cmd, "SELECT 1 from brestore_knownjobid LIMIT 1"); - /* TODO: Add this code in the make_bacula_table script */ +#ifdef xxx + /* TODO: Remove this code when updating make_bacula_table script */ + Mmsg(mdb->cmd, "SELECT 1 FROM Job WHERE HasCache<>2 LIMIT 1"); if (!QUERY_DB(jcr, mdb, mdb->cmd)) { Dmsg0(dbglevel, "Creating cache table\n"); - Mmsg(mdb->cmd, - "CREATE TABLE brestore_knownjobid (" - "JobId integer NOT NULL, " - "CONSTRAINT brestore_knownjobid_pkey PRIMARY KEY (JobId))"); + Mmsg(mdb->cmd, "ALTER TABLE Job ADD HasCache int DEFAULT 0"); QUERY_DB(jcr, mdb, mdb->cmd); Mmsg(mdb->cmd, - "CREATE TABLE brestore_pathhierarchy ( " + "CREATE TABLE PathHierarchy ( " "PathId integer NOT NULL, " "PPathId integer NOT NULL, " - "CONSTRAINT brestore_pathhierarchy_pkey " + "CONSTRAINT pathhierarchy_pkey " "PRIMARY KEY (PathId))"); QUERY_DB(jcr, mdb, mdb->cmd); Mmsg(mdb->cmd, - "CREATE INDEX brestore_pathhierarchy_ppathid " - "ON brestore_pathhierarchy (PPathId)"); + "CREATE INDEX pathhierarchy_ppathid " + "ON PathHierarchy (PPathId)"); QUERY_DB(jcr, mdb, mdb->cmd); Mmsg(mdb->cmd, - "CREATE TABLE brestore_pathvisibility (" + "CREATE TABLE PathVisibility (" "PathId integer NOT NULL, " "JobId integer NOT NULL, " "Size int8 DEFAULT 0, " "Files int4 DEFAULT 0, " - "CONSTRAINT brestore_pathvisibility_pkey " + "CONSTRAINT pathvisibility_pkey " "PRIMARY KEY (JobId, PathId))"); QUERY_DB(jcr, mdb, mdb->cmd); Mmsg(mdb->cmd, - "CREATE INDEX brestore_pathvisibility_jobid " - "ON brestore_pathvisibility (JobId)"); + "CREATE INDEX pathvisibility_jobid " + "ON PathVisibility (JobId)"); QUERY_DB(jcr, mdb, mdb->cmd); } +#endif Mmsg(mdb->cmd, "SELECT JobId from Job " - "WHERE JobId NOT IN (SELECT JobId FROM brestore_knownjobid) " + "WHERE HasCache = 0 " "AND Type IN ('B') AND JobStatus IN ('T', 'f', 'A') " "ORDER BY JobId"); - db_sql_query(mdb, mdb->cmd, db_list_handler, &jobids); + db_sql_query(mdb, mdb->cmd, db_list_handler, &jobids_list); - bvfs_update_path_hierarchy_cache(jcr, mdb, &jobids); + bvfs_update_path_hierarchy_cache(jcr, mdb, jobids_list.list); db_end_transaction(jcr, mdb); db_start_transaction(jcr, mdb); Dmsg0(dbglevel, "Cleaning pathvisibility\n"); Mmsg(mdb->cmd, - "DELETE FROM brestore_pathvisibility " - "WHERE NOT EXISTS " - "(SELECT 1 FROM Job WHERE JobId=brestore_pathvisibility.JobId)"); - nb = DELETE_DB(jcr, mdb, mdb->cmd); - Dmsg1(dbglevel, "Affected row(s) = %d\n", nb); - - Dmsg0(dbglevel, "Cleaning knownjobid\n"); - Mmsg(mdb->cmd, - "DELETE FROM brestore_knownjobid " + "DELETE FROM PathVisibility " "WHERE NOT EXISTS " - "(SELECT 1 FROM Job WHERE JobId=brestore_knownjobid.JobId)"); + "(SELECT 1 FROM Job WHERE JobId=PathVisibility.JobId)"); nb = DELETE_DB(jcr, mdb, mdb->cmd); Dmsg1(dbglevel, "Affected row(s) = %d\n", nb); db_end_transaction(jcr, mdb); + db_unlock(mdb); } /* * Update the bvfs cache for given jobids (1,2,3,4) */ void -bvfs_update_path_hierarchy_cache(JCR *jcr, B_DB *mdb, db_list_ctx *jobids) +bvfs_update_path_hierarchy_cache(JCR *jcr, B_DB *mdb, char *jobids) { pathid_cache ppathid_cache; JobId_t JobId; char *p; - for (p=jobids->list; ; ) { + for (p=jobids; ; ) { int stat = get_next_jobid_from_list(&p, &JobId); - Dmsg1(dbglevel, "Updating cache for %lld\n", (uint64_t)JobId); if (stat < 0) { return; } @@ -469,7 +471,7 @@ bvfs_update_path_hierarchy_cache(JCR *jcr, B_DB *mdb, db_list_ctx *jobids) */ void Bvfs::update_cache() { - bvfs_update_path_hierarchy_cache(jcr, db, &jobids); + bvfs_update_path_hierarchy_cache(jcr, db, jobids); } /* Change the current directory, returns true if the path exists */ @@ -483,6 +485,7 @@ bool Bvfs::ch_dir(const char *path) /* * Get all file versions for a specified client + * TODO: Handle basejobs using different client */ void Bvfs::get_all_file_versions(DBId_t pathid, DBId_t fnid, const char *client) { @@ -498,19 +501,21 @@ void Bvfs::get_all_file_versions(DBId_t pathid, DBId_t fnid, const char *client) POOL_MEM query; - Mmsg(query,//0 1 2 3 -"SELECT File.FileId, File.Md5, File.JobId, File.LStat, " -// 4 5 + Mmsg(query,// 1 2 3 +"SELECT 'V', File.PathId, File.FilenameId, File.Md5, " +// 4 5 6 + "File.JobId, File.LStat, File.FileId, " +// 7 8 "Media.VolumeName, Media.InChanger " "FROM File, Job, Client, JobMedia, Media " "WHERE File.FilenameId = %s " "AND File.PathId=%s " "AND File.JobId = Job.JobId " - "AND Job.ClientId = Client.ClientId " "AND Job.JobId = JobMedia.JobId " "AND File.FileIndex >= JobMedia.FirstIndex " "AND File.FileIndex <= JobMedia.LastIndex " "AND JobMedia.MediaId = Media.MediaId " + "AND Job.ClientId = Client.ClientId " "AND Client.Name = '%s' " "%s ORDER BY FileId LIMIT %d OFFSET %d" ,edit_uint64(fnid, ed1), edit_uint64(pathid, ed2), client, q.c_str(), @@ -533,7 +538,7 @@ static int path_handler(void *ctx, int fields, char **row) int Bvfs::_handle_path(void *ctx, int fields, char **row) { - if (fields == BVFS_DIR_RECORD) { + if (bvfs_is_dir(row)) { /* can have the same path 2 times */ if (strcmp(row[BVFS_Name], prev_dir)) { pm_strcpy(prev_dir, row[BVFS_Name]); @@ -550,7 +555,7 @@ void Bvfs::ls_special_dirs() { Dmsg1(dbglevel, "ls_special_dirs(%lld)\n", (uint64_t)pwd_id); char ed1[50], ed2[50]; - if (jobids.count == 0) { + if (*jobids == 0) { return; } if (!dir_filenameid) { @@ -563,23 +568,23 @@ void Bvfs::ls_special_dirs() POOL_MEM query; Mmsg(query, "((SELECT PPathId AS PathId, '..' AS Path " - "FROM brestore_pathhierarchy " + "FROM PathHierarchy " "WHERE PathId = %s) " "UNION " "(SELECT %s AS PathId, '.' AS Path))", edit_uint64(pwd_id, ed1), ed1); POOL_MEM query2; - Mmsg(query2, -"SELECT tmp.PathId, tmp.Path, JobId, LStat " + Mmsg(query2,// 1 2 3 4 5 6 +"SELECT 'D', tmp.PathId, 0, tmp.Path, JobId, LStat, FileId " "FROM %s AS tmp LEFT JOIN ( " // get attributes if any "SELECT File1.PathId AS PathId, File1.JobId AS JobId, " - "File1.LStat AS LStat FROM File AS File1 " + "File1.LStat AS LStat, File1.FileId AS FileId FROM File AS File1 " "WHERE File1.FilenameId = %s " "AND File1.JobId IN (%s)) AS listfile1 " "ON (tmp.PathId = listfile1.PathId) " "ORDER BY tmp.Path, JobId DESC ", - query.c_str(), edit_uint64(dir_filenameid, ed2), jobids.list); + query.c_str(), edit_uint64(dir_filenameid, ed2), jobids); Dmsg1(dbglevel_sql, "q=%s\n", query2.c_str()); db_sql_query(db, query2.c_str(), path_handler, this); @@ -590,7 +595,7 @@ bool Bvfs::ls_dirs() { Dmsg1(dbglevel, "ls_dirs(%lld)\n", (uint64_t)pwd_id); char ed1[50], ed2[50]; - if (jobids.count == 0) { + if (*jobids == 0) { return false; } @@ -614,36 +619,37 @@ bool Bvfs::ls_dirs() /* Then we get all the dir entries from File ... */ POOL_MEM query; Mmsg(query, -// 0 1 2 3 -"SELECT PathId, Path, JobId, LStat FROM ( " +// 0 1 2 3 4 5 6 +"SELECT 'D', PathId, 0, Path, JobId, LStat, FileId FROM ( " "SELECT Path1.PathId AS PathId, Path1.Path AS Path, " "lower(Path1.Path) AS lpath, " - "listfile1.JobId AS JobId, listfile1.LStat AS LStat " + "listfile1.JobId AS JobId, listfile1.LStat AS LStat, " + "listfile1.FileId AS FileId " "FROM ( " - "SELECT DISTINCT brestore_pathhierarchy1.PathId AS PathId " - "FROM brestore_pathhierarchy AS brestore_pathhierarchy1 " + "SELECT DISTINCT PathHierarchy1.PathId AS PathId " + "FROM PathHierarchy AS PathHierarchy1 " "JOIN Path AS Path2 " - "ON (brestore_pathhierarchy1.PathId = Path2.PathId) " - "JOIN brestore_pathvisibility AS brestore_pathvisibility1 " - "ON (brestore_pathhierarchy1.PathId = brestore_pathvisibility1.PathId) " - "WHERE brestore_pathhierarchy1.PPathId = %s " - "AND brestore_pathvisibility1.jobid IN (%s) " + "ON (PathHierarchy1.PathId = Path2.PathId) " + "JOIN PathVisibility AS PathVisibility1 " + "ON (PathHierarchy1.PathId = PathVisibility1.PathId) " + "WHERE PathHierarchy1.PPathId = %s " + "AND PathVisibility1.jobid IN (%s) " "%s " ") AS listpath1 " "JOIN Path AS Path1 ON (listpath1.PathId = Path1.PathId) " "LEFT JOIN ( " /* get attributes if any */ "SELECT File1.PathId AS PathId, File1.JobId AS JobId, " - "File1.LStat AS LStat FROM File AS File1 " + "File1.LStat AS LStat, File1.FileId AS FileId FROM File AS File1 " "WHERE File1.FilenameId = %s " "AND File1.JobId IN (%s)) AS listfile1 " "ON (listpath1.PathId = listfile1.PathId) " ") AS A ORDER BY 2,3 DESC LIMIT %d OFFSET %d", edit_uint64(pwd_id, ed1), - jobids.list, + jobids, filter.c_str(), edit_uint64(dir_filenameid, ed2), - jobids.list, + jobids, limit, offset); Dmsg1(dbglevel_sql, "q=%s\n", query.c_str()); @@ -661,7 +667,7 @@ bool Bvfs::ls_files() { Dmsg1(dbglevel, "ls_files(%lld)\n", (uint64_t)pwd_id); char ed1[50]; - if (jobids.count == 0) { + if (*jobids == 0) { return false; } @@ -673,10 +679,11 @@ bool Bvfs::ls_files() if (*pattern) { Mmsg(filter, " AND Filename.Name %s '%s' ", SQL_MATCH, pattern); } - + /* TODO: Use JobTDate instead of FileId to determine the latest version */ POOL_MEM query; - Mmsg(query, // 0 1 2 3 4 -"SELECT File.FilenameId, listfiles.Name, File.JobId, File.LStat, listfiles.id " + Mmsg(query, // 1 2 3 4 +"SELECT 'F', File.PathId, File.FilenameId, listfiles.Name, File.JobId, " + "File.LStat, listfiles.id " "FROM File, ( " "SELECT Filename.Name as Name, max(File.FileId) as id " "FROM File, Filename " @@ -690,7 +697,7 @@ bool Bvfs::ls_files() ") AS listfiles " "WHERE File.FileId = listfiles.id", edit_uint64(pwd_id, ed1), - jobids.list, + jobids, filter.c_str(), limit, offset); @@ -703,3 +710,197 @@ bool Bvfs::ls_files() return nb_record == limit; } + + +/* + * Return next Id from comma separated list + * + * Returns: + * 1 if next Id returned + * 0 if no more Ids are in list + * -1 there is an error + * TODO: merge with get_next_jobid_from_list() and get_next_dbid_from_list() + */ +static int get_next_id_from_list(char **p, int64_t *Id) +{ + const int maxlen = 30; + char id[maxlen+1]; + char *q = *p; + + id[0] = 0; + for (int i=0; i