2 Bacula(R) - The Network Backup Solution
4 Copyright (C) 2000-2017 Kern Sibbald
6 The original author of Bacula is Kern Sibbald, with contributions
7 from many others, a complete list can be found in the file AUTHORS.
9 You may use this file and others of this release according to the
10 license defined in the LICENSE file, which includes the Affero General
11 Public License, v3.0 ("AGPLv3") and some additional permissions and
12 terms pursuant to its AGPLv3 Section 7.
14 This notice must be preserved when any source code is
15 conveyed and/or propagated.
17 Bacula(R) is a registered trademark of Kern Sibbald.
22 #if HAVE_SQLITE3 || HAVE_MYSQL || HAVE_POSTGRESQL
23 #include "lib/htable.h"
26 #define can_access(x) (true)
29 extern int decode_stat(char *buf, struct stat *statp, int stat_size, int32_t *LinkFI);
31 #define dbglevel DT_BVFS|10
32 #define dbglevel_sql DT_SQL|15
34 static int result_handler(void *ctx, int fields, char **row)
37 Pmsg4(0, "%s\t%s\t%s\t%s\n",
38 row[0], row[1], row[2], row[3]);
39 } else if (fields == 5) {
40 Pmsg5(0, "%s\t%s\t%s\t%s\t%s\n",
41 row[0], row[1], row[2], row[3], row[4]);
42 } else if (fields == 6) {
43 Pmsg6(0, "%s\t%s\t%s\t%s\t%s\t%s\n",
44 row[0], row[1], row[2], row[3], row[4], row[5]);
45 } else if (fields == 7) {
46 Pmsg7(0, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
47 row[0], row[1], row[2], row[3], row[4], row[5], row[6]);
52 Bvfs::Bvfs(JCR *j, BDB *mdb)
56 db = mdb; /* need to inc ref count */
57 jobids = get_pool_memory(PM_NAME);
58 prev_dir = get_pool_memory(PM_NAME);
59 pattern = get_pool_memory(PM_NAME);
60 filename = get_pool_memory(PM_NAME);
61 tmp = get_pool_memory(PM_NAME);
62 escaped_list = get_pool_memory(PM_NAME);
63 *filename = *jobids = *prev_dir = *pattern = 0;
65 see_copies = see_all_versions = false;
69 list_entries = result_handler;
72 job_acl = client_acl = pool_acl = fileset_acl = NULL;
76 dir_filenameid = 0; /* special FilenameId where Name='' */
80 free_pool_memory(jobids);
81 free_pool_memory(pattern);
82 free_pool_memory(prev_dir);
83 free_pool_memory(filename);
84 free_pool_memory(tmp);
85 free_pool_memory(escaped_list);
96 char *Bvfs::escape_list(alist *lst)
101 /* List is empty, reject everything */
102 if (!lst || lst->size() == 0) {
103 Mmsg(escaped_list, "''");
110 foreach_alist(elt, lst) {
114 tmp = check_pool_memory_size(tmp, 2 * len + 2 + 2);
117 db->bdb_escape_string(jcr, tmp + 1 , elt, len);
121 pm_strcat(escaped_list, ",");
124 pm_strcat(escaped_list, tmp);
130 /* Returns the number of jobids in the result */
131 int Bvfs::filter_jobid()
137 /* No ACL, no username, no check */
138 if (!job_acl && !fileset_acl && !client_acl && !pool_acl && !username) {
139 Dmsg0(dbglevel_sql, "No ACL\n");
140 /* Just count the number of items in the list */
141 int nb = (*jobids != 0) ? 1 : 0;
142 for (char *p=jobids; *p ; p++) {
151 Mmsg(sub_where, " AND Job.Name IN (%s) ", escape_list(job_acl));
155 Mmsg(query, " AND FileSet.FileSet IN (%s) ", escape_list(fileset_acl));
156 pm_strcat(sub_where, query.c_str());
157 pm_strcat(sub_join, " JOIN FileSet USING (FileSetId) ");
161 Mmsg(query, " AND Client.Name IN (%s) ", escape_list(client_acl));
162 pm_strcat(sub_where, query.c_str());
166 Mmsg(query, " AND Pool.Name IN (%s) ", escape_list(pool_acl));
167 pm_strcat(sub_where, query.c_str());
168 pm_strcat(sub_join, " JOIN Pool USING (PoolId) ");
172 /* Query used by Bweb to filter clients, activated when using
176 "SELECT DISTINCT JobId FROM Job JOIN Client USING (ClientId) %s "
177 "JOIN (SELECT ClientId FROM client_group_member "
178 "JOIN client_group USING (client_group_id) "
179 "JOIN bweb_client_group_acl USING (client_group_id) "
180 "JOIN bweb_user USING (userid) "
181 "WHERE bweb_user.username = '%s' "
182 ") AS filter USING (ClientId) "
183 " WHERE JobId IN (%s) %s",
184 sub_join.c_str(), username, jobids, sub_where.c_str());
188 "SELECT DISTINCT JobId FROM Job JOIN Client USING (ClientId) %s "
189 " WHERE JobId IN (%s) %s",
190 sub_join.c_str(), jobids, sub_where.c_str());
194 Dmsg1(dbglevel_sql, "q=%s\n", query.c_str());
195 db->bdb_sql_query(query.c_str(), db_list_handler, &ctx);
196 pm_strcpy(jobids, ctx.list);
200 /* Return the number of jobids after the filter */
201 int Bvfs::set_jobid(JobId_t id)
203 Mmsg(jobids, "%lld", (uint64_t)id);
204 return filter_jobid();
207 /* Return the number of jobids after the filter */
208 int Bvfs::set_jobids(char *ids)
210 pm_strcpy(jobids, ids);
211 return filter_jobid();
215 * TODO: Find a way to let the user choose how he wants to display
216 * files and directories
221 * Working Object to store PathId already seen (avoid
222 * database queries), equivalent to %cache_ppathid in perl
234 htable *cache_ppathid;
239 cache_ppathid = (htable *)malloc(sizeof(htable));
240 cache_ppathid->init(&link, &link, NITEMS);
242 nodes = (hlink *) malloc(max_node * sizeof (hlink));
244 table_node = New(alist(5, owned_by_alist));
245 table_node->append(nodes);
249 if (++nb_node >= max_node) {
251 nodes = (hlink *)malloc(max_node * sizeof(hlink));
252 table_node->append(nodes);
254 return nodes + nb_node;
257 bool lookup(char *pathid) {
258 bool ret = cache_ppathid->lookup(pathid) != NULL;
262 void insert(char *pathid) {
263 hlink *h = get_hlink();
264 cache_ppathid->insert(pathid, h);
268 cache_ppathid->destroy();
273 pathid_cache(const pathid_cache &); /* prohibit pass by value */
274 pathid_cache &operator= (const pathid_cache &);/* prohibit class assignment*/
277 /* Return the parent_dir with the trailing / (update the given string)
278 * TODO: see in the rest of bacula if we don't have already this function
284 char *bvfs_parent_dir(char *path)
287 int len = strlen(path) - 1;
289 /* windows directory / */
290 if (len == 2 && B_ISALPHA(path[0])
298 if (len >= 0 && path[len] == '/') { /* if directory, skip last / */
304 while (p > path && !IsPathSeparator(*p)) {
312 /* Return the basename of the with the trailing /
313 * TODO: see in the rest of bacula if we don't have
314 * this function already
316 char *bvfs_basename_dir(char *path)
319 int len = strlen(path) - 1;
321 if (path[len] == '/') { /* if directory, skip last / */
327 while (p > path && !IsPathSeparator(*p)) {
331 p++; /* skip first / */
337 static void build_path_hierarchy(JCR *jcr, BDB *mdb,
338 pathid_cache &ppathid_cache,
339 char *org_pathid, char *path)
341 Dmsg1(dbglevel, "build_path_hierarchy(%s)\n", path);
344 char *bkp = mdb->path;
345 strncpy(pathid, org_pathid, sizeof(pathid));
347 /* Does the ppathid exist for this ? we use a memory cache... In order to
348 * avoid the full loop, we consider that if a dir is allready in the
349 * PathHierarchy table, then there is no need to calculate all the
352 while (path && *path)
354 if (!ppathid_cache.lookup(pathid))
357 "SELECT PPathId FROM PathHierarchy WHERE PathId = %s",
360 if (!mdb->QueryDB(jcr, mdb->cmd)) {
361 goto bail_out; /* Query failed, just leave */
364 /* Do we have a result ? */
365 if (mdb->sql_num_rows() > 0) {
366 ppathid_cache.insert(pathid);
367 /* This dir was in the db ...
368 * It means we can leave, the tree has allready been built for
373 /* search or create parent PathId in Path table */
374 mdb->path = bvfs_parent_dir(path);
375 mdb->pnl = strlen(mdb->path);
376 if (!mdb->bdb_create_path_record(jcr, &parent)) {
379 ppathid_cache.insert(pathid);
382 "INSERT INTO PathHierarchy (PathId, PPathId) "
384 pathid, (uint64_t) parent.PathId);
386 if (!mdb->InsertDB(jcr, mdb->cmd)) {
387 goto bail_out; /* Can't insert the record, just leave */
390 edit_uint64(parent.PathId, pathid);
391 path = mdb->path; /* already done */
394 /* It's already in the cache. We can leave, no time to waste here,
395 * all the parent dirs have allready been done
407 * Internal function to update path_hierarchy cache with a shared pathid cache
411 static int update_path_hierarchy_cache(JCR *jcr,
413 pathid_cache &ppathid_cache,
416 Dmsg0(dbglevel, "update_path_hierarchy_cache()\n");
420 edit_uint64(JobId, jobid);
424 /* We don't really want to harm users with spurious messages,
425 * everything is handled by transaction
427 mdb->set_use_fatal_jmsg(false);
429 mdb->bdb_start_transaction(jcr);
431 Mmsg(mdb->cmd, "SELECT 1 FROM Job WHERE JobId = %s AND HasCache=1", jobid);
433 if (!mdb->QueryDB(jcr, mdb->cmd) || mdb->sql_num_rows() > 0) {
434 Dmsg1(dbglevel, "already computed %d\n", (uint32_t)JobId );
439 /* Inserting path records for JobId */
440 Mmsg(mdb->cmd, "INSERT INTO PathVisibility (PathId, JobId) "
441 "SELECT DISTINCT PathId, JobId "
442 "FROM (SELECT PathId, JobId FROM File WHERE JobId = %s "
444 "SELECT PathId, BaseFiles.JobId "
445 "FROM BaseFiles JOIN File AS F USING (FileId) "
446 "WHERE BaseFiles.JobId = %s) AS B",
449 if (!mdb->QueryDB(jcr, mdb->cmd)) {
450 Dmsg1(dbglevel, "Can't fill PathVisibility %d\n", (uint32_t)JobId );
454 /* Now we have to do the directory recursion stuff to determine missing
455 * visibility We try to avoid recursion, to be as fast as possible We also
456 * only work on not allready hierarchised directories...
459 "SELECT PathVisibility.PathId, Path "
460 "FROM PathVisibility "
461 "JOIN Path ON( PathVisibility.PathId = Path.PathId) "
462 "LEFT JOIN PathHierarchy "
463 "ON (PathVisibility.PathId = PathHierarchy.PathId) "
464 "WHERE PathVisibility.JobId = %s "
465 "AND PathHierarchy.PathId IS NULL "
466 "ORDER BY Path", jobid);
467 Dmsg1(dbglevel_sql, "q=%s\n", mdb->cmd);
469 if (!mdb->QueryDB(jcr, mdb->cmd)) {
470 Dmsg1(dbglevel, "Can't get new Path %d\n", (uint32_t)JobId );
474 /* TODO: I need to reuse the DB connection without emptying the result
475 * So, now i'm copying the result in memory to be able to query the
476 * catalog descriptor again.
478 num = mdb->sql_num_rows();
480 char **result = (char **)malloc (num * 2 * sizeof(char *));
484 while((row = mdb->sql_fetch_row())) {
485 result[i++] = bstrdup(row[0]);
486 result[i++] = bstrdup(row[1]);
491 build_path_hierarchy(jcr, mdb, ppathid_cache, result[i], result[i+1]);
499 if (mdb->bdb_get_type_index() == SQL_TYPE_SQLITE3) {
501 "INSERT INTO PathVisibility (PathId, JobId) "
502 "SELECT DISTINCT h.PPathId AS PathId, %s "
503 "FROM PathHierarchy AS h "
504 "WHERE h.PathId IN (SELECT PathId FROM PathVisibility WHERE JobId=%s) "
505 "AND h.PPathId NOT IN (SELECT PathId FROM PathVisibility WHERE JobId=%s)",
506 jobid, jobid, jobid );
508 } else if (mdb->bdb_get_type_index() == SQL_TYPE_MYSQL) {
510 "INSERT INTO PathVisibility (PathId, JobId) "
511 "SELECT a.PathId,%s "
513 "SELECT DISTINCT h.PPathId AS PathId "
514 "FROM PathHierarchy AS h "
515 "JOIN PathVisibility AS p ON (h.PathId=p.PathId) "
516 "WHERE p.JobId=%s) AS a "
517 "LEFT JOIN PathVisibility AS b ON (b.JobId=%s and a.PathId = b.PathId) "
518 "WHERE b.PathId IS NULL", jobid, jobid, jobid);
522 "INSERT INTO PathVisibility (PathId, JobId) "
523 "SELECT a.PathId,%s "
525 "SELECT DISTINCT h.PPathId AS PathId "
526 "FROM PathHierarchy AS h "
527 "JOIN PathVisibility AS p ON (h.PathId=p.PathId) "
528 "WHERE p.JobId=%s) AS a LEFT JOIN "
530 "FROM PathVisibility "
531 "WHERE JobId=%s) AS b ON (a.PathId = b.PathId) "
532 "WHERE b.PathId IS NULL", jobid, jobid, jobid);
536 ret = mdb->QueryDB(jcr, mdb->cmd);
537 } while (ret && mdb->sql_affected_rows() > 0);
539 Mmsg(mdb->cmd, "UPDATE Job SET HasCache=1 WHERE JobId=%s", jobid);
540 ret = mdb->UpdateDB(jcr, mdb->cmd, false);
543 mdb->bdb_end_transaction(jcr);
546 Mmsg(mdb->cmd, "SELECT HasCache FROM Job WHERE JobId=%s", jobid);
547 mdb->bdb_sql_query(mdb->cmd, db_int_handler, &ret);
550 /* Enable back the FATAL message if something is wrong */
551 mdb->set_use_fatal_jmsg(true);
557 /* Compute the cache for the bfileview compoment */
558 void Bvfs::fv_update_cache()
561 int64_t size=0, count=0;
563 Dmsg0(dbglevel, "fv_update_cache()\n");
566 return; /* Nothing to build */
570 /* We don't really want to harm users with spurious messages,
571 * everything is handled by transaction
573 db->set_use_fatal_jmsg(false);
575 db->bdb_start_transaction(jcr);
579 fv_compute_size_and_count(pathid, &size, &count);
581 db->bdb_end_transaction(jcr);
583 /* Enable back the FATAL message if something is wrong */
584 db->set_use_fatal_jmsg(true);
590 * Find an store the filename descriptor for empty directories Filename.Name=''
592 DBId_t Bvfs::get_dir_filenameid()
595 if (dir_filenameid) {
596 return dir_filenameid;
598 Mmsg(db->cmd, "SELECT FilenameId FROM Filename WHERE Name = ''");
599 db_sql_query(db, db->cmd, db_int_handler, &id);
601 return dir_filenameid;
604 /* Not yet working */
605 void Bvfs::fv_get_big_files(int64_t pathid, int64_t min_size, int32_t limit)
608 "SELECT FilenameId AS filenameid, Name AS name, size "
610 "SELECT FilenameId, base64_decode_lstat(8,LStat) AS size "
612 "WHERE PathId = %lld "
614 ") AS S INNER JOIN Filename USING (FilenameId) "
615 "WHERE S.size > %lld "
616 "ORDER BY S.size DESC "
617 "LIMIT %d ", pathid, jobids, min_size, limit);
621 /* Get the current path size and files count */
622 void Bvfs::fv_get_current_size_and_count(int64_t pathid, int64_t *size, int64_t *count)
629 "SELECT Size AS size, Files AS files "
630 " FROM PathVisibility "
631 " WHERE PathId = %lld "
632 " AND JobId = %s ", pathid, jobids);
634 if (!db->QueryDB(jcr, db->cmd)) {
638 if ((row = db->sql_fetch_row())) {
639 *size = str_to_int64(row[0]);
640 *count = str_to_int64(row[1]);
644 /* Compute for the current path the size and files count */
645 void Bvfs::fv_get_size_and_count(int64_t pathid, int64_t *size, int64_t *count)
652 "SELECT sum(base64_decode_lstat(8,LStat)) AS size, count(1) AS files "
654 " WHERE PathId = %lld "
655 " AND JobId = %s ", pathid, jobids);
657 if (!db->QueryDB(jcr, db->cmd)) {
661 if ((row = db->sql_fetch_row())) {
662 *size = str_to_int64(row[0]);
663 *count = str_to_int64(row[1]);
667 void Bvfs::fv_compute_size_and_count(int64_t pathid, int64_t *size, int64_t *count)
669 Dmsg1(dbglevel, "fv_compute_size_and_count(%lld)\n", pathid);
671 fv_get_current_size_and_count(pathid, size, count);
676 /* Update stats for the current directory */
677 fv_get_size_and_count(pathid, size, count);
679 /* Update stats for all sub directories */
682 " FROM PathVisibility "
683 " INNER JOIN PathHierarchy USING (PathId) "
684 " WHERE PPathId = %lld "
685 " AND JobId = %s ", pathid, jobids);
687 db->QueryDB(jcr, db->cmd);
688 int num = db->sql_num_rows();
691 int64_t *result = (int64_t *)malloc (num * sizeof(int64_t));
695 while((row = db->sql_fetch_row())) {
696 result[i++] = str_to_int64(row[0]); /* PathId */
702 fv_compute_size_and_count(result[i], &s, &c);
712 fv_update_size_and_count(pathid, *size, *count);
715 void Bvfs::fv_update_size_and_count(int64_t pathid, int64_t size, int64_t count)
718 "UPDATE PathVisibility SET Files = %lld, Size = %lld "
720 " AND PathId = %lld ", count, size, jobids, pathid);
722 db->UpdateDB(jcr, db->cmd, false);
725 void bvfs_update_cache(JCR *jcr, BDB *mdb)
728 db_list_ctx jobids_list;
733 /* TODO: Remove this code when updating make_bacula_table script */
734 Mmsg(mdb->cmd, "SELECT 1 FROM Job WHERE HasCache<>2 LIMIT 1");
735 if (!mdb->QueryDB(jcr, mdb->cmd)) {
736 Dmsg0(dbglevel, "Creating cache table\n");
737 Mmsg(mdb->cmd, "ALTER TABLE Job ADD HasCache int DEFAULT 0");
738 mdb->QueryDB(jcr, mdb->cmd);
741 "CREATE TABLE PathHierarchy ( "
742 "PathId integer NOT NULL, "
743 "PPathId integer NOT NULL, "
744 "CONSTRAINT pathhierarchy_pkey "
745 "PRIMARY KEY (PathId))");
746 mdb->QueryDB(jcr, mdb->cmd);
749 "CREATE INDEX pathhierarchy_ppathid "
750 "ON PathHierarchy (PPathId)");
751 mdb->QueryDB(jcr, mdb->cmd);
754 "CREATE TABLE PathVisibility ("
755 "PathId integer NOT NULL, "
756 "JobId integer NOT NULL, "
757 "Size int8 DEFAULT 0, "
758 "Files int4 DEFAULT 0, "
759 "CONSTRAINT pathvisibility_pkey "
760 "PRIMARY KEY (JobId, PathId))");
761 mdb->QueryDB(jcr, mdb->cmd);
764 "CREATE INDEX pathvisibility_jobid "
765 "ON PathVisibility (JobId)");
766 mdb->QueryDB(jcr, mdb->cmd);
772 "SELECT JobId from Job "
773 "WHERE HasCache = 0 "
774 "AND Type IN ('B') AND JobStatus IN ('T', 'f', 'A') "
777 mdb->bdb_sql_query(mdb->cmd, db_list_handler, &jobids_list);
779 bvfs_update_path_hierarchy_cache(jcr, mdb, jobids_list.list);
781 mdb->bdb_start_transaction(jcr);
782 Dmsg0(dbglevel, "Cleaning pathvisibility\n");
784 "DELETE FROM PathVisibility "
786 "(SELECT 1 FROM Job WHERE JobId=PathVisibility.JobId)");
787 nb = mdb->DeleteDB(jcr, mdb->cmd);
788 Dmsg1(dbglevel, "Affected row(s) = %d\n", nb);
790 mdb->bdb_end_transaction(jcr);
795 * Update the bvfs cache for given jobids (1,2,3,4)
798 bvfs_update_path_hierarchy_cache(JCR *jcr, BDB *mdb, char *jobids)
800 pathid_cache ppathid_cache;
806 int stat = get_next_jobid_from_list(&p, &JobId);
814 Dmsg1(dbglevel, "Updating cache for %lld\n", (uint64_t)JobId);
815 if (!update_path_hierarchy_cache(jcr, mdb, ppathid_cache, JobId)) {
823 * Update the bvfs fileview for given jobids
826 bvfs_update_fv_cache(JCR *jcr, BDB *mdb, char *jobids)
833 int stat = get_next_jobid_from_list(&p, &JobId);
841 Dmsg1(dbglevel, "Trying to create cache for %lld\n", (int64_t)JobId);
843 bvfs.set_jobid(JobId);
844 bvfs.fv_update_cache();
849 * Update the bvfs cache for current jobids
851 void Bvfs::update_cache()
853 bvfs_update_path_hierarchy_cache(jcr, db, jobids);
857 bool Bvfs::ch_dir(DBId_t pathid)
866 /* Change the current directory, returns true if the path exists */
867 bool Bvfs::ch_dir(const char *path)
870 pm_strcpy(db->path, path);
871 db->pnl = strlen(db->path);
872 ch_dir(db->bdb_get_path_record(jcr));
878 * Get all file versions for a specified client
879 * TODO: Handle basejobs using different client
881 void Bvfs::get_all_file_versions(DBId_t pathid, FileId_t fnid, const char *client)
883 Dmsg3(dbglevel, "get_all_file_versions(%lld, %lld, %s)\n", (uint64_t)pathid,
884 (uint64_t)fnid, client);
885 char ed1[50], ed2[50];
888 Mmsg(q, " AND Job.Type IN ('C', 'B') ");
890 Mmsg(q, " AND Job.Type = 'B' ");
895 Mmsg(query,// 1 2 3 4
896 "SELECT 'V', File.PathId, File.FilenameId, 0, File.JobId, "
898 "File.LStat, File.FileId, File.Md5, "
900 "Media.VolumeName, Media.InChanger "
901 "FROM File, Job, Client, JobMedia, Media "
902 "WHERE File.FilenameId = %s "
903 "AND File.PathId=%s "
904 "AND File.JobId = Job.JobId "
905 "AND Job.JobId = JobMedia.JobId "
906 "AND File.FileIndex >= JobMedia.FirstIndex "
907 "AND File.FileIndex <= JobMedia.LastIndex "
908 "AND JobMedia.MediaId = Media.MediaId "
909 "AND Job.ClientId = Client.ClientId "
910 "AND Client.Name = '%s' "
911 "%s ORDER BY FileId LIMIT %d OFFSET %d"
912 ,edit_uint64(fnid, ed1), edit_uint64(pathid, ed2), client, q.c_str(),
914 Dmsg1(dbglevel_sql, "q=%s\n", query.c_str());
915 db->bdb_sql_query(query.c_str(), list_entries, user_data);
919 * Get all file versions for a specified client
920 * TODO: Handle basejobs using different client
922 bool Bvfs::get_delta(FileId_t fileid)
924 Dmsg1(dbglevel, "get_delta(%lld)\n", (uint64_t)fileid);
934 /* Check if some FileId have DeltaSeq > 0
935 * Foreach of them we need to get the accurate_job list, and compute
936 * what are dependencies
939 "SELECT F.JobId, FN.Name, F.PathId, F.DeltaSeq "
940 "FROM File AS F, Filename AS FN WHERE FileId = %lld "
941 "AND FN.FilenameId = F.FilenameId AND DeltaSeq > 0", fileid);
943 if (!db->QueryDB(jcr, query.c_str())) {
944 Dmsg1(dbglevel_sql, "Can't execute query=%s\n", query.c_str());
948 /* TODO: Use an other DB connection can avoid to copy the result of the
949 * previous query into a temporary buffer
951 num = db->sql_num_rows();
952 Dmsg2(dbglevel, "Found %d Delta parts q=%s\n",
955 if (num > 0 && (row = db->sql_fetch_row())) {
958 memset(&jr, 0, sizeof(jr));
959 memset(&jr2, 0, sizeof(jr2));
961 fn = bstrdup(row[1]); /* Filename */
962 int64_t jid = str_to_int64(row[0]); /* JobId */
963 int64_t pid = str_to_int64(row[2]); /* PathId */
965 /* Need to limit the query to StartTime, Client/Fileset */
967 if (!db->bdb_get_job_record(jcr, &jr2)) {
968 Dmsg1(0, "Unable to get job record for jobid %d\n", jid);
973 jr.ClientId = jr2.ClientId;
974 jr.FileSetId = jr2.FileSetId;
975 jr.JobLevel = L_INCREMENTAL;
976 jr.StartTime = jr2.StartTime;
978 /* Get accurate jobid list */
979 if (!db->bdb_get_accurate_jobids(jcr, &jr, &lst)) {
980 Dmsg1(0, "Unable to get Accurate list for jobid %d\n", jid);
984 /* Escape filename */
985 db->fnl = strlen(fn);
986 db->esc_name = check_pool_memory_size(db->esc_name, 2*db->fnl+2);
987 db->bdb_escape_string(jcr, db->esc_name, fn, db->fnl);
989 edit_int64(pid, ed1); /* pathid */
991 int id=db->bdb_get_type_index();
992 Mmsg(query, bvfs_select_delta_version_with_basejob_and_delta[id],
993 lst.list, db->esc_name, ed1,
994 lst.list, db->esc_name, ed1,
999 "SELECT 'd', PathId, 0, JobId, LStat, FileId, DeltaSeq, JobTDate"
1001 "ORDER BY DeltaSeq ASC",
1004 Dmsg1(dbglevel_sql, "q=%s\n", db->cmd);
1006 if (!db->bdb_sql_query(db->cmd, list_entries, user_data)) {
1007 Dmsg1(dbglevel_sql, "Can't exec q=%s\n", db->cmd);
1021 * Get all volumes for a specific file
1023 void Bvfs::get_volumes(FileId_t fileid)
1025 Dmsg1(dbglevel, "get_volumes(%lld)\n", (uint64_t)fileid);
1032 "SELECT DISTINCT 'L',0,0,0,0,0,0, Media.VolumeName, Media.InChanger "
1033 "FROM File JOIN JobMedia USING (JobId) JOIN Media USING (MediaId) "
1034 "WHERE File.FileId = %s "
1035 "AND File.FileIndex >= JobMedia.FirstIndex "
1036 "AND File.FileIndex <= JobMedia.LastIndex "
1037 " LIMIT %d OFFSET %d"
1038 ,edit_uint64(fileid, ed1), limit, offset);
1039 Dmsg1(dbglevel_sql, "q=%s\n", query.c_str());
1040 db->bdb_sql_query(query.c_str(), list_entries, user_data);
1043 DBId_t Bvfs::get_root()
1048 p = db->bdb_get_path_record(jcr);
1053 static int path_handler(void *ctx, int fields, char **row)
1055 Bvfs *fs = (Bvfs *) ctx;
1056 return fs->_handle_path(ctx, fields, row);
1059 int Bvfs::_handle_path(void *ctx, int fields, char **row)
1061 if (bvfs_is_dir(row)) {
1062 /* can have the same path 2 times */
1063 if (strcmp(row[BVFS_PathId], prev_dir)) {
1064 pm_strcpy(prev_dir, row[BVFS_PathId]);
1065 if (strcmp(NPRTB(row[BVFS_FileIndex]), "0") == 0 &&
1066 strcmp(NPRTB(row[BVFS_FileId]), "0") != 0)
1068 /* The directory was probably deleted */
1071 return list_entries(user_data, fields, row);
1078 * Retrieve . and .. information
1080 void Bvfs::ls_special_dirs()
1082 Dmsg1(dbglevel, "ls_special_dirs(%lld)\n", (uint64_t)pwd_id);
1083 char ed1[50], ed2[50];
1087 if (!dir_filenameid) {
1088 get_dir_filenameid();
1091 /* Will fetch directories */
1096 "(SELECT PathHierarchy.PPathId AS PathId, '..' AS Path "
1097 "FROM PathHierarchy JOIN PathVisibility USING (PathId) "
1098 "WHERE PathHierarchy.PathId = %s "
1099 "AND PathVisibility.JobId IN (%s) "
1101 "SELECT %s AS PathId, '.' AS Path)",
1102 edit_uint64(pwd_id, ed1), jobids, ed1);
1105 Mmsg(query2,// 1 2 3 4 5 6
1106 "SELECT 'D', tmp.PathId, 0, tmp.Path, JobId, LStat, FileId, FileIndex "
1107 "FROM %s AS tmp LEFT JOIN ( " // get attributes if any
1108 "SELECT File1.PathId AS PathId, File1.JobId AS JobId, "
1109 "File1.LStat AS LStat, File1.FileId AS FileId, "
1110 "File1.FileIndex AS FileIndex, "
1111 "Job1.JobTDate AS JobTDate "
1112 "FROM File AS File1 JOIN Job AS Job1 USING (JobId)"
1113 "WHERE File1.FilenameId = %s "
1114 "AND File1.JobId IN (%s)) AS listfile1 "
1115 "ON (tmp.PathId = listfile1.PathId) "
1116 "ORDER BY tmp.Path, JobTDate DESC ",
1117 query.c_str(), edit_uint64(dir_filenameid, ed2), jobids);
1119 Dmsg1(dbglevel_sql, "q=%s\n", query2.c_str());
1120 db->bdb_sql_query(query2.c_str(), path_handler, this);
1123 /* Returns true if we have dirs to read */
1124 bool Bvfs::ls_dirs()
1126 Dmsg1(dbglevel, "ls_dirs(%lld)\n", (uint64_t)pwd_id);
1127 char ed1[50], ed2[50];
1135 Mmsg(filter, " AND Path2.Path %s '%s' ",
1136 match_query[db->bdb_get_type_index()], pattern);
1140 if (!dir_filenameid) {
1141 get_dir_filenameid();
1144 /* the sql query displays same directory multiple time, take the first one */
1147 /* Let's retrieve the list of the visible dirs in this dir ...
1148 * First, I need the empty filenameid to locate efficiently
1149 * the dirs in the file table
1150 * my $dir_filenameid = $self->get_dir_filenameid();
1152 /* Then we get all the dir entries from File ... */
1155 "SELECT 'D', PathId, 0, Path, JobId, LStat, FileId, FileIndex FROM ( "
1156 "SELECT Path1.PathId AS PathId, Path1.Path AS Path, "
1157 "lower(Path1.Path) AS lpath, "
1158 "listfile1.JobId AS JobId, listfile1.LStat AS LStat, "
1159 "listfile1.FileId AS FileId, "
1160 "listfile1.JobTDate AS JobTDate, "
1161 "listfile1.FileIndex AS FileIndex "
1163 "SELECT DISTINCT PathHierarchy1.PathId AS PathId "
1164 "FROM PathHierarchy AS PathHierarchy1 "
1165 "JOIN Path AS Path2 "
1166 "ON (PathHierarchy1.PathId = Path2.PathId) "
1167 "JOIN PathVisibility AS PathVisibility1 "
1168 "ON (PathHierarchy1.PathId = PathVisibility1.PathId) "
1169 "WHERE PathHierarchy1.PPathId = %s "
1170 "AND PathVisibility1.JobId IN (%s) "
1173 "JOIN Path AS Path1 ON (listpath1.PathId = Path1.PathId) "
1175 "LEFT JOIN ( " /* get attributes if any */
1176 "SELECT File1.PathId AS PathId, File1.JobId AS JobId, "
1177 "File1.LStat AS LStat, File1.FileId AS FileId, "
1178 "File1.FileIndex, Job1.JobTDate AS JobTDate "
1179 "FROM File AS File1 JOIN Job AS Job1 USING (JobId) "
1180 "WHERE File1.FilenameId = %s "
1181 "AND File1.JobId IN (%s)) AS listfile1 "
1182 "ON (listpath1.PathId = listfile1.PathId) "
1183 ") AS A ORDER BY Path,JobTDate DESC LIMIT %d OFFSET %d",
1184 edit_uint64(pwd_id, ed1),
1187 edit_uint64(dir_filenameid, ed2),
1191 Dmsg1(dbglevel_sql, "q=%s\n", query.c_str());
1194 db->bdb_sql_query(query.c_str(), path_handler, this);
1195 nb_record = db->sql_num_rows();
1198 return nb_record == limit;
1201 void build_ls_files_query(BDB *db, POOL_MEM &query,
1202 const char *JobId, const char *PathId,
1203 const char *filter, int64_t limit, int64_t offset)
1205 if (db->bdb_get_type_index() == SQL_TYPE_POSTGRESQL) {
1206 Mmsg(query, sql_bvfs_list_files[db->bdb_get_type_index()],
1207 JobId, PathId, JobId, PathId,
1208 filter, limit, offset);
1210 Mmsg(query, sql_bvfs_list_files[db->bdb_get_type_index()],
1211 JobId, PathId, JobId, PathId,
1212 limit, offset, filter, JobId, JobId);
1216 /* Returns true if we have files to read */
1217 bool Bvfs::ls_files()
1223 Dmsg1(dbglevel, "ls_files(%lld)\n", (uint64_t)pwd_id);
1232 edit_uint64(pwd_id, pathid);
1234 Mmsg(filter, " AND Filename.Name %s '%s' ",
1235 match_query[db_get_type_index(db)], pattern);
1237 } else if (*filename) {
1238 Mmsg(filter, " AND Filename.Name = '%s' ", filename);
1241 build_ls_files_query(db, query,
1242 jobids, pathid, filter.c_str(),
1245 Dmsg1(dbglevel_sql, "q=%s\n", query.c_str());
1248 db->bdb_sql_query(query.c_str(), list_entries, user_data);
1249 nb_record = db->sql_num_rows();
1252 return nb_record == limit;
1257 * Return next Id from comma separated list
1260 * 1 if next Id returned
1261 * 0 if no more Ids are in list
1262 * -1 there is an error
1263 * TODO: merge with get_next_jobid_from_list() and get_next_dbid_from_list()
1265 static int get_next_id_from_list(char **p, int64_t *Id)
1267 const int maxlen = 30;
1272 for (int i=0; i<maxlen; i++) {
1275 } else if (*q == ',') {
1284 } else if (!is_a_number(id)) {
1285 return -1; /* error */
1288 *Id = str_to_int64(id);
1292 static int get_path_handler(void *ctx, int fields, char **row)
1294 POOL_MEM *buf = (POOL_MEM *) ctx;
1295 pm_strcpy(*buf, row[0]);
1299 static bool check_temp(char *output_table)
1301 if (output_table[0] == 'b' &&
1302 output_table[1] == '2' &&
1303 is_an_integer(output_table + 2))
1310 void Bvfs::clear_cache()
1312 db->bdb_sql_query("BEGIN", NULL, NULL);
1313 db->bdb_sql_query("UPDATE Job SET HasCache=0", NULL, NULL);
1314 db->bdb_sql_query("TRUNCATE PathHierarchy", NULL, NULL);
1315 db->bdb_sql_query("TRUNCATE PathVisibility", NULL, NULL);
1316 db->bdb_sql_query("COMMIT", NULL, NULL);
1319 bool Bvfs::drop_restore_list(char *output_table)
1322 if (check_temp(output_table)) {
1323 Mmsg(query, "DROP TABLE %s", output_table);
1324 db->bdb_sql_query(query.c_str(), NULL, NULL);
1330 bool Bvfs::compute_restore_list(char *fileid, char *dirid, char *hardlink,
1335 int64_t id, jobid, prev_jobid;
1340 if ((*fileid && !is_a_number_list(fileid)) ||
1341 (*dirid && !is_a_number_list(dirid)) ||
1342 (*hardlink && !is_a_number_list(hardlink))||
1343 (!*hardlink && !*fileid && !*dirid && !*hardlink))
1345 Dmsg0(dbglevel, "ERROR: One or more of FileId, DirId or HardLink is not given or not a number.\n");
1348 if (!check_temp(output_table)) {
1349 Dmsg0(dbglevel, "ERROR: Wrong format for table name (in path field).\n");
1355 /* Cleanup old tables first */
1356 Mmsg(query, "DROP TABLE btemp%s", output_table);
1357 db->bdb_sql_query(query.c_str());
1359 Mmsg(query, "DROP TABLE %s", output_table);
1360 db->bdb_sql_query(query.c_str());
1362 Mmsg(query, "CREATE TABLE btemp%s AS ", output_table);
1364 if (*fileid) { /* Select files with their direct id */
1366 Mmsg(tmp,"SELECT Job.JobId, JobTDate, FileIndex, FilenameId, "
1368 "FROM File,Job WHERE Job.JobId=File.Jobid "
1369 "AND FileId IN (%s)",
1371 pm_strcat(query, tmp.c_str());
1374 /* Add a directory content */
1375 while (get_next_id_from_list(&dirid, &id) == 1) {
1376 Mmsg(tmp, "SELECT Path FROM Path WHERE PathId=%lld", id);
1378 if (!db->bdb_sql_query(tmp.c_str(), get_path_handler, (void *)&tmp2)) {
1379 Dmsg3(dbglevel, "ERROR: Path not found %lld q=%s s=%s\n",
1380 id, tmp.c_str(), tmp2.c_str());
1384 if (!strcmp(tmp2.c_str(), "")) { /* path not found */
1385 Dmsg3(dbglevel, "ERROR: Path not found %lld q=%s s=%s\n",
1386 id, tmp.c_str(), tmp2.c_str());
1389 /* escape % and _ for LIKE search */
1390 tmp.check_size((strlen(tmp2.c_str())+1) * 2);
1391 char *p = tmp.c_str();
1392 for (char *s = tmp2.c_str(); *s ; s++) {
1393 if (*s == '%' || *s == '_' || *s == '\\') {
1403 size_t len = strlen(tmp.c_str());
1404 tmp2.check_size((len+1) * 2);
1405 db->bdb_escape_string(jcr, tmp2.c_str(), tmp.c_str(), len);
1408 query.strcat(" UNION ");
1411 Mmsg(tmp, "SELECT Job.JobId, JobTDate, File.FileIndex, File.FilenameId, "
1412 "File.PathId, FileId "
1413 "FROM Path JOIN File USING (PathId) JOIN Job USING (JobId) "
1414 "WHERE Path.Path LIKE '%s' AND File.JobId IN (%s) ",
1415 tmp2.c_str(), jobids);
1416 query.strcat(tmp.c_str());
1419 query.strcat(" UNION ");
1421 /* A directory can have files from a BaseJob */
1422 Mmsg(tmp, "SELECT File.JobId, JobTDate, BaseFiles.FileIndex, "
1423 "File.FilenameId, File.PathId, BaseFiles.FileId "
1425 "JOIN File USING (FileId) "
1426 "JOIN Job ON (BaseFiles.JobId = Job.JobId) "
1427 "JOIN Path USING (PathId) "
1428 "WHERE Path.Path LIKE '%s' AND BaseFiles.JobId IN (%s) ",
1429 tmp2.c_str(), jobids);
1430 query.strcat(tmp.c_str());
1433 /* expect jobid,fileindex */
1435 while (get_next_id_from_list(&hardlink, &jobid) == 1) {
1436 if (get_next_id_from_list(&hardlink, &id) != 1) {
1437 Dmsg0(dbglevel, "ERROR: hardlink should be two by two\n");
1440 if (jobid != prev_jobid) { /* new job */
1441 if (prev_jobid == 0) { /* first jobid */
1443 query.strcat(" UNION ");
1445 } else { /* end last job, start new one */
1446 tmp.strcat(") UNION ");
1447 query.strcat(tmp.c_str());
1449 Mmsg(tmp, "SELECT Job.JobId, JobTDate, FileIndex, FilenameId, "
1451 "FROM File JOIN Job USING (JobId) WHERE JobId = %lld "
1452 "AND FileIndex IN (%lld", jobid, id);
1455 } else { /* same job, add new findex */
1456 Mmsg(tmp2, ", %lld", id);
1457 tmp.strcat(tmp2.c_str());
1461 if (prev_jobid != 0) { /* end last job */
1463 query.strcat(tmp.c_str());
1467 Dmsg1(dbglevel_sql, "query=%s\n", query.c_str());
1469 if (!db->bdb_sql_query(query.c_str(), NULL, NULL)) {
1470 Dmsg1(dbglevel, "ERROR executing query=%s\n", query.c_str());
1474 Mmsg(query, sql_bvfs_select[db->bdb_get_type_index()],
1475 output_table, output_table, output_table);
1477 /* TODO: handle jobid filter */
1478 Dmsg1(dbglevel_sql, "query=%s\n", query.c_str());
1479 if (!db->bdb_sql_query(query.c_str(), NULL, NULL)) {
1480 Dmsg1(dbglevel, "ERROR executing query=%s\n", query.c_str());
1484 /* MySQL needs the index */
1485 if (db->bdb_get_type_index() == SQL_TYPE_MYSQL) {
1486 Mmsg(query, "CREATE INDEX idx_%s ON %s (JobId)",
1487 output_table, output_table);
1488 Dmsg1(dbglevel_sql, "query=%s\n", query.c_str());
1489 if (!db->bdb_sql_query(query.c_str(), NULL, NULL)) {
1490 Dmsg1(dbglevel, "ERROR executing query=%s\n", query.c_str());
1495 /* Check if some FileId have DeltaSeq > 0
1496 * Foreach of them we need to get the accurate_job list, and compute
1497 * what are dependencies
1500 "SELECT F.FileId, F.JobId, F.FilenameId, F.PathId, F.DeltaSeq "
1501 "FROM File AS F JOIN Job USING (JobId) JOIN %s USING (FileId) "
1502 "WHERE DeltaSeq > 0", output_table);
1504 if (!db->QueryDB(jcr, query.c_str())) {
1505 Dmsg1(dbglevel_sql, "Can't execute query=%s\n", query.c_str());
1508 /* TODO: Use an other DB connection can avoid to copy the result of the
1509 * previous query into a temporary buffer
1511 num = db->sql_num_rows();
1512 Dmsg2(dbglevel, "Found %d Delta parts in restore selection q=%s\n", num, query.c_str());
1515 int64_t *result = (int64_t *)malloc (num * 4 * sizeof(int64_t));
1519 while((row = db->sql_fetch_row())) {
1520 result[i++] = str_to_int64(row[0]); /* FileId */
1521 result[i++] = str_to_int64(row[1]); /* JobId */
1522 result[i++] = str_to_int64(row[2]); /* FilenameId */
1523 result[i++] = str_to_int64(row[3]); /* PathId */
1528 insert_missing_delta(output_table, result + i);
1538 Mmsg(query, "DROP TABLE btemp%s", output_table);
1539 db->bdb_sql_query(query.c_str(), NULL, NULL);
1544 void Bvfs::insert_missing_delta(char *output_table, int64_t *res)
1550 memset(&jr, 0, sizeof(jr));
1551 memset(&jr2, 0, sizeof(jr2));
1553 /* Need to limit the query to StartTime, Client/Fileset */
1555 db->bdb_get_job_record(jcr, &jr2);
1558 jr.ClientId = jr2.ClientId;
1559 jr.FileSetId = jr2.FileSetId;
1560 jr.JobLevel = L_INCREMENTAL;
1561 jr.StartTime = jr2.StartTime;
1563 /* Get accurate jobid list */
1564 db->bdb_get_accurate_jobids(jcr, &jr, &lst);
1566 Dmsg2(dbglevel_sql, "JobId list for %lld is %s\n", res[0], lst.list);
1568 /* The list contains already the last DeltaSeq element, so
1569 * we don't need to select it in the next query
1571 for (int l = strlen(lst.list); l > 0; l--) {
1572 if (lst.list[l] == ',') {
1578 Dmsg1(dbglevel_sql, "JobId list after strip is %s\n", lst.list);
1580 /* Escape filename */
1581 db->fnl = strlen((char *)res[2]);
1582 db->esc_name = check_pool_memory_size(db->esc_name, 2*db->fnl+2);
1583 db->bdb_escape_string(jcr, db->esc_name, (char *)res[2], db->fnl);
1585 edit_int64(res[3], ed1); /* pathid */
1587 int id=db->bdb_get_type_index();
1588 Mmsg(query, bvfs_select_delta_version_with_basejob_and_delta[id],
1589 lst.list, db->esc_name, ed1,
1590 lst.list, db->esc_name, ed1,
1591 lst.list, lst.list);
1593 Mmsg(db->cmd, "INSERT INTO %s "
1594 "SELECT JobId, FileIndex, FileId FROM (%s) AS F1",
1595 output_table, query.c_str());
1597 if (!db->bdb_sql_query(db->cmd, NULL, NULL)) {
1598 Dmsg1(dbglevel_sql, "Can't exec q=%s\n", db->cmd);
1602 #endif /* HAVE_SQLITE3 || HAVE_MYSQL || HAVE_POSTGRESQL */