From 82b332d1bac62606803a851e3ffc62ec38df31ac Mon Sep 17 00:00:00 2001 From: Kern Sibbald Date: Sun, 24 Feb 2008 11:29:53 +0000 Subject: [PATCH] Implement Catalog in Pool resource. It overrides catalog specified in the Client resource. git-svn-id: https://bacula.svn.sourceforge.net/svnroot/bacula/trunk@6477 91ce42f0-d328-0410-95d8-f526ca767f89 --- bacula/kernstodo | 74 +++++++++---------------------------- bacula/src/dird/backup.c | 2 + bacula/src/dird/catreq.c | 2 +- bacula/src/dird/dird.c | 14 ++++--- bacula/src/dird/dird_conf.c | 10 ++--- bacula/src/dird/dird_conf.h | 2 +- bacula/src/dird/job.c | 26 ++++++++++++- bacula/src/dird/migrate.c | 2 + bacula/src/dird/ua_cmds.c | 6 ++- bacula/src/dird/ua_run.c | 3 +- bacula/src/jcr.h | 1 + bacula/src/version.h | 4 +- bacula/technotes-2.3 | 3 ++ 13 files changed, 74 insertions(+), 75 deletions(-) diff --git a/bacula/kernstodo b/bacula/kernstodo index a178b1fb8c..d472d1d590 100644 --- a/bacula/kernstodo +++ b/bacula/kernstodo @@ -1,5 +1,5 @@ Kern's ToDo List - 15 February 2008 + 23 February 2008 Document: @@ -66,16 +66,22 @@ Professional Needs: Complete rework of the scheduling system (not in list) Performance and usage instrumentation (not in list) See email of 21Aug2007 for details. -- Implement Diff,Inc Retention Periods - Look at: http://tech.groups.yahoo.com/group/cfg2html and http://www.openeyet.nl/scc/ for managing customer changes Priority: +- Add Catalog = to Pool resource so that pools will exist + in only one catalog -- currently Pools are "global". +- New directive "Delete purged Volumes" +- Prune by Job +- Prune by Job Level (Full, Differential, Incremental) +- Strict automatic pruning - Implement unmount of USB volumes. - Use "./config no-idea no-mdc2 no-rc5" on building OpenSSL for Win32 to avoid patent problems. -- Plugins -- Implement Despooling data status. +- Implement Bacula plugins -- design API +- modify pruning to keep a fixed number of versions of a file, + if requested. === Duplicate jobs === hese apply only to backup jobs. @@ -141,31 +147,7 @@ Priority: running of that Job (i.e. lets any previous invocation finish before doing Interval testing). - Look at simplifying File exclusions. -- New directive "Delete purged Volumes" -- It appears to me that you have run into some sort of race - condition where two threads want to use the same Volume and they - were both given access. Normally that is no problem. However, - one thread wanted the particular Volume in drive 0, but it was - loaded into drive 1 so it decided to unload it from drive 1 and - then loaded it into drive 0, while the second thread went on - thinking that the Volume could be used in drive 1 not realizing - that in between time, it was loaded in drive 0. - I'll look at the code to see if there is some way we can avoid - this kind of problem. Probably the best solution is to make the - first thread simply start using the Volume in drive 1 rather than - transferring it to drive 0. ->>>>>>> .r6288 -- Complete Catalog in Pool -- Implement Bacula plugins -- design API ->>>>>>> .r6079 - Scripts -- Prune by Job -- Prune by Job Level -- True automatic pruning -- Duplicate Jobs - Run, Fail, Skip, Higher, Promote, CancelLowerLevel - Proximity - New directive. - Auto update of slot: rufus-dir: ua_run.c:456-10 JobId=10 NewJobId=10 using pool Full priority=10 02-Nov 12:58 rufus-dir JobId 10: Start Backup JobId 10, Job=kernsave.2007-11-02_12.58.03 @@ -181,8 +163,6 @@ Priority: - Create FileVersions table - Look at rsysnc for incremental updates and dedupping - Add MD5 or SHA1 check in SD for data validation -- modify pruning to keep a fixed number of versions of a file, - if requested. - finish implementation of fdcalled -- see ua_run.c:105 - Fix problem in postgresql.c in my_postgresql_query, where the generation of the error message doesn't differentiate result==NULL @@ -194,22 +174,17 @@ Priority: - Implement continue spooling while despooling. - Remove all install temp files in Win32 PLUGINSDIR. - Audit retention periods to make sure everything is 64 bit. -- Use E'xxx' to escape PostgreSQL strings. - No where in restore causes kaboom. - Performance: multiple spool files for a single job. - Performance: despool attributes when despooling data (problem multiplexing Dir connection). - Make restore use the in-use volume reservation algorithm. -- Look at mincore: http://insights.oetiker.ch/linux/fadvise.html -- Unicode input http://en.wikipedia.org/wiki/Byte_Order_Mark - Add TLS to bat (should be done). - When Pool specifies Storage command override does not work. - Implement wait_for_sysop() message display in wait_for_device(), which now prints warnings too often. - Ensure that each device in an Autochanger has a different Device Index. -- Add Catalog = to Pool resource so that pools will exist - in only one catalog -- currently Pools are "global". - Look at sg_logs -a /dev/sg0 for getting soft errors. - btape "test" command with Offline on Unmount = yes @@ -241,8 +216,6 @@ Priority: and possibly changing the blobs into varchar. - Ensure that the SD re-reads the Media record if the JobFiles does not match -- it may have been updated by another job. -- Look at moving the Storage directive from the Job to the - Pool in the default conf files. - Doc items - Test Volume compatibility between machine architectures - Encryption documentation @@ -1612,10 +1585,6 @@ Jerry Schieffer ============================== Longer term to do: -- Implement wait on multiple objects - - Multiple max times - - pthread signal - - socket input ready - Design at hierarchial storage for Bacula. Migration and Clone. - Implement FSM (File System Modules). - Audit M_ error codes to ensure they are correct and consistent. @@ -1692,22 +1661,6 @@ Need: ========================================================= -========================================================== - Unsaved File design -For each Incremental job that is run, there may be files that -were found but not saved because they were locked (this applies -only to Windows). Such a system could send back to the Director -a list of Unsaved files. -Need: -- New UnSavedFiles table that contains: - JobId - PathId - FilenameId -- Then in the next Incremental job, the list of Unsaved Files will be - feed to the FD, who will ensure that they are explicitly chosen even - if standard date/time check would not have selected them. -============================================================= - ===== Multiple drive autochanger data: see Alan Brown @@ -1878,3 +1831,10 @@ Block Position: 0 #define S_IFDOOR in st_mode. see: http://docs.sun.com/app/docs/doc/816-5173/6mbb8ae23?a=view#indexterm-360 - Figure out how to recycle Scratch volumes back to the Scratch Pool. +- Implement Despooling data status. +- Use E'xxx' to escape PostgreSQL strings. +- Look at mincore: http://insights.oetiker.ch/linux/fadvise.html +- Unicode input http://en.wikipedia.org/wiki/Byte_Order_Mark +- Look at moving the Storage directive from the Job to the + Pool in the default conf files. + diff --git a/bacula/src/dird/backup.c b/bacula/src/dird/backup.c index 6d7c43d172..57f6f07f40 100644 --- a/bacula/src/dird/backup.c +++ b/bacula/src/dird/backup.c @@ -461,6 +461,7 @@ void backup_cleanup(JCR *jcr, int TermCode) " Client: \"%s\" %s\n" " FileSet: \"%s\" %s\n" " Pool: \"%s\" (From %s)\n" +" Catalog: \"%s\" (From %s)\n" " Storage: \"%s\" (From %s)\n" " Scheduled time: %s\n" " Start time: %s\n" @@ -492,6 +493,7 @@ void backup_cleanup(JCR *jcr, int TermCode) jcr->client->name(), cr.Uname, jcr->fileset->name(), jcr->FSCreateTime, jcr->pool->name(), jcr->pool_source, + jcr->catalog->name(), jcr->catalog_source, jcr->wstore->name(), jcr->wstore_source, schedt, sdt, diff --git a/bacula/src/dird/catreq.c b/bacula/src/dird/catreq.c index 709707069d..8aa7b550b2 100644 --- a/bacula/src/dird/catreq.c +++ b/bacula/src/dird/catreq.c @@ -1,7 +1,7 @@ /* Bacula® - The Network Backup Solution - Copyright (C) 2001-2007 Free Software Foundation Europe e.V. + Copyright (C) 2001-2008 Free Software Foundation Europe e.V. The main author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. diff --git a/bacula/src/dird/dird.c b/bacula/src/dird/dird.c index ea0a6987e5..d6722a66a7 100644 --- a/bacula/src/dird/dird.c +++ b/bacula/src/dird/dird.c @@ -907,12 +907,14 @@ static bool check_catalog() /* Loop over all pools, defining/updating them in each database */ POOL *pool; foreach_res(pool, R_POOL) { - create_pool(NULL, db, pool, POOL_OP_UPDATE); /* update request */ - } - - /* Loop over all pools for updating RecyclePool */ - foreach_res(pool, R_POOL) { - update_pool_recyclepool(NULL, db, pool); + /* + * If the Pool has a catalog resource create the pool only + * in that catalog. + */ + if (!pool->catalog || pool->catalog == catalog) { + create_pool(NULL, db, pool, POOL_OP_UPDATE); /* update request */ + update_pool_recyclepool(NULL, db, pool); + } } STORE *store; diff --git a/bacula/src/dird/dird_conf.c b/bacula/src/dird/dird_conf.c index e85d189561..27de4b1f12 100644 --- a/bacula/src/dird/dird_conf.c +++ b/bacula/src/dird/dird_conf.c @@ -1,7 +1,7 @@ /* Bacula® - The Network Backup Solution - Copyright (C) 2000-2007 Free Software Foundation Europe e.V. + Copyright (C) 2000-2008 Free Software Foundation Europe e.V. The main author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -382,7 +382,7 @@ static RES_ITEM pool_items[] = { {"recycle", store_bool, ITEM(res_pool.Recycle), 0, ITEM_DEFAULT, true}, {"recyclepool", store_res, ITEM(res_pool.RecyclePool), R_POOL, 0, 0}, {"copypool", store_alist_res, ITEM(res_pool.CopyPool), R_POOL, 0, 0}, - {"catalog", store_res, ITEM(res_pool.Catalog), R_CATALOG, 0, 0}, + {"catalog", store_res, ITEM(res_pool.catalog), R_CATALOG, 0, 0}, {NULL, NULL, {0}, 0, 0, 0} }; @@ -910,8 +910,8 @@ next_run: if (res->res_pool.RecyclePool) { sendit(sock, _(" RecyclePool=%s\n"), res->res_pool.RecyclePool->name()); } - if (res->res_pool.Catalog) { - sendit(sock, _(" Catalog=%s\n"), res->res_pool.Catalog->name()); + if (res->res_pool.catalog) { + sendit(sock, _(" Catalog=%s\n"), res->res_pool.catalog->name()); } if (res->res_pool.storage) { STORE *store; @@ -1339,7 +1339,7 @@ void save_resource(int type, RES_ITEM *items, int pass) res->res_pool.NextPool = res_all.res_pool.NextPool; res->res_pool.RecyclePool = res_all.res_pool.RecyclePool; res->res_pool.storage = res_all.res_pool.storage; - res->res_pool.Catalog = res_all.res_pool.Catalog; + res->res_pool.catalog = res_all.res_pool.catalog; break; case R_CONSOLE: if ((res = (URES *)GetResWithName(R_CONSOLE, res_all.res_con.hdr.name)) == NULL) { diff --git a/bacula/src/dird/dird_conf.h b/bacula/src/dird/dird_conf.h index dc7be31305..f35890ff7f 100644 --- a/bacula/src/dird/dird_conf.h +++ b/bacula/src/dird/dird_conf.h @@ -546,7 +546,7 @@ public: bool Recycle; /* default for media recycle yes/no */ POOL *RecyclePool; /* RecyclePool destination when media is purged */ alist *CopyPool; /* List of copy pools */ - CAT *Catalog; /* Catalog to be used */ + CAT *catalog; /* Catalog to be used */ /* Methods */ char *name() const; }; diff --git a/bacula/src/dird/job.c b/bacula/src/dird/job.c index 4b1b858b56..ea8a41bc8f 100644 --- a/bacula/src/dird/job.c +++ b/bacula/src/dird/job.c @@ -597,6 +597,8 @@ DBId_t get_or_create_pool_record(JCR *jcr, char *pool_name) void apply_pool_overrides(JCR *jcr) { + bool pool_override = false; + if (jcr->run_pool_override) { pm_strcpy(jcr->pool_source, _("Run pool override")); } @@ -607,6 +609,7 @@ void apply_pool_overrides(JCR *jcr) case L_FULL: if (jcr->full_pool) { jcr->pool = jcr->full_pool; + pool_override = true; if (jcr->run_full_pool_override) { pm_strcpy(jcr->pool_source, _("Run FullPool override")); } else { @@ -617,6 +620,7 @@ void apply_pool_overrides(JCR *jcr) case L_INCREMENTAL: if (jcr->inc_pool) { jcr->pool = jcr->inc_pool; + pool_override = true; if (jcr->run_inc_pool_override) { pm_strcpy(jcr->pool_source, _("Run IncPool override")); } else { @@ -627,6 +631,7 @@ void apply_pool_overrides(JCR *jcr) case L_DIFFERENTIAL: if (jcr->diff_pool) { jcr->pool = jcr->diff_pool; + pool_override = true; if (jcr->run_diff_pool_override) { pm_strcpy(jcr->pool_source, _("Run DiffPool override")); } else { @@ -635,6 +640,11 @@ void apply_pool_overrides(JCR *jcr) } break; } + /* Update catalog if pool overridden */ + if (pool_override && jcr->pool->catalog) { + jcr->catalog = jcr->pool->catalog; + pm_strcpy(jcr->catalog_source, _("Pool resource")); + } } @@ -875,6 +885,10 @@ void dird_free_jcr(JCR *jcr) free_pool_memory(jcr->pool_source); jcr->pool_source = NULL; } + if (jcr->catalog_source) { + free_pool_memory(jcr->catalog_source); + jcr->catalog_source = NULL; + } if (jcr->rpool_source) { free_pool_memory(jcr->rpool_source); jcr->rpool_source = NULL; @@ -959,6 +973,10 @@ void set_jcr_defaults(JCR *jcr, JOB *job) jcr->pool_source = get_pool_memory(PM_MESSAGE); pm_strcpy(jcr->pool_source, _("unknown source")); } + if (!jcr->catalog_source) { + jcr->catalog_source = get_pool_memory(PM_MESSAGE); + pm_strcpy(jcr->catalog_source, _("unknown source")); + } jcr->JobPriority = job->Priority; /* Copy storage definitions -- deleted in dir_free_jcr above */ @@ -977,7 +995,13 @@ void set_jcr_defaults(JCR *jcr, JOB *job) jcr->full_pool = job->full_pool; jcr->inc_pool = job->inc_pool; jcr->diff_pool = job->diff_pool; - jcr->catalog = job->client->catalog; + if (job->pool->catalog) { + jcr->catalog = job->pool->catalog; + pm_strcpy(jcr->catalog_source, _("Pool resource")); + } else { + jcr->catalog = job->client->catalog; + pm_strcpy(jcr->catalog_source, _("Client resource")); + } jcr->fileset = job->fileset; jcr->messages = job->messages; jcr->spool_data = job->spool_data; diff --git a/bacula/src/dird/migrate.c b/bacula/src/dird/migrate.c index 5de5fe8697..ed08899552 100644 --- a/bacula/src/dird/migrate.c +++ b/bacula/src/dird/migrate.c @@ -1178,6 +1178,7 @@ void migration_cleanup(JCR *jcr, int TermCode) " Read Storage: \"%s\" (From %s)\n" " Write Pool: \"%s\" (From %s)\n" " Write Storage: \"%s\" (From %s)\n" +" Catalog: \"%s\" (From %s)\n" " Start time: %s\n" " End time: %s\n" " Elapsed time: %s\n" @@ -1207,6 +1208,7 @@ void migration_cleanup(JCR *jcr, int TermCode) jcr->pool->name(), jcr->pool_source, jcr->wstore?jcr->wstore->name():"*None*", NPRT(jcr->wstore_source), + jcr->catalog->name(), jcr->catalog_source, sdt, edt, edit_utime(RunTime, elapsed, sizeof(elapsed)), diff --git a/bacula/src/dird/ua_cmds.c b/bacula/src/dird/ua_cmds.c index 65ff4d3d0d..ce5d8f09bc 100644 --- a/bacula/src/dird/ua_cmds.c +++ b/bacula/src/dird/ua_cmds.c @@ -1141,7 +1141,11 @@ static int estimate_cmd(UAContext *ua, const char *cmd) jcr->client = client; jcr->fileset = fileset; close_db(ua); - ua->catalog = client->catalog; + if (job->pool->catalog) { + ua->catalog = job->pool->catalog; + } else { + ua->catalog = client->catalog; + } if (!open_db(ua)) { return 1; diff --git a/bacula/src/dird/ua_run.c b/bacula/src/dird/ua_run.c index 5b1267d70d..41af09f17c 100644 --- a/bacula/src/dird/ua_run.c +++ b/bacula/src/dird/ua_run.c @@ -1,7 +1,7 @@ /* Bacula® - The Network Backup Solution - Copyright (C) 2001-2007 Free Software Foundation Europe e.V. + Copyright (C) 2001-2008 Free Software Foundation Europe e.V. The main author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -132,6 +132,7 @@ try_again: jcr->ExpectedFiles = rc.files; if (rc.catalog) { jcr->catalog = rc.catalog; + pm_strcpy(jcr->catalog_source, _("User input")); } if (rc.where) { if (jcr->where) { diff --git a/bacula/src/jcr.h b/bacula/src/jcr.h index 4f3f50c4de..60c9dcba1d 100644 --- a/bacula/src/jcr.h +++ b/bacula/src/jcr.h @@ -271,6 +271,7 @@ public: POOLMEM *rpool_source; /* Where migrate read pool came from */ POOLMEM *rstore_source; /* Where read storage came from */ POOLMEM *wstore_source; /* Where write storage came from */ + POOLMEM *catalog_source; /* Where catalog came from */ int replace; /* Replace option */ int NumVols; /* Number of Volume used in pool */ int reschedule_count; /* Number of times rescheduled */ diff --git a/bacula/src/version.h b/bacula/src/version.h index 8aa124b187..d6988953a5 100644 --- a/bacula/src/version.h +++ b/bacula/src/version.h @@ -4,8 +4,8 @@ #undef VERSION #define VERSION "2.3.10" -#define BDATE "22 February 2008" -#define LSMDATE "22Feb08" +#define BDATE "24 February 2008" +#define LSMDATE "24Feb08" #define PROG_COPYRIGHT "Copyright (C) %d-2008 Free Software Foundation Europe e.V.\n" #define BYEAR "2008" /* year for copyright messages in progs */ diff --git a/bacula/technotes-2.3 b/bacula/technotes-2.3 index 7d27060e4a..55e43ac0cc 100644 --- a/bacula/technotes-2.3 +++ b/bacula/technotes-2.3 @@ -1,6 +1,9 @@ Technical notes on version 2.3 General: +24Feb08 +kes Implement Catalog in Pool resource. It overrides catalog specified + in the Client resource. 22Feb08 kes Apply patch (with some difficulties) from Joao Henrique Freitas , which adds support for libdbi as a Bacula -- 2.39.5