From 2e57a996f8b2016f012b1bd478e2f59175bf0ec5 Mon Sep 17 00:00:00 2001 From: Kern Sibbald Date: Tue, 18 Nov 2008 13:22:02 +0000 Subject: [PATCH] kes Implement bsr block level checking for disk files. However, it does not work correctly in accurate tests, and all the migration and copy tests, so it is turned off. git-svn-id: https://bacula.svn.sourceforge.net/svnroot/bacula/trunk@8054 91ce42f0-d328-0410-95d8-f526ca767f89 --- bacula/src/dird/autoprune.c | 3 +- bacula/src/dird/protos.h | 1 + bacula/src/dird/ua_prune.c | 39 +++++++++----- bacula/src/dird/ua_purge.c | 1 + bacula/src/stored/match_bsr.c | 91 ++++++++++++++++++++++++++++++--- bacula/src/stored/read_record.c | 4 +- bacula/src/stored/record.c | 9 ++-- bacula/src/version.h | 4 +- bacula/technotes-2.5 | 5 +- 9 files changed, 128 insertions(+), 29 deletions(-) diff --git a/bacula/src/dird/autoprune.c b/bacula/src/dird/autoprune.c index 9109240762..9632aeb3a5 100644 --- a/bacula/src/dird/autoprune.c +++ b/bacula/src/dird/autoprune.c @@ -182,9 +182,10 @@ void prune_volumes(JCR *jcr, bool InChanger, MEDIA_DBR *mr) prune_list.num_ids = 0; /* reset count */ } if (!is_volume_purged(ua, &lmr)) { - Dmsg1(100, "Vol=%s not pruned\n", lmr.VolumeName); + Dmsg1(050, "Vol=%s not pruned\n", lmr.VolumeName); continue; } + Dmsg1(050, "Vol=%s is purged\n", lmr.VolumeName); /* * Since we are also pruning the Scratch pool, continue diff --git a/bacula/src/dird/protos.h b/bacula/src/dird/protos.h index 7f29af765d..123350f673 100644 --- a/bacula/src/dird/protos.h +++ b/bacula/src/dird/protos.h @@ -283,6 +283,7 @@ int job_delete_handler(void *ctx, int num_fields, char **row); int del_count_handler(void *ctx, int num_fields, char **row); int file_delete_handler(void *ctx, int num_fields, char **row); int get_prune_list_for_volume(UAContext *ua, MEDIA_DBR *mr, del_ctx *del); +int exclude_running_jobs_from_list(del_ctx *prune_list); /* ua_purge.c */ bool is_volume_purged(UAContext *ua, MEDIA_DBR *mr); diff --git a/bacula/src/dird/ua_prune.c b/bacula/src/dird/ua_prune.c index e1ffb7937f..f0d50e2297 100644 --- a/bacula/src/dird/ua_prune.c +++ b/bacula/src/dird/ua_prune.c @@ -465,11 +465,8 @@ int get_prune_list_for_volume(UAContext *ua, MEDIA_DBR *mr, del_ctx *del) { POOL_MEM query(PM_MESSAGE); int count = 0; - int i; utime_t now, period; char ed1[50], ed2[50]; - JCR *jcr; - bool skip; if (mr->Enabled == 2) { return 0; /* cannot prune Archived volumes */ @@ -496,27 +493,45 @@ int get_prune_list_for_volume(UAContext *ua, MEDIA_DBR *mr, del_ctx *del) Dmsg0(050, "Count failed\n"); goto bail_out; } + count = exclude_running_jobs_from_list(del); + +bail_out: + db_unlock(ua->db); + return count; +} + +/* + * We have a list of jobs to prune or purge. If any of them is + * currently running, we set its JobId to zero which effectively + * excludes it. + * + * Returns the number of jobs that can be prunned or purged. + * + */ +int exclude_running_jobs_from_list(del_ctx *prune_list) +{ + int count = 0; + JCR *jcr; + bool skip; + int i; /* Do not prune any job currently running */ - for (i=0; i < del->num_ids; i++) { + for (i=0; i < prune_list->num_ids; i++) { skip = false; foreach_jcr(jcr) { - if (jcr->JobId == del->JobId[i]) { - Dmsg2(150, "skip same job JobId[%d]=%d\n", i, (int)del->JobId[i]); - del->JobId[i] = 0; + if (jcr->JobId == prune_list->JobId[i]) { + Dmsg2(050, "skip running job JobId[%d]=%d\n", i, (int)prune_list->JobId[i]); + prune_list->JobId[i] = 0; skip = true; break; } } endeach_jcr(jcr); if (skip) { - continue; + continue; /* don't increment count */ } - Dmsg2(150, "accept JobId[%d]=%d\n", i, (int)del->JobId[i]); + Dmsg2(050, "accept JobId[%d]=%d\n", i, (int)prune_list->JobId[i]); count++; } - -bail_out: - db_unlock(ua->db); return count; } diff --git a/bacula/src/dird/ua_purge.c b/bacula/src/dird/ua_purge.c index 59400b5f60..d4b66e0ce9 100644 --- a/bacula/src/dird/ua_purge.c +++ b/bacula/src/dird/ua_purge.c @@ -467,6 +467,7 @@ bool is_volume_purged(UAContext *ua, MEDIA_DBR *mr) purged = true; goto bail_out; } + /* If purged, mark it so */ cnt.count = 0; Mmsg(query, "SELECT count(*) FROM JobMedia WHERE MediaId=%s", diff --git a/bacula/src/stored/match_bsr.c b/bacula/src/stored/match_bsr.c index 0f950e903e..83aa323960 100644 --- a/bacula/src/stored/match_bsr.c +++ b/bacula/src/stored/match_bsr.c @@ -1,7 +1,7 @@ /* Bacula® - The Network Backup Solution - Copyright (C) 2002-2007 Free Software Foundation Europe e.V. + Copyright (C) 2002-2008 Free Software Foundation Europe e.V. The main author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -73,6 +73,7 @@ static int match_job_level(BSR *bsr, BSR_JOBLEVEL *job_level, SESSION_LABEL *ses static int match_jobid(BSR *bsr, BSR_JOBID *jobid, SESSION_LABEL *sessrec, bool done); static int match_findex(BSR *bsr, BSR_FINDEX *findex, DEV_RECORD *rec, bool done); static int match_volfile(BSR *bsr, BSR_VOLFILE *volfile, DEV_RECORD *rec, bool done); +static int match_volblock(BSR *bsr, BSR_VOLBLOCK *volblock, DEV_RECORD *rec, bool done); static int match_stream(BSR *bsr, BSR_STREAM *stream, DEV_RECORD *rec, bool done); static int match_all(BSR *bsr, DEV_RECORD *rec, VOLUME_LABEL *volrec, SESSION_LABEL *sessrec, bool done, JCR *jcr); static int match_block_sesstime(BSR *bsr, BSR_SESSTIME *sesstime, DEV_BLOCK *block); @@ -357,20 +358,35 @@ bool is_this_bsr_done(BSR *bsr, DEV_RECORD *rec) static int match_all(BSR *bsr, DEV_RECORD *rec, VOLUME_LABEL *volrec, SESSION_LABEL *sessrec, bool done, JCR *jcr) { + Dmsg0(050, "Enter match_all\n"); if (bsr->done) { // Dmsg0(dbglevel, "bsr->done set\n"); goto no_match; } if (!match_volume(bsr, bsr->volume, volrec, 1)) { - Dmsg2(dbglevel, "bsr fail vol=%s != rec vol=%s\n", bsr->volume->VolumeName, + Dmsg2(dbglevel, "bsr fail bsr_vol=%s != rec read_vol=%s\n", bsr->volume->VolumeName, volrec->VolumeName); goto no_match; } + Dmsg2(dbglevel, "OK bsr match bsr_vol=%s read_vol=%s\n", bsr->volume->VolumeName, + volrec->VolumeName); + if (!match_volfile(bsr, bsr->volfile, rec, 1)) { Dmsg3(dbglevel, "Fail on file=%d. bsr=%d,%d\n", rec->File, bsr->volfile->sfile, bsr->volfile->efile); goto no_match; } + Dmsg3(dbglevel, "OK bsr file=%d. bsr=%d,%d\n", + rec->File, bsr->volfile->sfile, bsr->volfile->efile); + + if (!match_volblock(bsr, bsr->volblock, rec, 1)) { + Dmsg3(dbglevel, "Fail on Block=%d. bsr=%d,%d\n", + rec->Block, bsr->volblock->sblock, bsr->volblock->eblock); + goto no_match; + } + Dmsg3(dbglevel, "OK bsr Block=%d. bsr=%d,%d\n", + rec->Block, bsr->volblock->sblock, bsr->volblock->eblock); + if (!match_sesstime(bsr, bsr->sesstime, rec, 1)) { Dmsg2(dbglevel, "Fail on sesstime. bsr=%d rec=%d\n", bsr->sesstime->sesstime, rec->VolSessionTime); @@ -390,6 +406,8 @@ static int match_all(BSR *bsr, DEV_RECORD *rec, VOLUME_LABEL *volrec, rec->FileIndex, bsr->FileIndex->findex, bsr->FileIndex->findex2); goto no_match; } + Dmsg3(dbglevel, "match on findex=%d. bsr=%d,%d\n", + rec->FileIndex, bsr->FileIndex->findex, bsr->FileIndex->findex2); if (!match_fileregex(bsr, rec, jcr)) { Dmsg1(dbglevel, "Fail on fileregex='%s'\n", bsr->fileregex); @@ -412,6 +430,7 @@ static int match_all(BSR *bsr, DEV_RECORD *rec, VOLUME_LABEL *volrec, */ if (bsr->count && bsr->FileIndex) { rec->bsr = bsr; + Dmsg0(050, "Leave match_all 1\n"); return 1; /* this is a complete match */ } @@ -445,6 +464,7 @@ static int match_all(BSR *bsr, DEV_RECORD *rec, VOLUME_LABEL *volrec, Dmsg0(dbglevel, "fail on stream\n"); goto no_match; } + Dmsg0(050, "Leave match_all 1\n"); return 1; no_match: @@ -452,8 +472,10 @@ no_match: return match_all(bsr->next, rec, volrec, sessrec, bsr->done && done, jcr); } if (bsr->done && done) { + Dmsg0(050, "Leave match all -1\n"); return -1; } + Dmsg0(050, "Leave match all 0\n"); return 0; } @@ -463,6 +485,7 @@ static int match_volume(BSR *bsr, BSR_VOLUME *volume, VOLUME_LABEL *volrec, bool return 0; /* Volume must match */ } if (strcmp(volume->VolumeName, volrec->VolumeName) == 0) { + Dmsg1(050, "match_volume=%s\n", volrec->VolumeName); return 1; } if (volume->next) { @@ -546,12 +569,19 @@ static int match_volfile(BSR *bsr, BSR_VOLFILE *volfile, DEV_RECORD *rec, bool d if (!volfile) { return 1; /* no specification matches all */ } +/* + * The following code is turned off because this should now work + * with disk files too, though since a "volfile" is 4GB, it does + * not improve performance much. + */ +#ifdef xxx /* For the moment, these tests work only with tapes. */ if (!(rec->state & REC_ISTAPE)) { return 1; /* All File records OK for this match */ } -// Dmsg3(dbglevel, "match_volfile: sfile=%d efile=%d recfile=%d\n", -// volfile->sfile, volfile->efile, rec->File); + Dmsg3(dbglevel, "match_volfile: sfile=%d efile=%d recfile=%d\n", + volfile->sfile, volfile->efile, rec->File); +#endif if (volfile->sfile <= rec->File && volfile->efile >= rec->File) { return 1; } @@ -573,6 +603,53 @@ static int match_volfile(BSR *bsr, BSR_VOLFILE *volfile, DEV_RECORD *rec, bool d return 0; } +static int match_volblock(BSR *bsr, BSR_VOLBLOCK *volblock, DEV_RECORD *rec, bool done) +{ + /* + * Currently block matching does not work correctly for disk + * files in all cases, so it is "turned off" by the following + * return statement. + */ + return 1; + + + if (!volblock) { + return 1; /* no specification matches all */ + } + /* For the moment, these tests work only with disk. */ + if (rec->state & REC_ISTAPE) { + return 1; /* All File records OK for this match */ + } +// Dmsg3(dbglevel, "match_volblock: sblock=%d eblock=%d recblock=%d\n", +// volblock->sblock, volblock->eblock, rec->Block); + if (volblock->sblock <= rec->Block && volblock->eblock >= rec->Block) { + return 1; + } + /* Once we get past last eblock, we are done */ + if (rec->Block > volblock->eblock) { + volblock->done = true; /* set local done */ + } + if (volblock->next) { + return match_volblock(bsr, volblock->next, rec, volblock->done && done); + } + +/* + * This is turned off because I do not believe that we can mark + * the bsr as done at this level. + */ +#ifdef xxx + /* If we are done and all prior matches are done, this bsr is finished */ + if (volblock->done && done) { + bsr->done = true; + bsr->root->reposition = true; + Dmsg2(dbglevel, "bsr done from volblock rec=%d voleblock=%d\n", + rec->Block, volblock->eblock); + } +#endif + return 0; +} + + static int match_stream(BSR *bsr, BSR_STREAM *stream, DEV_RECORD *rec, bool done) { if (!stream) { @@ -628,9 +705,7 @@ static int match_sessid(BSR *bsr, BSR_SESSID *sessid, DEV_RECORD *rec) * are found in sequential order. Thus we can make optimizations. * * ***FIXME*** optimizations - * We could optimize a lot here by removing the recursion, and - * stopping the search earlier -- say when rec->FileIndex > findex->findex2 - * and findex->next == NULL. + * We could optimize by removing the recursion. */ static int match_findex(BSR *bsr, BSR_FINDEX *findex, DEV_RECORD *rec, bool done) { @@ -639,7 +714,7 @@ static int match_findex(BSR *bsr, BSR_FINDEX *findex, DEV_RECORD *rec, bool done } if (!findex->done) { if (findex->findex <= rec->FileIndex && findex->findex2 >= rec->FileIndex) { - Dmsg3(dbglevel, "Match on findex=%d. bsr=%d,%d\n", + Dmsg3(dbglevel, "Match on findex=%d. bsrFIs=%d,%d\n", rec->FileIndex, findex->findex, findex->findex2); return 1; } diff --git a/bacula/src/stored/read_record.c b/bacula/src/stored/read_record.c index 9dbbf58ad7..0ea2ce0491 100644 --- a/bacula/src/stored/read_record.c +++ b/bacula/src/stored/read_record.c @@ -258,10 +258,10 @@ bool read_records(DCR *dcr, rec->match_stat = match_bsr(jcr->bsr, rec, &dev->VolHdr, &sessrec, jcr); if (rec->match_stat == -1) { /* no more possible matches */ done = true; /* all items found, stop */ - Dmsg2(dbglvl, "All done=(file:block) %u:%u\n", dev->file, dev->block_num); + Dmsg2(100, "All done=(file:block) %u:%u\n", dev->file, dev->block_num); break; } else if (rec->match_stat == 0) { /* no match */ - Dmsg4(dbglvl, "BSR no match: clear rem=%d FI=%d before set_eof pos %u:%u\n", + Dmsg4(100, "BSR no match: clear rem=%d FI=%d before set_eof pos %u:%u\n", rec->remainder, rec->FileIndex, dev->file, dev->block_num); rec->remainder = 0; rec->state &= ~REC_PARTIAL_RECORD; diff --git a/bacula/src/stored/record.c b/bacula/src/stored/record.c index 61cd686a3d..b5893981f2 100644 --- a/bacula/src/stored/record.c +++ b/bacula/src/stored/record.c @@ -444,14 +444,17 @@ bool read_record_from_block(DCR *dcr, DEV_BLOCK *block, DEV_RECORD *rec) char buf1[100], buf2[100]; remlen = block->binbuf; - rec->Block = block->BlockNumber; - rec->File = ((DEVICE *)block->dev)->file; /* Clear state flags */ rec->state = 0; if (block->dev->is_tape()) { rec->state |= REC_ISTAPE; - } + rec->Block = block->BlockNumber; + rec->File = ((DEVICE *)block->dev)->file; + } else { + rec->Block = ((DEVICE *)block->dev)->EndBlock; + rec->File = ((DEVICE *)block->dev)->EndFile; + } /* diff --git a/bacula/src/version.h b/bacula/src/version.h index 1c0b2aaa23..8714af4460 100644 --- a/bacula/src/version.h +++ b/bacula/src/version.h @@ -4,8 +4,8 @@ #undef VERSION #define VERSION "2.5.20" -#define BDATE "13 November 2008" -#define LSMDATE "13Nov08" +#define BDATE "18 November 2008" +#define LSMDATE "18Nov08" #define PROG_COPYRIGHT "Copyright (C) %d-2008 Free Software Foundation Europe e.V.\n" #define BYEAR "2008" /* year for copyright messages in progs */ diff --git a/bacula/technotes-2.5 b/bacula/technotes-2.5 index 4ba4977b46..34cccf50f6 100644 --- a/bacula/technotes-2.5 +++ b/bacula/technotes-2.5 @@ -11,7 +11,10 @@ mixed priorities General: 18Nov08 -ebl Make SD plugins working. +kes Implement bsr block level checking for disk files. However, + it does not work correctly in accurate tests, and all the + migration and copy tests, so it is turned off. +ebl Make SD plugins work. 14Nov08 ebl Apply Riccardo's patch to compile bacula+mysql on mandriva 13Nov08 -- 2.39.5