From: Kern Sibbald Date: Sun, 5 Oct 2008 19:01:24 +0000 (+0000) Subject: kes Integrate patch from Kjetil Torgrim Homme X-Git-Tag: Release-7.0.0~4021 X-Git-Url: https://git.sur5r.net/?a=commitdiff_plain;h=4230964aa7b89e73164faf8aaa3ec281ecb7249f;p=bacula%2Fbacula kes Integrate patch from Kjetil Torgrim Homme It provides FileRegex in bsr files and code to collect regex from the user during restore, an Allow Mixed Priorities feature, and documentation for the above. git-svn-id: https://bacula.svn.sourceforge.net/svnroot/bacula/trunk@7713 91ce42f0-d328-0410-95d8-f526ca767f89 --- diff --git a/bacula/AUTHORS b/bacula/AUTHORS index 1f017fbd30..34607fe655 100644 --- a/bacula/AUTHORS +++ b/bacula/AUTHORS @@ -57,6 +57,7 @@ Juan Luis Frances Juergen Lock Karl Cunningham Kern Sibbald +Kjetil Torgrim Homme Landon Fuller Luca Berra Lucas B. Cohen diff --git a/bacula/src/dird/bsr.c b/bacula/src/dird/bsr.c index 1b57b1882b..fe38277f28 100644 --- a/bacula/src/dird/bsr.c +++ b/bacula/src/dird/bsr.c @@ -351,6 +351,9 @@ static uint32_t write_bsr(UAContext *ua, RESTORE_CTX &rx, FILE *fd) } fprintf(fd, "Volume=\"%s\"\n", bsr->VolParams[i].VolumeName); fprintf(fd, "MediaType=\"%s\"\n", bsr->VolParams[i].MediaType); + if (bsr->fileregex) { + fprintf(fd, "FileRegex=%s\n", bsr->fileregex); + } if (get_storage_device(device, bsr->VolParams[i].Storage)) { fprintf(fd, "Device=\"%s\"\n", device); } @@ -414,6 +417,9 @@ static uint32_t write_bsr(UAContext *ua, RESTORE_CTX &rx, FILE *fd) } fprintf(fd, "Volume=\"%s\"\n", bsr->VolParams[i].VolumeName); fprintf(fd, "MediaType=\"%s\"\n", bsr->VolParams[i].MediaType); + if (bsr->fileregex) { + fprintf(fd, "FileRegex=%s\n", bsr->fileregex); + } if (get_storage_device(device, bsr->VolParams[i].Storage)) { fprintf(fd, "Device=\"%s\"\n", device); } diff --git a/bacula/src/dird/bsr.h b/bacula/src/dird/bsr.h index c4cf12f437..2aafc13ccc 100644 --- a/bacula/src/dird/bsr.h +++ b/bacula/src/dird/bsr.h @@ -62,5 +62,6 @@ struct RBSR { int VolCount; /* Volume parameter count */ VOL_PARAMS *VolParams; /* Volume, start/end file/blocks */ RBSR_FINDEX *fi; /* File indexes this JobId */ + char *fileregex; /* Only restore files matching regex */ }; diff --git a/bacula/src/dird/dird_conf.c b/bacula/src/dird/dird_conf.c index d574464cdc..7b24659153 100644 --- a/bacula/src/dird/dird_conf.c +++ b/bacula/src/dird/dird_conf.c @@ -324,6 +324,7 @@ RES_ITEM job_items[] = { {"rescheduleinterval", store_time, ITEM(res_job.RescheduleInterval), 0, ITEM_DEFAULT, 60 * 30}, {"rescheduletimes", store_pint32, ITEM(res_job.RescheduleTimes), 0, 0, 0}, {"priority", store_pint32, ITEM(res_job.Priority), 0, ITEM_DEFAULT, 10}, + {"allowmixedpriority", store_bool, ITEM(res_job.allow_mixed_priority), 0, ITEM_DEFAULT, false}, {"writepartafterjob", store_bool, ITEM(res_job.write_part_after_job), 0, ITEM_DEFAULT, true}, {"selectionpattern", store_str, ITEM(res_job.selection_pattern), 0, 0, 0}, {"runscript", store_runscript, ITEM(res_job.RunScripts), 0, ITEM_NO_EQUALS, 0}, diff --git a/bacula/src/dird/dird_conf.h b/bacula/src/dird/dird_conf.h index 9a3fabbe91..a44cc64523 100644 --- a/bacula/src/dird/dird_conf.h +++ b/bacula/src/dird/dird_conf.h @@ -398,6 +398,7 @@ public: int64_t spool_size; /* Size of spool file for this job */ uint32_t MaxConcurrentJobs; /* Maximum concurrent jobs */ uint32_t NumConcurrentJobs; /* number of concurrent jobs running */ + bool allow_mixed_priority; /* Allow jobs with higher priority concurrently with this */ MSGS *messages; /* How and where to send messages */ SCHED *schedule; /* When -- Automatic schedule */ diff --git a/bacula/src/dird/jobq.c b/bacula/src/dird/jobq.c index bcbe85b030..1e7fddc844 100644 --- a/bacula/src/dird/jobq.c +++ b/bacula/src/dird/jobq.c @@ -504,11 +504,26 @@ void *jobq_server(void *arg) Dmsg0(2300, "Done check ready, now check wait queue.\n"); if (!jq->waiting_jobs->empty() && !jq->quit) { int Priority; + bool running_allow_mix = false; je = (jobq_item_t *)jq->waiting_jobs->first(); jobq_item_t *re = (jobq_item_t *)jq->running_jobs->first(); if (re) { Priority = re->jcr->JobPriority; - Dmsg2(2300, "JobId %d is running. Look for pri=%d\n", re->jcr->JobId, Priority); + Dmsg2(2300, "JobId %d is running. Look for pri=%d\n", + re->jcr->JobId, Priority); + running_allow_mix = true; + for ( ; re; ) { + Dmsg2(2300, "JobId %d is also running with %s\n", + re->jcr->JobId, + re->jcr->job->allow_mixed_priority ? "mix" : "no mix"); + if (!re->jcr->job->allow_mixed_priority) { + running_allow_mix = false; + break; + } + re = (jobq_item_t *)jq->running_jobs->next(re); + } + Dmsg1(2300, "The running job(s) %s mixing priorities.\n", + running_allow_mix ? "allow" : "don't allow"); } else { Priority = je->jcr->JobPriority; Dmsg1(2300, "No job running. Look for Job pri=%d\n", Priority); @@ -522,11 +537,14 @@ void *jobq_server(void *arg) JCR *jcr = je->jcr; jobq_item_t *jn = (jobq_item_t *)jq->waiting_jobs->next(je); - Dmsg3(2300, "Examining Job=%d JobPri=%d want Pri=%d\n", - jcr->JobId, jcr->JobPriority, Priority); + Dmsg4(2300, "Examining Job=%d JobPri=%d want Pri=%d (%s)\n", + jcr->JobId, jcr->JobPriority, Priority, + jcr->job->allow_mixed_priority ? "mix" : "no mix"); /* Take only jobs of correct Priority */ - if (jcr->JobPriority != Priority) { + if (!(jcr->JobPriority == Priority + || (jcr->JobPriority < Priority && + jcr->job->allow_mixed_priority && running_allow_mix))) { set_jcr_job_status(jcr, JS_WaitPriority); break; } diff --git a/bacula/src/dird/ua_restore.c b/bacula/src/dird/ua_restore.c index b7b80569b8..7e4d8d3183 100644 --- a/bacula/src/dird/ua_restore.c +++ b/bacula/src/dird/ua_restore.c @@ -987,6 +987,39 @@ static void split_path_and_filename(UAContext *ua, RESTORE_CTX *rx, char *name) Dmsg2(100, "split path=%s file=%s\n", rx->path, rx->fname); } +static bool ask_for_fileregex(UAContext *ua, RESTORE_CTX *rx) +{ + ua->send_msg(_("\nThere were no files inserted into the tree, so file selection\n" + "is not possible.Most likely your retention policy pruned the files\n")); + if (get_yesno(ua, _("\nDo you want to restore all the files? (yes|no): "))) { + if (ua->pint32_val == 1) + return true; + while (get_cmd(ua, _("\nRegexp matching files to restore? (empty to abort): "))) { + if (ua->cmd[0] == '\0') { + break; + } else { + regex_t *fileregex_re = NULL; + int rc; + char errmsg[500] = ""; + + fileregex_re = (regex_t *)bmalloc(sizeof(regex_t)); + rc = regcomp(fileregex_re, ua->cmd, REG_EXTENDED|REG_NOSUB); + if (rc != 0) + regerror(rc, fileregex_re, errmsg, sizeof(errmsg)); + regfree(fileregex_re); + free(fileregex_re); + if (*errmsg) { + ua->send_msg(_("Regex compile error: %s\n"), errmsg); + } else { + rx->bsr->fileregex = bstrdup(ua->cmd); + return true; + } + } + } + } + return false; +} + static bool build_directory_tree(UAContext *ua, RESTORE_CTX *rx) { TREE_CTX tree; @@ -1048,19 +1081,15 @@ static bool build_directory_tree(UAContext *ua, RESTORE_CTX *rx) } #endif if (tree.FileCount == 0) { - ua->send_msg(_("\nThere were no files inserted into the tree, so file selection\n" - "is not possible.Most likely your retention policy pruned the files\n")); - if (!get_yesno(ua, _("\nDo you want to restore all the files? (yes|no): "))) { - OK = false; - } else { + OK = ask_for_fileregex(ua, rx); + if (OK) { last_JobId = 0; for (p=rx->JobIds; get_next_jobid_from_list(&p, &JobId) > 0; ) { if (JobId == last_JobId) { continue; /* eliminate duplicate JobIds */ } add_findex_all(rx->bsr, JobId); - } - OK = true; + } } } else { char ec1[50]; diff --git a/bacula/src/stored/bsr.h b/bacula/src/stored/bsr.h index c8fa67cc37..4782aec87e 100644 --- a/bacula/src/stored/bsr.h +++ b/bacula/src/stored/bsr.h @@ -38,6 +38,12 @@ #ifndef __BSR_H #define __BSR_H 1 +#ifndef HAVE_REGEX_H +#include "lib/bregex.h" +#else +#include +#endif + /* * List of Volume names to be read by Storage daemon. * Formed by Storage daemon from BSR @@ -143,6 +149,7 @@ struct BSR { bool done; /* set when everything found for this bsr */ bool use_fast_rejection; /* set if fast rejection can be used */ bool use_positioning; /* set if we can position the archive */ + bool skip_file; /* skip all records for current file */ BSR_VOLUME *volume; uint32_t count; /* count of files to restore this bsr */ uint32_t found; /* count of restored files this bsr */ @@ -157,6 +164,9 @@ struct BSR { BSR_JOBTYPE *JobType; BSR_JOBLEVEL *JobLevel; BSR_STREAM *stream; + char *fileregex; /* set if restore is filtered on filename */ + regex_t *fileregex_re; + ATTR *attr; /* scratch space for unpacking */ }; diff --git a/bacula/src/stored/match_bsr.c b/bacula/src/stored/match_bsr.c index 350de85a84..0f950e903e 100644 --- a/bacula/src/stored/match_bsr.c +++ b/bacula/src/stored/match_bsr.c @@ -74,7 +74,7 @@ static int match_jobid(BSR *bsr, BSR_JOBID *jobid, SESSION_LABEL *sessrec, bool static int match_findex(BSR *bsr, BSR_FINDEX *findex, DEV_RECORD *rec, bool done); static int match_volfile(BSR *bsr, BSR_VOLFILE *volfile, DEV_RECORD *rec, bool done); static int match_stream(BSR *bsr, BSR_STREAM *stream, DEV_RECORD *rec, bool done); -static int match_all(BSR *bsr, DEV_RECORD *rec, VOLUME_LABEL *volrec, SESSION_LABEL *sessrec, bool done); +static int match_all(BSR *bsr, DEV_RECORD *rec, VOLUME_LABEL *volrec, SESSION_LABEL *sessrec, bool done, JCR *jcr); static int match_block_sesstime(BSR *bsr, BSR_SESSTIME *sesstime, DEV_BLOCK *block); static int match_block_sessid(BSR *bsr, BSR_SESSID *sessid, DEV_BLOCK *block); static BSR *find_smallest_volfile(BSR *fbsr, BSR *bsr); @@ -148,6 +148,33 @@ static int match_block_sessid(BSR *bsr, BSR_SESSID *sessid, DEV_BLOCK *block) return 0; } +static int match_fileregex(BSR *bsr, DEV_RECORD *rec, JCR *jcr) +{ + if (bsr->fileregex_re == NULL) + return 1; + + if (bsr->attr == NULL) + bsr->attr = new_attr(jcr); + + /* The code breaks if the first record associated with a file is + * not of this type + */ + if (rec->Stream == STREAM_UNIX_ATTRIBUTES || + rec->Stream == STREAM_UNIX_ATTRIBUTES_EX) { + bsr->skip_file = false; + if (unpack_attributes_record(jcr, rec->Stream, rec->data, bsr->attr)) { + if (regexec(bsr->fileregex_re, bsr->attr->fname, 0, NULL, 0) == 0) { + Dmsg2(dbglevel, "Matched pattern, fname=%s FI=%d\n", + bsr->attr->fname, rec->FileIndex); + } else { + Dmsg2(dbglevel, "Didn't match, skipping fname=%s FI=%d\n", + bsr->attr->fname, rec->FileIndex); + bsr->skip_file = true; + } + } + } + return 1; +} /********************************************************************* * @@ -157,7 +184,7 @@ static int match_block_sessid(BSR *bsr, BSR_SESSID *sessid, DEV_BLOCK *block) * reposition the tape * returns -1 no additional matches possible */ -int match_bsr(BSR *bsr, DEV_RECORD *rec, VOLUME_LABEL *volrec, SESSION_LABEL *sessrec) +int match_bsr(BSR *bsr, DEV_RECORD *rec, VOLUME_LABEL *volrec, SESSION_LABEL *sessrec, JCR *jcr) { int stat; @@ -168,7 +195,7 @@ int match_bsr(BSR *bsr, DEV_RECORD *rec, VOLUME_LABEL *volrec, SESSION_LABEL *se */ if (bsr) { bsr->reposition = false; - stat = match_all(bsr, rec, volrec, sessrec, true); + stat = match_all(bsr, rec, volrec, sessrec, true, jcr); /* * Note, bsr->reposition is set by match_all when * a bsr is done. We turn it off if a match was @@ -328,7 +355,7 @@ bool is_this_bsr_done(BSR *bsr, DEV_RECORD *rec) * returns -1 no additional matches possible */ static int match_all(BSR *bsr, DEV_RECORD *rec, VOLUME_LABEL *volrec, - SESSION_LABEL *sessrec, bool done) + SESSION_LABEL *sessrec, bool done, JCR *jcr) { if (bsr->done) { // Dmsg0(dbglevel, "bsr->done set\n"); @@ -363,6 +390,18 @@ static int match_all(BSR *bsr, DEV_RECORD *rec, VOLUME_LABEL *volrec, rec->FileIndex, bsr->FileIndex->findex, bsr->FileIndex->findex2); goto no_match; } + + if (!match_fileregex(bsr, rec, jcr)) { + Dmsg1(dbglevel, "Fail on fileregex='%s'\n", bsr->fileregex); + goto no_match; + } + + /* This flag is set by match_fileregex (and perhaps other tests) */ + if (bsr->skip_file) { + Dmsg1(dbglevel, "Skipping findex=%d\n", rec->FileIndex); + goto no_match; + } + /* * If a count was specified and we have a FileIndex, assume * it is a Bacula created bsr (or the equivalent). We @@ -410,7 +449,7 @@ static int match_all(BSR *bsr, DEV_RECORD *rec, VOLUME_LABEL *volrec, no_match: if (bsr->next) { - return match_all(bsr->next, rec, volrec, sessrec, bsr->done && done); + return match_all(bsr->next, rec, volrec, sessrec, bsr->done && done, jcr); } if (bsr->done && done) { return -1; diff --git a/bacula/src/stored/parse_bsr.c b/bacula/src/stored/parse_bsr.c index 83b0ff4142..12f43c54c6 100644 --- a/bacula/src/stored/parse_bsr.c +++ b/bacula/src/stored/parse_bsr.c @@ -57,6 +57,7 @@ static BSR *store_include(LEX *lc, BSR *bsr); static BSR *store_exclude(LEX *lc, BSR *bsr); static BSR *store_stream(LEX *lc, BSR *bsr); static BSR *store_slot(LEX *lc, BSR *bsr); +static BSR *store_fileregex(LEX *lc, BSR *bsr); static bool is_fast_rejection_ok(BSR *bsr); static bool is_positioning_ok(BSR *bsr); @@ -87,8 +88,8 @@ struct kw_items items[] = { {"stream", store_stream}, {"slot", store_slot}, {"device", store_device}, + {"fileregex", store_fileregex}, {NULL, NULL} - }; /* @@ -445,6 +446,32 @@ static BSR *store_count(LEX *lc, BSR *bsr) return bsr; } +static BSR *store_fileregex(LEX *lc, BSR *bsr) +{ + int token; + int rc; + + token = lex_get_token(lc, T_STRING); + if (token == T_ERROR) { + return NULL; + } + + if (bsr->fileregex) free(bsr->fileregex); + bsr->fileregex = bstrdup(lc->str); + + if (bsr->fileregex_re == NULL) + bsr->fileregex_re = (regex_t *)bmalloc(sizeof(regex_t)); + + rc = regcomp(bsr->fileregex_re, bsr->fileregex, REG_EXTENDED|REG_NOSUB); + if (rc != 0) { + char prbuf[500]; + regerror(rc, bsr->fileregex_re, prbuf, sizeof(prbuf)); + Emsg2(M_ERROR, 0, _("REGEX '%s' compile error. ERR=%s\n"), + bsr->fileregex, prbuf); + return NULL; + } + return bsr; +} static BSR *store_jobtype(LEX *lc, BSR *bsr) { @@ -817,6 +844,13 @@ void free_bsr(BSR *bsr) free_bsr_item((BSR *)bsr->FileIndex); free_bsr_item((BSR *)bsr->JobType); free_bsr_item((BSR *)bsr->JobLevel); + if (bsr->fileregex) bfree(bsr->fileregex); + if (bsr->fileregex_re) { + regfree(bsr->fileregex_re); + free(bsr->fileregex_re); + } + if (bsr->attr) free_attr(bsr->attr); + free_bsr(bsr->next); free(bsr); } diff --git a/bacula/src/stored/protos.h b/bacula/src/stored/protos.h index 019b3f2d04..05bb49c114 100644 --- a/bacula/src/stored/protos.h +++ b/bacula/src/stored/protos.h @@ -178,7 +178,7 @@ void _give_back_device_lock(const char *file, int line, DEVICE *dev, bsteal_ /* From match_bsr.c */ int match_bsr(BSR *bsr, DEV_RECORD *rec, VOLUME_LABEL *volrec, - SESSION_LABEL *sesrec); + SESSION_LABEL *sesrec, JCR *jcr); int match_bsr_block(BSR *bsr, DEV_BLOCK *block); void position_bsr_block(BSR *bsr, DEV_BLOCK *block); BSR *find_next_bsr(BSR *root_bsr, DEVICE *dev); diff --git a/bacula/src/stored/read_record.c b/bacula/src/stored/read_record.c index a5b97b4eda..9dbbf58ad7 100644 --- a/bacula/src/stored/read_record.c +++ b/bacula/src/stored/read_record.c @@ -255,7 +255,7 @@ bool read_records(DCR *dcr, * Apply BSR filter */ if (jcr->bsr) { - rec->match_stat = match_bsr(jcr->bsr, rec, &dev->VolHdr, &sessrec); + rec->match_stat = match_bsr(jcr->bsr, rec, &dev->VolHdr, &sessrec, jcr); if (rec->match_stat == -1) { /* no more possible matches */ done = true; /* all items found, stop */ Dmsg2(dbglvl, "All done=(file:block) %u:%u\n", dev->file, dev->block_num); @@ -278,6 +278,7 @@ bool read_records(DCR *dcr, rec->VolSessionId, rec->VolSessionTime, rec->FileIndex); break; /* read second part of record */ } + Dmsg6(dbglvl, "OK callback. recno=%d state=%s blk=%d SI=%d ST=%d FI=%d\n", record, rec_state_to_str(rec), block->BlockNumber, rec->VolSessionId, rec->VolSessionTime, rec->FileIndex); diff --git a/bacula/src/version.h b/bacula/src/version.h index d431a7ab0f..ad960d9424 100644 --- a/bacula/src/version.h +++ b/bacula/src/version.h @@ -3,9 +3,9 @@ */ #undef VERSION -#define VERSION "2.5.11" -#define BDATE "04 October 2008" -#define LSMDATE "04Oct08" +#define VERSION "2.5.12" +#define BDATE "05 October 2008" +#define LSMDATE "05Oct08" #define PROG_COPYRIGHT "Copyright (C) %d-2008 Free Software Foundation Europe e.V.\n" #define BYEAR "2008" /* year for copyright messages in progs */ diff --git a/bacula/technotes-2.5 b/bacula/technotes-2.5 index 5f461c97bc..6cdac4177a 100644 --- a/bacula/technotes-2.5 +++ b/bacula/technotes-2.5 @@ -18,6 +18,10 @@ remove reader/writer in FOPTS???? General: 05Oct08 +kes Integrate patch from Kjetil Torgrim Homme + It provides FileRegex in bsr files and code to collect regex + from the user during restore, an Allow Mixed Priorities feature, + and documentation for the above. ebl Remove db berkeley from configure process kes Do dirty check on hostname (Linux only) and if not resolvable use localhost as default.