From 7278876d2118b38ce09ed70f39650a8997415495 Mon Sep 17 00:00:00 2001 From: Kern Sibbald Date: Thu, 29 Apr 2004 21:43:36 +0000 Subject: [PATCH] Implement new style filesets git-svn-id: https://bacula.svn.sourceforge.net/svnroot/bacula/trunk@1325 91ce42f0-d328-0410-95d8-f526ca767f89 --- bacula/src/dird/dird_conf.h | 1 + bacula/src/dird/fd_cmds.c | 17 ++-- bacula/src/dird/inc_conf.c | 2 +- bacula/src/filed/job.c | 174 ++++++++++++++++++++++++++++++++++-- bacula/src/findlib/find.c | 101 ++++++++++++++++++++- bacula/src/findlib/find.h | 42 +++++++++ 6 files changed, 319 insertions(+), 18 deletions(-) diff --git a/bacula/src/dird/dird_conf.h b/bacula/src/dird/dird_conf.h index 7d8b0d6b3e..58eb7e0058 100644 --- a/bacula/src/dird/dird_conf.h +++ b/bacula/src/dird/dird_conf.h @@ -239,6 +239,7 @@ struct JOB { uint32_t NumConcurrentJobs; /* number of concurrent jobs running */ }; +#undef MAX_FOPTS #define MAX_FOPTS 30 /* File options structure */ diff --git a/bacula/src/dird/fd_cmds.c b/bacula/src/dird/fd_cmds.c index e43a87df54..c60c2d3420 100644 --- a/bacula/src/dird/fd_cmds.c +++ b/bacula/src/dird/fd_cmds.c @@ -38,7 +38,7 @@ /* Commands sent to File daemon */ static char inc[] = "include\n"; static char exc[] = "exclude\n"; -static char incopts[] = "incopts\n"; /* new include with options */ +static char fileset[] = "fileset\n"; /* set full fileset */ static char jobcmd[] = "JobId=%d Job=%s SDid=%u SDtime=%u Authorization=%s\n"; static char levelcmd[] = "level = %s%s mtime_only=%d\n"; static char runbefore[] = "RunBeforeJob %s\n"; @@ -231,7 +231,7 @@ static int send_list(JCR *jcr, int list) num = fileset->num_excludes; } - for (int i=0; i < num; i++) { + for (int i=0; ifileset; BSOCK *fd = jcr->file_bsock; @@ -362,6 +362,7 @@ static int send_incopts(JCR *jcr) int j, k; ie = fileset->include_items[i]; + bnet_fsend(fd, "I\n"); for (j=0; jnum_opts; j++) { FOPTS *fo = ie->opts_list[j]; @@ -390,7 +391,7 @@ static int send_incopts(JCR *jcr) p, strerror(errno)); goto bail_out; } - bstrncpy(buf, "I ", sizeof(buf)); + bstrncpy(buf, "F ", sizeof(buf)); Dmsg1(100, "Opts=%s\n", buf); optlen = strlen(buf); while (fgets(buf+optlen, sizeof(buf)-optlen, bpipe->rfd)) { @@ -414,7 +415,7 @@ static int send_incopts(JCR *jcr) p, strerror(errno)); goto bail_out; } - bstrncpy(buf, "I ", sizeof(buf)); + bstrncpy(buf, "F ", sizeof(buf)); Dmsg1(100, "Opts=%s\n", buf); optlen = strlen(buf); while (fgets(buf+optlen, sizeof(buf)-optlen, ffd)) { @@ -430,7 +431,7 @@ static int send_incopts(JCR *jcr) p++; /* skip over \ */ /* Note, fall through wanted */ default: - pm_strcpy(&fd->msg, "I "); + pm_strcpy(&fd->msg, "F "); fd->msglen = pm_strcat(&fd->msg, p); Dmsg1(100, "Inc/Exc name=%s\n", fd->msg); if (!bnet_send(fd)) { @@ -463,8 +464,8 @@ int send_include_list(JCR *jcr) { BSOCK *fd = jcr->file_bsock; if (jcr->fileset->new_include) { - bnet_fsend(fd, incopts); - return send_incopts(jcr); + bnet_fsend(fd, fileset); + return send_fileset(jcr); } else { bnet_fsend(fd, inc); } diff --git a/bacula/src/dird/inc_conf.c b/bacula/src/dird/inc_conf.c index 3e9e3d5f74..998b74e745 100644 --- a/bacula/src/dird/inc_conf.c +++ b/bacula/src/dird/inc_conf.c @@ -366,7 +366,7 @@ static void store_newinc(LEX *lc, RES_ITEM *item, int index, int pass) res_all.res_fs.have_MD5 = TRUE; } memset(&res_incexe, 0, sizeof(INCEXE)); - res_all.res_fs.new_include = TRUE; + res_all.res_fs.new_include = true; while ((token = lex_get_token(lc, T_ALL)) != T_EOF) { if (token == T_EOL) { continue; diff --git a/bacula/src/filed/job.c b/bacula/src/filed/job.c index abcb6f24e5..20f49058b8 100644 --- a/bacula/src/filed/job.c +++ b/bacula/src/filed/job.c @@ -45,7 +45,7 @@ static int exclude_cmd(JCR *jcr); static int hello_cmd(JCR *jcr); static int job_cmd(JCR *jcr); static int include_cmd(JCR *jcr); -static int incopts_cmd(JCR *jcr); +static int fileset_cmd(JCR *jcr); static int level_cmd(JCR *jcr); static int verify_cmd(JCR *jcr); static int restore_cmd(JCR *jcr); @@ -78,7 +78,7 @@ static struct s_cmds cmds[] = { {"exclude", exclude_cmd}, {"Hello", hello_cmd}, {"include", include_cmd}, - {"incopts", incopts_cmd}, + {"fileset", fileset_cmd}, {"JobId=", job_cmd}, {"level = ", level_cmd}, {"restore", restore_cmd}, @@ -219,6 +219,25 @@ void *handle_client_request(void *dirp) /* Inform Director that we are done */ bnet_sig(dir, BNET_TERMINATE); + /* Clean up fileset */ + FF_PKT *ff = (FF_PKT *)jcr->ff; + findFILESET *fileset = ff->fileset; + if (fileset) { + int i, j; + for (i=0; iinclude_list.size(); i++) { + findINCEXE *incexe = (findINCEXE *)fileset->include_list.get(i); + for (j=0; jopts_list.size(); j++) { + findFOPTS *fo = (findFOPTS *)incexe->opts_list.get(j); + fo->regex.destroy(); + fo->wild.destroy(); + fo->base.destroy(); + } + incexe->opts_list.destroy(); + incexe->name_list.destroy(); + } + fileset->include_list.destroy(); + free(fileset); + } Dmsg0(100, "Calling term_find_files\n"); term_find_files((FF_PKT *)jcr->ff); Dmsg0(100, "Done with term_find_files\n"); @@ -527,17 +546,156 @@ static int include_cmd(JCR *jcr) return bnet_fsend(dir, OKinc); } -static int incopts_cmd(JCR *jcr) +static bool init_fileset(JCR *jcr) +{ + FF_PKT *ff; + findFILESET *fileset; + + if (!jcr->ff) { + return false; + } + ff = (FF_PKT *)jcr->ff; + if (ff->fileset) { + return false; + } + fileset = (findFILESET *)malloc(sizeof(findFILESET)); + memset(fileset, 0, sizeof(findFILESET)); + ff->fileset = fileset; + fileset->state = state_none; + fileset->include_list.init(1, true); + return true; +} + +static findFOPTS *set_options(FF_PKT *ff) +{ + int state = ff->fileset->state; + findINCEXE *incexe = ff->fileset->incexe; + + if (state != state_options) { + ff->fileset->state = state_options; + findFOPTS *fo = (findFOPTS *)malloc(sizeof(findFOPTS)); + memset(fo, 0, sizeof(findFOPTS)); + fo->regex.init(1, true); + fo->wild.init(1, true); + fo->base.init(1, true); + incexe->current_opts = fo; + incexe->opts_list.append(fo); + } + return incexe->current_opts; + +} + + +static void add_fileset(JCR *jcr, const char *item) +{ + FF_PKT *ff = (FF_PKT *)jcr->ff; + findFILESET *fileset = ff->fileset; + int state = fileset->state; + findFOPTS *current_opts; + int code = item[0]; + if (item[1] == ' ') { /* If string follows */ + item += 2; /* point to string */ + } + + if (state == state_error) { + return; + } + switch (code) { + case 'I': + /* New include */ + fileset->incexe = (findINCEXE *)malloc(sizeof(findINCEXE)); + memset(fileset->incexe, 0, sizeof(findINCEXE)); + fileset->incexe->opts_list.init(1, true); + fileset->incexe->name_list.init(1, true); + fileset->include_list.append(fileset->incexe); + break; + case 'N': + state = state_none; + break; + case 'F': + /* File item */ + state = state_include; + fileset->incexe->name_list.append(bstrdup(item)); + break; + case 'R': + current_opts = set_options(ff); + current_opts->regex.append(bstrdup(item)); + state = state_options; + break; + case 'B': + current_opts = set_options(ff); + current_opts->base.append(bstrdup(item)); + state = state_options; + break; + case 'W': + current_opts = set_options(ff); + current_opts->wild.append(bstrdup(item)); + state = state_options; + break; + case 'O': + current_opts = set_options(ff); + bstrncpy(current_opts->opts, item, MAX_FOPTS); + state = state_options; + break; + default: + Jmsg(jcr, M_FATAL, 0, "Invalid FileSet command: %s\n", item); + state = state_error; + break; + } + ff->fileset->state = state; +} + +static bool term_fileset(JCR *jcr) +{ + FF_PKT *ff = (FF_PKT *)jcr->ff; + findFILESET *fileset = ff->fileset; + int i, j, k; + + for (i=0; iinclude_list.size(); i++) { + findINCEXE *incexe = (findINCEXE *)fileset->include_list.get(i); + for (j=0; jopts_list.size(); j++) { + findFOPTS *fo = (findFOPTS *)incexe->opts_list.get(j); + Dmsg1(400, "O %s\n", fo->opts); + for (k=0; kregex.size(); k++) { + Dmsg1(400, "R %s\n", (char *)fo->regex.get(k)); + } + for (k=0; kwild.size(); k++) { + Dmsg1(400, "W %s\n", (char *)fo->wild.get(k)); + } + for (k=0; kbase.size(); k++) { + Dmsg1(400, "B %s\n", (char *)fo->base.get(k)); + } + } + for (j=0; jname_list.size(); j++) { + Dmsg1(400, "F %s\n", (char *)incexe->name_list.get(j)); + } + + } + + + return ff->fileset->state != state_error; +} + + + +/* + * Director is passing his Fileset + */ +static int fileset_cmd(JCR *jcr) { BSOCK *dir = jcr->dir_bsock; + if (!init_fileset(jcr)) { + return 0; + } while (bnet_recv(dir) >= 0) { strip_trailing_junk(dir->msg); - Dmsg1(000, "Incopt: %s\n", dir->msg); -// add_fname_to_list(jcr, dir->msg, INC_LIST); - + Dmsg1(400, "Fileset: %s\n", dir->msg); + add_fileset(jcr, dir->msg); + } + if (!term_fileset(jcr)) { + return 0; } - return bnet_fsend(dir, OKinc); } @@ -1139,6 +1297,8 @@ static void filed_free_jcr(JCR *jcr) if (jcr->RunAfterJob) { free_pool_memory(jcr->RunAfterJob); } + + return; } diff --git a/bacula/src/findlib/find.c b/bacula/src/findlib/find.c index 9db0ed8c18..43fbca9581 100644 --- a/bacula/src/findlib/find.c +++ b/bacula/src/findlib/find.c @@ -36,6 +36,7 @@ int32_t path_max; /* path name max length */ /* ****FIXME**** debug until stable */ #undef bmalloc #define bmalloc(x) sm_malloc(__FILE__, __LINE__, x) +static void set_options(FF_PKT *ff, const char *opts); /* @@ -105,20 +106,116 @@ find_files(JCR *jcr, FF_PKT *ff, int callback(FF_PKT *ff_pkt, void *hpkt), void { struct s_included_file *inc = NULL; + /* This is the old deprecated way */ while (!job_canceled(jcr) && (inc = get_next_included_file(ff, inc))) { /* Copy options for this file */ bstrncpy(ff->VerifyOpts, inc->VerifyOpts, sizeof(ff->VerifyOpts)); Dmsg1(50, "find_files: file=%s\n", inc->fname); if (!file_is_excluded(ff, inc->fname)) { - if (!find_one_file(jcr, ff, callback, his_pkt, inc->fname, - (dev_t)-1, 1)) { + if (!find_one_file(jcr, ff, callback, his_pkt, inc->fname, (dev_t)-1, 1)) { return 0; /* error return */ } } } + + /* This is the new way */ + findFILESET *fileset = ff->fileset; + if (fileset) { + int i, j; + for (i=0; iinclude_list.size(); i++) { + findINCEXE *incexe = (findINCEXE *)fileset->include_list.get(i); + /* + * By setting all options, we in effect or the global options + * which is what we want. + */ + for (j=0; jopts_list.size(); j++) { + findFOPTS *fo = (findFOPTS *)incexe->opts_list.get(j); + Dmsg1(400, "Find global options O %s\n", fo->opts); + set_options(ff, fo->opts); + } + for (j=0; jname_list.size(); j++) { + Dmsg1(400, "F %s\n", (char *)incexe->name_list.get(j)); + char *fname = (char *)incexe->name_list.get(j); + if (!find_one_file(jcr, ff, callback, his_pkt, fname, (dev_t)-1, 1)) { + return 0; /* error return */ + } + } + } + } return 1; } +/* + * As an optimization, we should do this during + * "compile" time in filed/job.c, and keep only a bit mask + * and the Verify options. + */ +static void set_options(FF_PKT *ff, const char *opts) +{ + int j; + const char *p; + + for (p=opts; *p; p++) { + switch (*p) { + case 'a': /* alway replace */ + case '0': /* no option */ + break; + case 'f': + ff->flags |= FO_MULTIFS; + break; + case 'h': /* no recursion */ + ff->flags |= FO_NO_RECURSION; + break; + case 'M': /* MD5 */ + ff->flags |= FO_MD5; + break; + case 'n': + ff->flags |= FO_NOREPLACE; + break; + case 'p': /* use portable data format */ + ff->flags |= FO_PORTABLE; + break; + case 'r': /* read fifo */ + ff->flags |= FO_READFIFO; + break; + case 'S': + ff->flags |= FO_SHA1; + break; + case 's': + ff->flags |= FO_SPARSE; + break; + case 'm': + ff->flags |= FO_MTIMEONLY; + break; + case 'k': + ff->flags |= FO_KEEPATIME; + break; + case 'V': /* verify options */ + /* Copy Verify Options */ + for (j=0; *p && *p != ':'; p++) { + ff->VerifyOpts[j] = *p; + if (j < (int)sizeof(ff->VerifyOpts) - 1) { + j++; + } + } + ff->VerifyOpts[j] = 0; + break; + case 'w': + ff->flags |= FO_IF_NEWER; + break; + case 'Z': /* gzip compression */ + ff->flags |= FO_GZIP; + ff->GZIP_level = *++p - '0'; + Dmsg1(200, "Compression level=%d\n", ff->GZIP_level); + break; + default: + Emsg1(M_ERROR, 0, "Unknown include/exclude option: %c\n", *p); + break; + } + } +} + + /* * Terminate find_files() and release * all allocated memory diff --git a/bacula/src/findlib/find.h b/bacula/src/findlib/find.h index 3f3b1b900d..9c2ef7dbde 100755 --- a/bacula/src/findlib/find.h +++ b/bacula/src/findlib/find.h @@ -102,6 +102,47 @@ struct s_excluded_file { char fname[1]; }; +/* FileSet definitions very similar to the resource + * contained in the Director because the components + * of the structure are passed by the Director to the + * File daemon and recompiled back into this structure + */ +#undef MAX_FOPTS +#define MAX_FOPTS 30 + +enum { + state_none, + state_options, + state_include, + state_error +}; + +/* File options structure */ +struct findFOPTS { + char opts[MAX_FOPTS]; /* options string */ + alist regex; /* regex string(s) */ + alist wild; /* wild card strings */ + alist base; /* list of base names */ +}; + + +/* This is either an include item or an exclude item */ +struct findINCEXE { + findFOPTS *current_opts; /* points to current options structure */ + alist opts_list; /* options list */ + alist name_list; /* filename list -- holds char * */ +}; + +/* + * FileSet Resource + * + */ +struct findFILESET { + int state; + findINCEXE *incexe; /* current item */ + alist include_list; +}; + /* * Definition of the find_files packet passed as the @@ -128,6 +169,7 @@ struct FF_PKT { struct s_included_file *included_files_list; struct s_excluded_file *excluded_files_list; struct s_excluded_file *excluded_paths_list; + findFILESET *fileset; /* List of all hard linked files found */ struct f_link *linklist; /* hard linked files */ -- 2.39.5