X-Git-Url: https://git.sur5r.net/?a=blobdiff_plain;f=bacula%2Fsrc%2Fdird%2Fdird.c;h=c1c1b151a0bb08dbe73d8972033ce08d840f4ed1;hb=baa7ca256d3146c8991d27745825af2817ffddbc;hp=257e127bf56f6b4966e90af95078d3cfbc941364;hpb=d1cfa762a333e159b88a3eb49d1bf228770cf9d8;p=bacula%2Fbacula diff --git a/bacula/src/dird/dird.c b/bacula/src/dird/dird.c index 257e127bf5..c1c1b151a0 100644 --- a/bacula/src/dird/dird.c +++ b/bacula/src/dird/dird.c @@ -1,60 +1,48 @@ /* - Bacula® - The Network Backup Solution - - Copyright (C) 2000-2009 Free Software Foundation Europe e.V. - - The main author of Bacula is Kern Sibbald, with contributions from - many others, a complete list can be found in the file AUTHORS. - This program is Free Software; you can redistribute it and/or - modify it under the terms of version two of the GNU General Public - License as published by the Free Software Foundation and included - in the file LICENSE. - - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - 02110-1301, USA. - - Bacula® is a registered trademark of Kern Sibbald. - The licensor of Bacula is the Free Software Foundation Europe - (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich, - Switzerland, email:ftf@fsfeurope.org. + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. */ /* - * * Bacula Director daemon -- this is the main program * * Kern Sibbald, March MM - * - * Version $Id$ */ #include "bacula.h" #include "dird.h" - -#ifdef HAVE_PYTHON - -#undef _POSIX_C_SOURCE -#include - -#include "lib/pythonlib.h" - -/* Imported Functions */ -extern PyObject *job_getattr(PyObject *self, char *attrname); -extern int job_setattr(PyObject *self, char *attrname, PyObject *value); - -#endif /* HAVE_PYTHON */ +#ifndef HAVE_REGEX_H +#include "lib/bregex.h" +#else +#include +#endif +#ifdef HAVE_DIRENT_H +#include +#define NAMELEN(dirent) (strlen((dirent)->d_name)) +#endif +#ifndef HAVE_READDIR_R +int readdir_r(DIR *dirp, struct dirent *entry, struct dirent **result); +#endif /* Forward referenced subroutines */ void terminate_dird(int sig); static bool check_resources(); -static void dir_sql_query(JCR *jcr, const char *cmd); - +static void cleanup_old_files(); +static void resize_reload(int nb); + /* Exported subroutines */ extern "C" void reload_config(int sig); extern void invalidate_schedules(); @@ -73,19 +61,28 @@ void store_replace(LEX *lc, RES_ITEM *item, int index, int pass); void store_migtype(LEX *lc, RES_ITEM *item, int index, int pass); void init_device_resources(); + static char *runjob = NULL; -static int background = 1; +static bool foreground = false; +static bool make_pid_file = true; /* create pid file */ static void init_reload(void); static CONFIG *config; - +static bool test_config = false; + /* Globals Exported */ DIRRES *director; /* Director resource */ int FDConnectTimeout; int SDConnectTimeout; char *configfile = NULL; void *start_heap; +utime_t last_reload_time = 0; + /* Globals Imported */ +extern dlist client_globals; +extern dlist store_globals; +extern dlist job_globals; +extern dlist sched_globals; extern RES_ITEM job_items[]; #if defined(_MSC_VER) extern "C" { // work around visual compiler mangling variables @@ -104,33 +101,62 @@ static bool check_catalog(cat_op mode); #define CONFIG_FILE "bacula-dir.conf" /* default configuration file */ +static bool dir_sql_query(JCR *jcr, const char *cmd) +{ + if (jcr && jcr->db && jcr->db->is_connected()) { + return db_sql_query(jcr->db, cmd, NULL, NULL); + } + return false; +} + +static bool dir_sql_escape(JCR *jcr, BDB *mdb, char *snew, char *sold, int len) +{ + if (jcr && jcr->db && jcr->db->is_connected()) { + db_escape_string(jcr, mdb, snew, sold, len); + return true; + } + return false; +} static void usage() { fprintf(stderr, _( -PROG_COPYRIGHT -"\nVersion: %s (%s)\n\n" -"Usage: dird [-f -s] [-c config_file] [-d debug_level] [config_file]\n" -" -c set configuration file to file\n" -" -d set debug level to \n" -" -dt print timestamp in debug output\n" -" -f run in foreground (for debugging)\n" -" -g groupid\n" -" -r run now\n" -" -s no signals\n" -" -t test - read configuration and exit\n" -" -u userid\n" -" -v verbose user messages\n" -" -? print this message.\n" -"\n"), 2000, VERSION, BDATE); + PROG_COPYRIGHT + "\n%sVersion: %s (%s)\n\n" + "Usage: bacula-dir [-f -s] [-c config_file] [-d debug_level] [config_file]\n" + " -c set configuration file to file\n" + " -d [,] set debug level to , debug tags to \n" + " -dt print timestamp in debug output\n" + " -T set trace on\n" + " -f run in foreground (for debugging)\n" + " -g groupid\n" + " -m print kaboom output (for debugging)\n" + " -r run now\n" + " -P do not create pid file\n" + " -s no signals\n" + " -t test - read configuration and exit\n" + " -u userid\n" + " -v verbose user messages\n" + " -? print this message.\n" + "\n"), 2000, "", VERSION, BDATE); exit(1); } +/* + * !!! WARNING !!! Use this function only when bacula is stopped. + * ie, after a fatal signal and before exiting the program + * Print information about a JCR + */ +static void dir_debug_print(JCR *jcr, FILE *fp) +{ + fprintf(fp, "\twstore=%p rstore=%p wjcr=%p client=%p reschedule_count=%d SD_msg_chan_started=%d\n", + jcr->wstore, jcr->rstore, jcr->wjcr, jcr->client, jcr->reschedule_count, (int)jcr->SD_msg_chan_started); +} /********************************************************************* * - * Main Bacula Server program + * Main Bacula Director Server program * */ #if defined(HAVE_WIN32) @@ -138,17 +164,20 @@ PROG_COPYRIGHT #define main BaculaMain #endif +/* DELETE ME when bugs in MA1512, MA1632 MA1639 are fixed */ +extern void (*MA1512_reload_job_end_cb)(JCR *,void *); +static void reload_job_end_cb(JCR *jcr, void *ctx); + int main (int argc, char *argv[]) { int ch; JCR *jcr; bool no_signals = false; - bool test_config = false; char *uid = NULL; char *gid = NULL; -#ifdef HAVE_PYTHON - init_python_interpreter_args python_args; -#endif /* HAVE_PYTHON */ + + /* DELETE ME when bugs in MA1512, MA1632 MA1639 are fixed */ + MA1512_reload_job_end_cb = reload_job_end_cb; start_heap = sbrk(0); setlocale(LC_ALL, ""); @@ -160,10 +189,10 @@ int main (int argc, char *argv[]) init_msg(NULL, NULL); /* initialize message handler */ init_reload(); daemon_start_time = time(NULL); - + setup_daemon_message_queue(); console_command = run_console_command; - while ((ch = getopt(argc, argv, "c:d:fg:r:stu:v?")) != -1) { + while ((ch = getopt(argc, argv, "c:d:fg:mPr:stu:v?T")) != -1) { switch (ch) { case 'c': /* specify config file */ if (configfile != NULL) { @@ -176,22 +205,42 @@ int main (int argc, char *argv[]) if (*optarg == 't') { dbg_timestamp = true; } else { + char *p; + /* We probably find a tag list -d 10,sql,bvfs */ + if ((p = strchr(optarg, ',')) != NULL) { + *p = 0; + } debug_level = atoi(optarg); if (debug_level <= 0) { debug_level = 1; } + if (p) { + debug_parse_tags(p+1, &debug_level_tags); + } } - Dmsg1(10, "Debug level = %d\n", debug_level); + Dmsg1(10, "Debug level = %lld\n", debug_level); + break; + + case 'T': + set_trace(true); break; case 'f': /* run in foreground */ - background = FALSE; + foreground = true; break; case 'g': /* set group id */ gid = optarg; break; + case 'm': /* print kaboom output */ + prt_kaboom = true; + break; + + case 'P': /* no pid file */ + make_pid_file = false; + break; + case 'r': /* run job */ if (runjob != NULL) { free(runjob); @@ -226,10 +275,6 @@ int main (int argc, char *argv[]) argc -= optind; argv += optind; - if (!no_signals) { - init_signals(terminate_dird); - } - if (argc) { if (configfile != NULL) { free(configfile); @@ -242,11 +287,20 @@ int main (int argc, char *argv[]) usage(); } + if (!foreground && !test_config) { + daemon_start(); + init_stack_dump(); /* grab new pid */ + } + + if (!no_signals) { + init_signals(terminate_dird); + } + if (configfile == NULL) { configfile = bstrdup(CONFIG_FILE); } - config = new_config_parser(); + config = New(CONFIG()); parse_dir_config(config, configfile, M_ERROR_TERM); if (init_crypto() != 0) { @@ -257,17 +311,21 @@ int main (int argc, char *argv[]) Jmsg((JCR *)NULL, M_ERROR_TERM, 0, _("Please correct configuration file: %s\n"), configfile); } - if (!test_config) { /* we don't need to do this block in test mode */ - if (background) { - daemon_start(); - init_stack_dump(); /* grab new pid */ - } - + if (!test_config) { /* Create pid must come after we are a daemon -- so we have our final pid */ - create_pid_file(director->pid_directory, "bacula-dir", get_first_port_host_order(director->DIRaddrs)); - read_state_file(director->working_directory, "bacula-dir", get_first_port_host_order(director->DIRaddrs)); + if (make_pid_file) { + create_pid_file(director->pid_directory, "bacula-dir", + get_first_port_host_order(director->DIRaddrs)); + } + read_state_file(director->working_directory, "bacula-dir", + get_first_port_host_order(director->DIRaddrs)); } + set_jcr_in_tsd(INVALID_JCR); + set_thread_concurrency(director->MaxConcurrentJobs * 2 + + 4 /* UA */ + 5 /* sched+watchdog+jobsvr+misc */); + lmgr_init_thread(); /* initialize the lockmanager stack */ + load_dir_plugins(director->plugin_directory); drop(uid, gid, false); /* reduce privileges if requested */ @@ -278,42 +336,30 @@ int main (int argc, char *argv[]) if (!check_catalog(mode)) { Jmsg((JCR *)NULL, M_ERROR_TERM, 0, _("Please correct configuration file: %s\n"), configfile); } - - if (test_config) { + + if (test_config) { terminate_dird(0); } my_name_is(0, NULL, director->name()); /* set user defined name */ + cleanup_old_files(); + /* Plug database interface for library routines */ - p_sql_query = (sql_query)dir_sql_query; - p_sql_escape = (sql_escape)db_escape_string; + p_sql_query = (sql_query_call)dir_sql_query; + p_sql_escape = (sql_escape_call)dir_sql_escape; FDConnectTimeout = (int)director->FDConnectTimeout; SDConnectTimeout = (int)director->SDConnectTimeout; + resize_reload(director->MaxReload); + #if !defined(HAVE_WIN32) signal(SIGHUP, reload_config); #endif init_console_msg(working_directory); -#ifdef HAVE_PYTHON - python_args.progname = director->name(); - python_args.scriptdir = director->scripts_directory; - python_args.modulename = "DirStartUp"; - python_args.configfile = configfile; - python_args.workingdir = director->working_directory; - python_args.job_getattr = job_getattr; - python_args.job_setattr = job_setattr; - - init_python_interpreter(&python_args); -#endif /* HAVE_PYTHON */ - - set_jcr_in_tsd(INVALID_JCR); - set_thread_concurrency(director->MaxConcurrentJobs * 2 + - 4 /* UA */ + 4 /* sched+watchdog+jobsvr+misc */); - Dmsg0(200, "Start UA server\n"); start_UA_server(director->DIRaddrs); @@ -323,7 +369,8 @@ int main (int argc, char *argv[]) init_job_server(director->MaxConcurrentJobs); - dbg_jcr_add_hook(_dbg_print_db); /* used to debug B_DB connexion after fatal signal */ + dbg_jcr_add_hook(dir_debug_print); /* used to director variables */ + dbg_jcr_add_hook(bdb_debug_print); /* used to debug B_DB connexion after fatal signal */ // init_device_resources(); @@ -343,94 +390,68 @@ int main (int argc, char *argv[]) return 0; } -/* - * This allows the message handler to operate on the database - * by using a pointer to this function. The pointer is - * needed because the other daemons do not have access - * to the database. If the pointer is - * not defined (other daemons), then writing the database - * is disabled. - */ -static void dir_sql_query(JCR *jcr, const char *cmd) +struct RELOAD_TABLE { + int job_count; + RES_HEAD **res_head; +}; + +static int max_reloads = 32; +static RELOAD_TABLE *reload_table=NULL; + +static void resize_reload(int nb) { - if (!jcr || !jcr->db) { + if (nb <= max_reloads) { return; } - db_sql_query(jcr->db, cmd, NULL, NULL); -} - -/* Cleanup and then exit */ -void terminate_dird(int sig) -{ - static bool already_here = false; - if (already_here) { /* avoid recursive temination problems */ - bmicrosleep(2, 0); /* yield */ - exit(1); - } - already_here = true; - debug_level = 0; /* turn off debug */ - stop_watchdog(); - generate_daemon_event(NULL, "Exit"); - unload_plugins(); - write_state_file(director->working_directory, "bacula-dir", get_first_port_host_order(director->DIRaddrs)); - delete_pid_file(director->pid_directory, "bacula-dir", get_first_port_host_order(director->DIRaddrs)); - term_scheduler(); - term_job_server(); - if (runjob) { - free(runjob); - } - if (configfile != NULL) { - free(configfile); - } - if (debug_level > 5) { - print_memory_pool_stats(); - } - if (config) { - config->free_resources(); - free(config); - config = NULL; + reload_table = (RELOAD_TABLE*)realloc(reload_table, nb * sizeof(RELOAD_TABLE)); + for (int i=max_reloads; i < nb ; i++) { + reload_table[i].job_count = 0; + reload_table[i].res_head = NULL; } - term_ua_server(); - term_msg(); /* terminate message handler */ - cleanup_crypto(); - close_memory_pool(); /* release free memory in pool */ - lmgr_cleanup_main(); - sm_dump(false); - exit(sig); + max_reloads = nb; } -struct RELOAD_TABLE { - int job_count; - RES **res_table; -}; - -static const int max_reloads = 32; -static RELOAD_TABLE reload_table[max_reloads]; - static void init_reload(void) { + reload_table = (RELOAD_TABLE*)malloc(max_reloads * sizeof(RELOAD_TABLE)); for (int i=0; i < max_reloads; i++) { reload_table[i].job_count = 0; - reload_table[i].res_table = NULL; + reload_table[i].res_head = NULL; } } +/* + * This subroutine frees a saved resource table. + * It was saved when a new table was created with "reload" + */ static void free_saved_resources(int table) { + RES *next, *res; int num = r_last - r_first + 1; - RES **res_tab = reload_table[table].res_table; - if (!res_tab) { + RES_HEAD **res_tab = reload_table[table].res_head; + + if (res_tab == NULL) { Dmsg1(100, "res_tab for table %d already released.\n", table); return; } Dmsg1(100, "Freeing resources for table %d\n", table); for (int j=0; jfirst; + for ( ; next; ) { + res = next; + next = res->res_next; + free_resource(res, r_first + j); + } + free(res_tab[j]->res_list); + free(res_tab[j]); + res_tab[j] = NULL; + } } free(res_tab); reload_table[table].job_count = 0; - reload_table[table].res_table = NULL; + reload_table[table].res_head = NULL; } /* @@ -441,7 +462,7 @@ static void free_saved_resources(int table) */ static void reload_job_end_cb(JCR *jcr, void *ctx) { - int reload_id = (int)((long int)ctx); + int reload_id = (int)((intptr_t)ctx); Dmsg3(100, "reload job_end JobId=%d table=%d cnt=%d\n", jcr->JobId, reload_id, reload_table[reload_id].job_count); lock_jobs(); @@ -457,7 +478,7 @@ static int find_free_reload_table_entry() { int table = -1; for (int i=0; i < max_reloads; i++) { - if (reload_table[i].res_table == NULL) { + if (reload_table[i].res_head == NULL) { table = i; break; } @@ -465,6 +486,8 @@ static int find_free_reload_table_entry() return table; } +static pthread_mutex_t reload_mutex = PTHREAD_MUTEX_INITIALIZER; + /* * If we get here, we have received a SIGHUP, which means to * reread our configuration file. @@ -494,12 +517,27 @@ void reload_config(int sig) JCR *jcr; int njobs = 0; /* number of running jobs */ int table, rtable; - bool ok; - - if (already_here) { - abort(); /* Oops, recursion -> die */ - } - already_here = true; + bool ok=false; + int tries=0; + + /* Wait to do the reload */ + do { + P(reload_mutex); + if (already_here) { + V(reload_mutex); + if (tries++ > 10) { + Qmsg(NULL, M_INFO, 0, _("Already doing a reload request, " + "request ignored.\n")); + return; + } + Dmsg0(10, "Already doing a reload request, waiting a bit\n"); + bmicrosleep(1, 0); + } else { + already_here = true; + V(reload_mutex); + ok = true; + } + } while (!ok); #if !defined(HAVE_WIN32) sigemptyset(&set); @@ -512,50 +550,101 @@ void reload_config(int sig) table = find_free_reload_table_entry(); if (table < 0) { - Jmsg(NULL, M_ERROR, 0, _("Too many open reload requests. Request ignored.\n")); + Qmsg(NULL, M_ERROR, 0, _("Too many (%d) open reload requests. " + "Request ignored.\n"), max_reloads); goto bail_out; } Dmsg1(100, "Reload_config njobs=%d\n", njobs); - reload_table[table].res_table = config->save_resources(); + /* Save current res_head */ + reload_table[table].res_head = res_head; Dmsg1(100, "Saved old config in table %d\n", table); + /* Create a new res_head and parse into it */ ok = parse_dir_config(config, configfile, M_ERROR); Dmsg0(100, "Reloaded config file\n"); if (!ok || !check_resources() || !check_catalog(UPDATE_CATALOG)) { + /* + * We got an error, save broken point, restore old one, + * then release everything from broken pointer. + */ rtable = find_free_reload_table_entry(); /* save new, bad table */ if (rtable < 0) { - Jmsg(NULL, M_ERROR, 0, _("Please correct configuration file: %s\n"), configfile); - Jmsg(NULL, M_ERROR_TERM, 0, _("Out of reload table entries. Giving up.\n")); + Qmsg(NULL, M_ERROR, 0, _("Please correct configuration file: %s\n"), configfile); + Qmsg(NULL, M_ERROR_TERM, 0, _("Out of reload table entries. Giving up.\n")); } else { - Jmsg(NULL, M_ERROR, 0, _("Please correct configuration file: %s\n"), configfile); - Jmsg(NULL, M_ERROR, 0, _("Resetting previous configuration.\n")); - } - reload_table[rtable].res_table = config->save_resources(); - /* Now restore old resoure values */ - int num = r_last - r_first + 1; - RES **res_tab = reload_table[table].res_table; - for (int i=0; iget_JobType() != JT_SYSTEM) { + if (jcr->getJobType() != JT_SYSTEM) { reload_table[table].job_count++; job_end_push(jcr, reload_job_end_cb, (void *)((long int)table)); njobs++; } } endeach_jcr(jcr); + /* + * Now walk through globals tables and plug them into the + * new resources. + */ + CLIENT_GLOBALS *cg; + foreach_dlist(cg, &client_globals) { + CLIENT *client; + client = GetClientResWithName(cg->name); + if (!client) { + Qmsg(NULL, M_INFO, 0, _("Client=%s not found. Assuming it was removed!!!\n"), cg->name); + } else { + client->globals = cg; /* Set globals pointer */ + } + } + STORE_GLOBALS *sg; + foreach_dlist(sg, &store_globals) { + STORE *store; + store = GetStoreResWithName(sg->name); + if (!store) { + Qmsg(NULL, M_INFO, 0, _("Storage=%s not found. Assuming it was removed!!!\n"), sg->name); + } else { + store->globals = sg; /* set globals pointer */ + Dmsg2(200, "Reload found numConcurrent=%ld for Store %s\n", + sg->NumConcurrentJobs, sg->name); + } + } + JOB_GLOBALS *jg; + foreach_dlist(jg, &job_globals) { + JOB *job; + job = GetJobResWithName(jg->name); + if (!job) { + Qmsg(NULL, M_INFO, 0, _("Job=%s not found. Assuming it was removed!!!\n"), jg->name); + } else { + job->globals = jg; /* Set globals pointer */ + } + } + SCHED_GLOBALS *schg; + foreach_dlist(schg, &sched_globals) { + SCHED *sched; + sched = GetSchedResWithName(schg->name); + if (!sched) { + Qmsg(NULL, M_INFO, 0, _("Schedule=%s not found. Assuming it was removed!!!\n"), schg->name); + } else { + sched->globals = schg; /* Set globals pointer */ + } + } } - /* Reset globals */ + /* Reset other globals */ set_working_directory(director->working_directory); FDConnectTimeout = director->FDConnectTimeout; SDConnectTimeout = director->SDConnectTimeout; @@ -576,6 +665,79 @@ bail_out: already_here = false; } +/* Cleanup and then exit */ +void terminate_dird(int sig) +{ + static bool already_here = false; + + if (already_here) { /* avoid recursive temination problems */ + bmicrosleep(2, 0); /* yield */ + exit(1); + } + already_here = true; + debug_level = 0; /* turn off debug */ + stop_watchdog(); + generate_daemon_event(NULL, "Exit"); + unload_plugins(); + if (!test_config) { + write_state_file(director->working_directory, "bacula-dir", get_first_port_host_order(director->DIRaddrs)); + if (make_pid_file) { + delete_pid_file(director->pid_directory, "bacula-dir", get_first_port_host_order(director->DIRaddrs)); + } + } + term_scheduler(); + term_job_server(); + if (runjob) { + free(runjob); + } + if (configfile != NULL) { + free(configfile); + } + if (chk_dbglvl(5)) { + print_memory_pool_stats(); + } + if (config) { + delete config; + config = NULL; + } + term_ua_server(); + term_msg(); /* terminate message handler */ + cleanup_crypto(); + + free_daemon_message_queue(); + + if (reload_table) { + free(reload_table); + } + free(res_head); + res_head = NULL; + /* + * Now walk through resource globals tables and release them + */ + CLIENT_GLOBALS *cg; + foreach_dlist(cg, &client_globals) { + free(cg->name); + if (cg->SetIPaddress) { + free(cg->SetIPaddress); + } + free(cg); + } + STORE_GLOBALS *sg; + foreach_dlist(sg, &store_globals) { + free(sg->name); + free(sg); + } + JOB_GLOBALS *jg; + foreach_dlist(jg, &job_globals) { + free(jg->name); + free(jg); + } + close_memory_pool(); /* release free memory in pool */ + lmgr_cleanup_main(); + sm_dump(false); + exit(sig); +} + /* * Make a quick check to see that we have all the * resources needed. @@ -635,7 +797,7 @@ static bool check_resources() OK = false; } - if ((!director->tls_ca_certfile && !director->tls_ca_certdir) && + if ((!director->tls_ca_certfile && !director->tls_ca_certdir) && need_tls && director->tls_verify_peer) { Jmsg(NULL, M_FATAL, 0, _("Neither \"TLS CA Certificate\" or \"TLS CA" " Certificate Dir\" are defined for Director \"%s\" in %s." @@ -654,7 +816,7 @@ static bool check_resources() director->tls_ca_certdir, director->tls_certfile, director->tls_keyfile, NULL, NULL, director->tls_dhfile, director->tls_verify_peer); - + if (!director->tls_ctx) { Jmsg(NULL, M_FATAL, 0, _("Failed to initialize TLS context for Director \"%s\" in %s.\n"), director->name(), configfile); @@ -671,23 +833,15 @@ static bool check_resources() int i; if (job->jobdefs) { - /* Handle Storage alists specifically */ JOB *jobdefs = job->jobdefs; - if (jobdefs->storage && !job->storage) { - STORE *st; - job->storage = New(alist(10, not_owned_by_alist)); - foreach_alist(st, jobdefs->storage) { - job->storage->append(st); - } - } /* Handle RunScripts alists specifically */ if (jobdefs->RunScripts) { RUNSCRIPT *rs, *elt; - + if (!job->RunScripts) { job->RunScripts = New(alist(10, not_owned_by_alist)); } - + foreach_alist(rs, jobdefs->RunScripts) { elt = copy_runscript(rs); job->RunScripts->append(elt); /* we have to free it */ @@ -700,6 +854,7 @@ static bool check_resources() uint32_t *def_ivalue, *ivalue; /* integer value */ bool *def_bvalue, *bvalue; /* bool value */ int64_t *def_lvalue, *lvalue; /* 64 bit values */ + alist **def_avalue, **avalue; /* alist values */ uint32_t offset; Dmsg4(1400, "Job \"%s\", field \"%s\" bit=%d def=%d\n", @@ -743,9 +898,17 @@ static bool check_resources() * Handle alist resources */ } else if (job_items[i].handler == store_alist_res) { - if (bit_is_set(i, job->jobdefs->hdr.item_present)) { - set_bit(i, job->hdr.item_present); + void *elt; + + def_avalue = (alist **)((char *)(job->jobdefs) + offset); + avalue = (alist **)((char *)job + offset); + + *avalue = New(alist(10, not_owned_by_alist)); + + foreach_alist(elt, (*def_avalue)) { + (*avalue)->append(elt); } + set_bit(i, job->hdr.item_present); /* * Handle integer fields * Note, our store_bit does not handle bitmaped fields @@ -769,6 +932,7 @@ static bool check_resources() */ } else if (job_items[i].handler == store_time || job_items[i].handler == store_size64 || + job_items[i].handler == store_speed || job_items[i].handler == store_int64) { def_lvalue = (int64_t *)((char *)(job->jobdefs) + offset); Dmsg5(400, "Job \"%s\", field \"%s\" def_lvalue=%" lld " item %d offset=%u\n", @@ -811,6 +975,16 @@ static bool check_resources() job->name()); OK = false; } + + /* Make sure the job doesn't use the Scratch Pool to start with */ + const char *name; + if (!check_pool(job->JobType, job->JobLevel, + job->pool, job->next_pool, &name)) { + Jmsg(NULL, M_FATAL, 0, + _("%s \"Scratch\" not valid in Job \"%s\".\n"), + name, job->name()); + OK = false; + } } /* End loop over Job res */ @@ -829,7 +1003,7 @@ static bool check_resources() } need_tls = cons->tls_enable || cons->tls_authenticate; - + if (!cons->tls_certfile && need_tls) { Jmsg(NULL, M_FATAL, 0, _("\"TLS Certificate\" file not defined for Console \"%s\" in %s.\n"), cons->name(), configfile); @@ -842,7 +1016,7 @@ static bool check_resources() OK = false; } - if ((!cons->tls_ca_certfile && !cons->tls_ca_certdir) + if ((!cons->tls_ca_certfile && !cons->tls_ca_certdir) && need_tls && cons->tls_verify_peer) { Jmsg(NULL, M_FATAL, 0, _("Neither \"TLS CA Certificate\" or \"TLS CA" " Certificate Dir\" are defined for Console \"%s\" in %s." @@ -859,7 +1033,7 @@ static bool check_resources() cons->tls_ctx = new_tls_context(cons->tls_ca_certfile, cons->tls_ca_certdir, cons->tls_certfile, cons->tls_keyfile, NULL, NULL, cons->tls_dhfile, cons->tls_verify_peer); - + if (!cons->tls_ctx) { Jmsg(NULL, M_FATAL, 0, _("Failed to initialize TLS context for File daemon \"%s\" in %s.\n"), cons->name(), configfile); @@ -899,7 +1073,7 @@ static bool check_resources() client->tls_ca_certdir, client->tls_certfile, client->tls_keyfile, NULL, NULL, NULL, true); - + if (!client->tls_ctx) { Jmsg(NULL, M_FATAL, 0, _("Failed to initialize TLS context for File daemon \"%s\" in %s.\n"), client->name(), configfile); @@ -908,37 +1082,74 @@ static bool check_resources() } } + /* Loop over all pools, check PoolType */ + POOL *pool; + foreach_res(pool, R_POOL) { + if (!pool->pool_type) { + /* This case is checked by the parse engine, we should not */ + Jmsg(NULL, M_FATAL, 0, _("PoolType required in Pool resource \"%s\".\n"), pool->hdr.name); + OK = false; + continue; + } + if ((strcasecmp(pool->pool_type, NT_("backup")) != 0) && + (strcasecmp(pool->pool_type, NT_("copy")) != 0) && + (strcasecmp(pool->pool_type, NT_("cloned")) != 0) && + (strcasecmp(pool->pool_type, NT_("archive")) != 0) && + (strcasecmp(pool->pool_type, NT_("migration")) != 0) && + (strcasecmp(pool->pool_type, NT_("scratch")) != 0)) + { + Jmsg(NULL, M_FATAL, 0, _("Invalid PoolType \"%s\" in Pool resource \"%s\".\n"), pool->pool_type, pool->hdr.name); + OK = false; + } + + if (pool->NextPool && strcmp(pool->NextPool->name(), "Scratch") == 0) { + Jmsg(NULL, M_FATAL, 0, + _("NextPool \"Scratch\" not valid in Pool \"%s\".\n"), + pool->name()); + OK = false; + } + } + UnlockRes(); if (OK) { close_msg(NULL); /* close temp message handler */ init_msg(NULL, director->messages); /* open daemon message handler */ + last_reload_time = time(NULL); } return OK; } -/* - * In this routine, +/* + * In this routine, * - we can check the connection (mode=CHECK_CONNECTION) * - we can synchronize the catalog with the configuration (mode=UPDATE_CATALOG) * - we can synchronize, and fix old job records (mode=UPDATE_AND_FIX) + * - we hook up the Autochange children with the parent, and + * we hook the shared autochangers together. */ static bool check_catalog(cat_op mode) { bool OK = true; bool need_tls; + STORE *store, *ac_child; /* Loop over databases */ CAT *catalog; foreach_res(catalog, R_CATALOG) { - B_DB *db; + BDB *db; /* * Make sure we can open catalog, otherwise print a warning * message because the server is probably not running. */ - db = db_init(NULL, catalog->db_driver, catalog->db_name, catalog->db_user, - catalog->db_password, catalog->db_address, - catalog->db_port, catalog->db_socket, - catalog->mult_db_connections); + db = db_init_database(NULL, catalog->db_driver, catalog->db_name, + catalog->db_user, + catalog->db_password, catalog->db_address, + catalog->db_port, catalog->db_socket, + catalog->db_ssl_mode, catalog->db_ssl_key, + catalog->db_ssl_cert, catalog->db_ssl_ca, + catalog->db_ssl_capath, catalog->db_ssl_cipher, + catalog->mult_db_connections, + catalog->disable_batch_insert); if (!db || !db_open_database(NULL, db)) { Pmsg2(000, _("Could not open Catalog \"%s\", database \"%s\".\n"), catalog->name(), catalog->db_name); @@ -953,9 +1164,15 @@ static bool check_catalog(cat_op mode) continue; } + /* Display a message if the db max_connections is too low */ + if (!db_check_max_connections(NULL, db, director->MaxConcurrentJobs)) { + Pmsg1(000, "Warning, settings problem for Catalog=%s\n", catalog->name()); + Pmsg1(000, "%s", db_strerror(db)); + } + /* we are in testing mode, so don't touch anything in the catalog */ if (mode == CHECK_CONNECTION) { - db_close_database(NULL, db); + if (db) db_close_database(NULL, db); continue; } @@ -988,32 +1205,48 @@ static bool check_catalog(cat_op mode) CLIENT *client; foreach_res(client, R_CLIENT) { CLIENT_DBR cr; + /* Create clients only if they use the current catalog */ + if (client->catalog != catalog) { + Dmsg3(500, "Skip client=%s with cat=%s not catalog=%s\n", + client->name(), client->catalog->name(), catalog->name()); + continue; + } + Dmsg2(500, "create cat=%s for client=%s\n", + client->catalog->name(), client->name()); memset(&cr, 0, sizeof(cr)); bstrncpy(cr.Name, client->name(), sizeof(cr.Name)); + db_create_client_record(NULL, db, &cr); } /* Ensure basic storage record is in DB */ - STORE *store; foreach_res(store, R_STORAGE) { STORAGE_DBR sr; - MEDIATYPE_DBR mr; + MEDIATYPE_DBR mtr; memset(&sr, 0, sizeof(sr)); - memset(&mr, 0, sizeof(mr)); + memset(&mtr, 0, sizeof(mtr)); if (store->media_type) { - bstrncpy(mr.MediaType, store->media_type, sizeof(mr.MediaType)); - mr.ReadOnly = 0; - db_create_mediatype_record(NULL, db, &mr); + bstrncpy(mtr.MediaType, store->media_type, sizeof(mtr.MediaType)); + mtr.ReadOnly = 0; + db_create_mediatype_record(NULL, db, &mtr); } else { - mr.MediaTypeId = 0; + mtr.MediaTypeId = 0; } bstrncpy(sr.Name, store->name(), sizeof(sr.Name)); sr.AutoChanger = store->autochanger; - db_create_storage_record(NULL, db, &sr); + if (!db_create_storage_record(NULL, db, &sr)) { + Jmsg(NULL, M_FATAL, 0, _("Could not create storage record for %s\n"), + store->name()); + OK = false; + } store->StorageId = sr.StorageId; /* set storage Id */ if (!sr.created) { /* if not created, update it */ sr.AutoChanger = store->autochanger; - db_update_storage_record(NULL, db, &sr); + if (!db_update_storage_record(NULL, db, &sr)) { + Jmsg(NULL, M_FATAL, 0, _("Could not update storage record for %s\n"), + store->name()); + OK = false; + } } /* tls_require implies tls_enable */ @@ -1024,7 +1257,7 @@ static bool check_catalog(cat_op mode) Jmsg(NULL, M_FATAL, 0, _("TLS required but not configured in Bacula.\n")); OK = false; } - } + } need_tls = store->tls_enable || store->tls_authenticate; @@ -1043,7 +1276,7 @@ static bool check_catalog(cat_op mode) store->tls_ctx = new_tls_context(store->tls_ca_certfile, store->tls_ca_certdir, store->tls_certfile, store->tls_keyfile, NULL, NULL, NULL, true); - + if (!store->tls_ctx) { Jmsg(NULL, M_FATAL, 0, _("Failed to initialize TLS context for Storage \"%s\" in %s.\n"), store->name(), configfile); @@ -1052,6 +1285,38 @@ static bool check_catalog(cat_op mode) } } + /* Link up all the children for each changer */ + foreach_res(store, R_STORAGE) { + char sid[50]; + if (store->changer == store) { /* we are a real Autochanger */ + store->ac_group = get_pool_memory(PM_FNAME); + store->ac_group[0] = 0; + pm_strcat(store->ac_group, edit_int64(store->StorageId, sid)); + /* Now look for children who point to this storage */ + foreach_res(ac_child, R_STORAGE) { + if (ac_child != store && ac_child->changer == store) { + /* Found a child -- add StorageId */ + pm_strcat(store->ac_group, ","); + pm_strcat(store->ac_group, edit_int64(ac_child->StorageId, sid)); + } + } + } + } + + /* Link up all the shared storage devices */ + foreach_res(store, R_STORAGE) { + if (store->ac_group) { /* we are a real Autochanger */ + /* Now look for Shared Storage who point to this storage */ + foreach_res(ac_child, R_STORAGE) { + if (ac_child->shared_storage == store && ac_child->ac_group && + ac_child->shared_storage != ac_child) { + pm_strcat(store->ac_group, ","); + pm_strcat(store->ac_group, ac_child->ac_group); + } + } + } + } + /* Loop over all counters, defining them in each database */ /* Set default value in all counters */ COUNTER *counter; @@ -1082,12 +1347,88 @@ static bool check_catalog(cat_op mode) if (mode == UPDATE_AND_FIX) { db_sql_query(db, cleanup_created_job, NULL, NULL); db_sql_query(db, cleanup_running_job, NULL, NULL); - db_sql_query(db, "CREATE INDEX basefiles_jobid_idx ON BaseFiles ( JobId )" , NULL, NULL); } - db_close_database(NULL, db); + /* Set SQL engine name in global for debugging */ + set_db_engine_name(db_get_engine_name(db)); + if (db) db_close_database(NULL, db); } - /* Set type in global for debugging */ - set_db_type(db_get_type()); return OK; } + +static void cleanup_old_files() +{ + DIR* dp; + struct dirent *entry, *result; + int rc, name_max; + int my_name_len = strlen(my_name); + int len = strlen(director->working_directory); + POOLMEM *cleanup = get_pool_memory(PM_MESSAGE); + POOLMEM *basename = get_pool_memory(PM_MESSAGE); + regex_t preg1; + char prbuf[500]; + const int nmatch = 30; + regmatch_t pmatch[nmatch]; + berrno be; + + /* Exclude spaces and look for .mail, .tmp or .restore.xx.bsr files */ + const char *pat1 = "^[^ ]+\\.(restore\\.[^ ]+\\.bsr|mail|tmp)$"; + + /* Setup working directory prefix */ + pm_strcpy(basename, director->working_directory); + if (len > 0 && !IsPathSeparator(director->working_directory[len-1])) { + pm_strcat(basename, "/"); + } + + /* Compile regex expressions */ + rc = regcomp(&preg1, pat1, REG_EXTENDED); + if (rc != 0) { + regerror(rc, &preg1, prbuf, sizeof(prbuf)); + Pmsg2(000, _("Could not compile regex pattern \"%s\" ERR=%s\n"), + pat1, prbuf); + goto get_out2; + } + + name_max = pathconf(".", _PC_NAME_MAX); + if (name_max < 1024) { + name_max = 1024; + } + + if (!(dp = opendir(director->working_directory))) { + berrno be; + Pmsg2(000, "Failed to open working dir %s for cleanup: ERR=%s\n", + director->working_directory, be.bstrerror()); + goto get_out1; + return; + } + + entry = (struct dirent *)malloc(sizeof(struct dirent) + name_max + 1000); + while (1) { + if ((readdir_r(dp, entry, &result) != 0) || (result == NULL)) { + break; + } + /* Exclude any name with ., .., not my_name or containing a space */ + if (strcmp(result->d_name, ".") == 0 || strcmp(result->d_name, "..") == 0 || + strncmp(result->d_name, my_name, my_name_len) != 0) { + Dmsg1(500, "Skipped: %s\n", result->d_name); + continue; + } + + /* Unlink files that match regexes */ + if (regexec(&preg1, result->d_name, nmatch, pmatch, 0) == 0) { + pm_strcpy(cleanup, basename); + pm_strcat(cleanup, result->d_name); + Dmsg1(100, "Unlink: %s\n", cleanup); + unlink(cleanup); + } + } + + free(entry); + closedir(dp); +/* Be careful to free up the correct resources */ +get_out1: + regfree(&preg1); +get_out2: + free_pool_memory(cleanup); + free_pool_memory(basename); +}