X-Git-Url: https://git.sur5r.net/?a=blobdiff_plain;f=bacula%2Fsrc%2Fdird%2Fdird.c;h=c1c1b151a0bb08dbe73d8972033ce08d840f4ed1;hb=baa7ca256d3146c8991d27745825af2817ffddbc;hp=8f596ca73591676cb97314a078bf96519fdcccd7;hpb=eb326beecb6fc8803ab73e6395c68c5a29ac8cc2;p=bacula%2Fbacula diff --git a/bacula/src/dird/dird.c b/bacula/src/dird/dird.c index 8f596ca735..c1c1b151a0 100644 --- a/bacula/src/dird/dird.c +++ b/bacula/src/dird/dird.c @@ -1,52 +1,52 @@ /* - Bacula® - The Network Backup Solution - - Copyright (C) 2000-2007 Free Software Foundation Europe e.V. - - The main author of Bacula is Kern Sibbald, with contributions from - many others, a complete list can be found in the file AUTHORS. - This program is Free Software; you can redistribute it and/or - modify it under the terms of version two of the GNU General Public - License as published by the Free Software Foundation and included - in the file LICENSE. - - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - 02110-1301, USA. - - Bacula® is a registered trademark of John Walker. - The licensor of Bacula is the Free Software Foundation Europe - (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich, - Switzerland, email:ftf@fsfeurope.org. + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2017 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. */ /* - * * Bacula Director daemon -- this is the main program * * Kern Sibbald, March MM - * - * Version $Id$ */ #include "bacula.h" #include "dird.h" +#ifndef HAVE_REGEX_H +#include "lib/bregex.h" +#else +#include +#endif +#ifdef HAVE_DIRENT_H +#include +#define NAMELEN(dirent) (strlen((dirent)->d_name)) +#endif +#ifndef HAVE_READDIR_R +int readdir_r(DIR *dirp, struct dirent *entry, struct dirent **result); +#endif /* Forward referenced subroutines */ void terminate_dird(int sig); static bool check_resources(); -static bool check_catalog(); -static void dir_sql_query(JCR *jcr, const char *cmd); - +static void cleanup_old_files(); +static void resize_reload(int nb); + /* Exported subroutines */ extern "C" void reload_config(int sig); extern void invalidate_schedules(); - +extern bool parse_dir_config(CONFIG *config, const char *configfile, int exit_code); /* Imported subroutines */ JCR *wait_for_next_job(char *runjob); @@ -58,11 +58,16 @@ void term_job_server(); void store_jobtype(LEX *lc, RES_ITEM *item, int index, int pass); void store_level(LEX *lc, RES_ITEM *item, int index, int pass); void store_replace(LEX *lc, RES_ITEM *item, int index, int pass); +void store_migtype(LEX *lc, RES_ITEM *item, int index, int pass); void init_device_resources(); + static char *runjob = NULL; -static int background = 1; +static bool foreground = false; +static bool make_pid_file = true; /* create pid file */ static void init_reload(void); +static CONFIG *config; +static bool test_config = false; /* Globals Exported */ DIRRES *director; /* Director resource */ @@ -70,48 +75,88 @@ int FDConnectTimeout; int SDConnectTimeout; char *configfile = NULL; void *start_heap; +utime_t last_reload_time = 0; + /* Globals Imported */ -extern int r_first, r_last; /* first and last resources */ -extern RES_TABLE resources[]; -extern RES **res_head; +extern dlist client_globals; +extern dlist store_globals; +extern dlist job_globals; +extern dlist sched_globals; extern RES_ITEM job_items[]; - #if defined(_MSC_VER) extern "C" { // work around visual compiler mangling variables - extern URES res_all; + extern URES res_all; } #else extern URES res_all; #endif +typedef enum { + CHECK_CONNECTION, /* Check catalog connection */ + UPDATE_CATALOG, /* Ensure that catalog is ok with conf */ + UPDATE_AND_FIX /* Ensure that catalog is ok, and fix old jobs */ +} cat_op; +static bool check_catalog(cat_op mode); + #define CONFIG_FILE "bacula-dir.conf" /* default configuration file */ +static bool dir_sql_query(JCR *jcr, const char *cmd) +{ + if (jcr && jcr->db && jcr->db->is_connected()) { + return db_sql_query(jcr->db, cmd, NULL, NULL); + } + return false; +} + +static bool dir_sql_escape(JCR *jcr, BDB *mdb, char *snew, char *sold, int len) +{ + if (jcr && jcr->db && jcr->db->is_connected()) { + db_escape_string(jcr, mdb, snew, sold, len); + return true; + } + return false; +} + static void usage() { fprintf(stderr, _( -PROG_COPYRIGHT -"\nVersion: %s (%s)\n\n" -"Usage: dird [-f -s] [-c config_file] [-d debug_level] [config_file]\n" -" -c set configuration file to file\n" -" -dnn set debug level to nn\n" -" -f run in foreground (for debugging)\n" -" -g groupid\n" -" -r run now\n" -" -s no signals\n" -" -t test - read configuration and exit\n" -" -u userid\n" -" -v verbose user messages\n" -" -? print this message.\n" -"\n"), 2000, VERSION, BDATE); + PROG_COPYRIGHT + "\n%sVersion: %s (%s)\n\n" + "Usage: bacula-dir [-f -s] [-c config_file] [-d debug_level] [config_file]\n" + " -c set configuration file to file\n" + " -d [,] set debug level to , debug tags to \n" + " -dt print timestamp in debug output\n" + " -T set trace on\n" + " -f run in foreground (for debugging)\n" + " -g groupid\n" + " -m print kaboom output (for debugging)\n" + " -r run now\n" + " -P do not create pid file\n" + " -s no signals\n" + " -t test - read configuration and exit\n" + " -u userid\n" + " -v verbose user messages\n" + " -? print this message.\n" + "\n"), 2000, "", VERSION, BDATE); exit(1); } +/* + * !!! WARNING !!! Use this function only when bacula is stopped. + * ie, after a fatal signal and before exiting the program + * Print information about a JCR + */ +static void dir_debug_print(JCR *jcr, FILE *fp) +{ + fprintf(fp, "\twstore=%p rstore=%p wjcr=%p client=%p reschedule_count=%d SD_msg_chan_started=%d\n", + jcr->wstore, jcr->rstore, jcr->wjcr, jcr->client, jcr->reschedule_count, (int)jcr->SD_msg_chan_started); +} /********************************************************************* * - * Main Bacula Server program + * Main Bacula Director Server program * */ #if defined(HAVE_WIN32) @@ -119,15 +164,21 @@ PROG_COPYRIGHT #define main BaculaMain #endif +/* DELETE ME when bugs in MA1512, MA1632 MA1639 are fixed */ +extern void (*MA1512_reload_job_end_cb)(JCR *,void *); +static void reload_job_end_cb(JCR *jcr, void *ctx); + int main (int argc, char *argv[]) { int ch; JCR *jcr; bool no_signals = false; - bool test_config = false; char *uid = NULL; char *gid = NULL; + /* DELETE ME when bugs in MA1512, MA1632 MA1639 are fixed */ + MA1512_reload_job_end_cb = reload_job_end_cb; + start_heap = sbrk(0); setlocale(LC_ALL, ""); bindtextdomain("bacula", LOCALEDIR); @@ -138,8 +189,10 @@ int main (int argc, char *argv[]) init_msg(NULL, NULL); /* initialize message handler */ init_reload(); daemon_start_time = time(NULL); + setup_daemon_message_queue(); + console_command = run_console_command; - while ((ch = getopt(argc, argv, "c:d:fg:r:stu:v?")) != -1) { + while ((ch = getopt(argc, argv, "c:d:fg:mPr:stu:v?T")) != -1) { switch (ch) { case 'c': /* specify config file */ if (configfile != NULL) { @@ -149,21 +202,45 @@ int main (int argc, char *argv[]) break; case 'd': /* set debug level */ - debug_level = atoi(optarg); - if (debug_level <= 0) { - debug_level = 1; + if (*optarg == 't') { + dbg_timestamp = true; + } else { + char *p; + /* We probably find a tag list -d 10,sql,bvfs */ + if ((p = strchr(optarg, ',')) != NULL) { + *p = 0; + } + debug_level = atoi(optarg); + if (debug_level <= 0) { + debug_level = 1; + } + if (p) { + debug_parse_tags(p+1, &debug_level_tags); + } } - Dmsg1(10, "Debug level = %d\n", debug_level); + Dmsg1(10, "Debug level = %lld\n", debug_level); + break; + + case 'T': + set_trace(true); break; case 'f': /* run in foreground */ - background = FALSE; + foreground = true; break; case 'g': /* set group id */ gid = optarg; break; + case 'm': /* print kaboom output */ + prt_kaboom = true; + break; + + case 'P': /* no pid file */ + make_pid_file = false; + break; + case 'r': /* run job */ if (runjob != NULL) { free(runjob); @@ -198,10 +275,6 @@ int main (int argc, char *argv[]) argc -= optind; argv += optind; - if (!no_signals) { - init_signals(terminate_dird); - } - if (argc) { if (configfile != NULL) { free(configfile); @@ -214,11 +287,21 @@ int main (int argc, char *argv[]) usage(); } + if (!foreground && !test_config) { + daemon_start(); + init_stack_dump(); /* grab new pid */ + } + + if (!no_signals) { + init_signals(terminate_dird); + } + if (configfile == NULL) { configfile = bstrdup(CONFIG_FILE); } - parse_config(configfile); + config = New(CONFIG()); + parse_dir_config(config, configfile, M_ERROR_TERM); if (init_crypto() != 0) { Jmsg((JCR *)NULL, M_ERROR_TERM, 0, _("Cryptography library initialization failed.\n")); @@ -228,9 +311,29 @@ int main (int argc, char *argv[]) Jmsg((JCR *)NULL, M_ERROR_TERM, 0, _("Please correct configuration file: %s\n"), configfile); } - drop(uid, gid); /* reduce privileges if requested */ + if (!test_config) { + /* Create pid must come after we are a daemon -- so we have our final pid */ + if (make_pid_file) { + create_pid_file(director->pid_directory, "bacula-dir", + get_first_port_host_order(director->DIRaddrs)); + } + read_state_file(director->working_directory, "bacula-dir", + get_first_port_host_order(director->DIRaddrs)); + } + + set_jcr_in_tsd(INVALID_JCR); + set_thread_concurrency(director->MaxConcurrentJobs * 2 + + 4 /* UA */ + 5 /* sched+watchdog+jobsvr+misc */); + lmgr_init_thread(); /* initialize the lockmanager stack */ + + load_dir_plugins(director->plugin_directory); + + drop(uid, gid, false); /* reduce privileges if requested */ - if (!check_catalog()) { + /* If we are in testing mode, we don't try to fix the catalog */ + cat_op mode=(test_config)?CHECK_CONNECTION:UPDATE_AND_FIX; + + if (!check_catalog(mode)) { Jmsg((JCR *)NULL, M_ERROR_TERM, 0, _("Please correct configuration file: %s\n"), configfile); } @@ -240,22 +343,16 @@ int main (int argc, char *argv[]) my_name_is(0, NULL, director->name()); /* set user defined name */ + cleanup_old_files(); + /* Plug database interface for library routines */ - p_sql_query = (sql_query)dir_sql_query; - p_sql_escape = (sql_escape)db_escape_string; + p_sql_query = (sql_query_call)dir_sql_query; + p_sql_escape = (sql_escape_call)dir_sql_escape; FDConnectTimeout = (int)director->FDConnectTimeout; SDConnectTimeout = (int)director->SDConnectTimeout; - if (background) { - daemon_start(); - init_stack_dump(); /* grab new pid */ - } - - /* Create pid must come after we are a daemon -- so we have our final pid */ - create_pid_file(director->pid_directory, "bacula-dir", get_first_port_host_order(director->DIRaddrs)); - read_state_file(director->working_directory, "bacula-dir", get_first_port_host_order(director->DIRaddrs)); - + resize_reload(director->MaxReload); #if !defined(HAVE_WIN32) signal(SIGHUP, reload_config); @@ -263,12 +360,6 @@ int main (int argc, char *argv[]) init_console_msg(working_directory); - init_python_interpreter(director->name(), director->scripts_directory, - "DirStartUp"); - - set_thread_concurrency(director->MaxConcurrentJobs * 2 + - 4 /* UA */ + 4 /* sched+watchdog+jobsvr+misc */); - Dmsg0(200, "Start UA server\n"); start_UA_server(director->DIRaddrs); @@ -278,6 +369,9 @@ int main (int argc, char *argv[]) init_job_server(director->MaxConcurrentJobs); + dbg_jcr_add_hook(dir_debug_print); /* used to director variables */ + dbg_jcr_add_hook(bdb_debug_print); /* used to debug B_DB connexion after fatal signal */ + // init_device_resources(); Dmsg0(200, "wait for next job\n"); @@ -285,6 +379,7 @@ int main (int argc, char *argv[]) while ( (jcr = wait_for_next_job(runjob)) ) { run_job(jcr); /* run job */ free_jcr(jcr); /* release jcr */ + set_jcr_in_tsd(INVALID_JCR); if (runjob) { /* command line, run a single job? */ break; /* yes, terminate */ } @@ -295,87 +390,68 @@ int main (int argc, char *argv[]) return 0; } -/* - * This allows the message handler to operate on the database - * by using a pointer to this function. The pointer is - * needed because the other daemons do not have access - * to the database. If the pointer is - * not defined (other daemons), then writing the database - * is disabled. - */ -static void dir_sql_query(JCR *jcr, const char *cmd) +struct RELOAD_TABLE { + int job_count; + RES_HEAD **res_head; +}; + +static int max_reloads = 32; +static RELOAD_TABLE *reload_table=NULL; + +static void resize_reload(int nb) { - if (!jcr || !jcr->db) { + if (nb <= max_reloads) { return; } - db_sql_query(jcr->db, cmd, NULL, NULL); -} - -/* Cleanup and then exit */ -void terminate_dird(int sig) -{ - static bool already_here = false; - if (already_here) { /* avoid recursive temination problems */ - exit(1); - } - already_here = true; - stop_watchdog(); - generate_daemon_event(NULL, "Exit"); - write_state_file(director->working_directory, "bacula-dir", get_first_port_host_order(director->DIRaddrs)); - delete_pid_file(director->pid_directory, "bacula-dir", get_first_port_host_order(director->DIRaddrs)); -// signal(SIGCHLD, SIG_IGN); /* don't worry about children now */ - term_scheduler(); - term_job_server(); - if (runjob) { - free(runjob); - } - if (configfile != NULL) { - free(configfile); - } - if (debug_level > 5) { - print_memory_pool_stats(); + reload_table = (RELOAD_TABLE*)realloc(reload_table, nb * sizeof(RELOAD_TABLE)); + for (int i=max_reloads; i < nb ; i++) { + reload_table[i].job_count = 0; + reload_table[i].res_head = NULL; } - free_config_resources(); - term_ua_server(); - term_msg(); /* terminate message handler */ - cleanup_crypto(); - close_memory_pool(); /* release free memory in pool */ - sm_dump(false); - exit(sig); + max_reloads = nb; } -struct RELOAD_TABLE { - int job_count; - RES **res_table; -}; - -static const int max_reloads = 32; -static RELOAD_TABLE reload_table[max_reloads]; - static void init_reload(void) { + reload_table = (RELOAD_TABLE*)malloc(max_reloads * sizeof(RELOAD_TABLE)); for (int i=0; i < max_reloads; i++) { reload_table[i].job_count = 0; - reload_table[i].res_table = NULL; + reload_table[i].res_head = NULL; } } +/* + * This subroutine frees a saved resource table. + * It was saved when a new table was created with "reload" + */ static void free_saved_resources(int table) { + RES *next, *res; int num = r_last - r_first + 1; - RES **res_tab = reload_table[table].res_table; - if (!res_tab) { + RES_HEAD **res_tab = reload_table[table].res_head; + + if (res_tab == NULL) { Dmsg1(100, "res_tab for table %d already released.\n", table); return; } Dmsg1(100, "Freeing resources for table %d\n", table); for (int j=0; jfirst; + for ( ; next; ) { + res = next; + next = res->res_next; + free_resource(res, r_first + j); + } + free(res_tab[j]->res_list); + free(res_tab[j]); + res_tab[j] = NULL; + } } free(res_tab); reload_table[table].job_count = 0; - reload_table[table].res_table = NULL; + reload_table[table].res_head = NULL; } /* @@ -386,7 +462,7 @@ static void free_saved_resources(int table) */ static void reload_job_end_cb(JCR *jcr, void *ctx) { - int reload_id = (int)((long int)ctx); + int reload_id = (int)((intptr_t)ctx); Dmsg3(100, "reload job_end JobId=%d table=%d cnt=%d\n", jcr->JobId, reload_id, reload_table[reload_id].job_count); lock_jobs(); @@ -402,7 +478,7 @@ static int find_free_reload_table_entry() { int table = -1; for (int i=0; i < max_reloads; i++) { - if (reload_table[i].res_table == NULL) { + if (reload_table[i].res_head == NULL) { table = i; break; } @@ -410,6 +486,8 @@ static int find_free_reload_table_entry() return table; } +static pthread_mutex_t reload_mutex = PTHREAD_MUTEX_INITIALIZER; + /* * If we get here, we have received a SIGHUP, which means to * reread our configuration file. @@ -439,12 +517,27 @@ void reload_config(int sig) JCR *jcr; int njobs = 0; /* number of running jobs */ int table, rtable; - bool ok; - - if (already_here) { - abort(); /* Oops, recursion -> die */ - } - already_here = true; + bool ok=false; + int tries=0; + + /* Wait to do the reload */ + do { + P(reload_mutex); + if (already_here) { + V(reload_mutex); + if (tries++ > 10) { + Qmsg(NULL, M_INFO, 0, _("Already doing a reload request, " + "request ignored.\n")); + return; + } + Dmsg0(10, "Already doing a reload request, waiting a bit\n"); + bmicrosleep(1, 0); + } else { + already_here = true; + V(reload_mutex); + ok = true; + } + } while (!ok); #if !defined(HAVE_WIN32) sigemptyset(&set); @@ -457,50 +550,101 @@ void reload_config(int sig) table = find_free_reload_table_entry(); if (table < 0) { - Jmsg(NULL, M_ERROR, 0, _("Too many open reload requests. Request ignored.\n")); + Qmsg(NULL, M_ERROR, 0, _("Too many (%d) open reload requests. " + "Request ignored.\n"), max_reloads); goto bail_out; } Dmsg1(100, "Reload_config njobs=%d\n", njobs); - reload_table[table].res_table = save_config_resources(); + /* Save current res_head */ + reload_table[table].res_head = res_head; Dmsg1(100, "Saved old config in table %d\n", table); - ok = parse_config(configfile, 0, M_ERROR); /* no exit on error */ + /* Create a new res_head and parse into it */ + ok = parse_dir_config(config, configfile, M_ERROR); Dmsg0(100, "Reloaded config file\n"); - if (!ok || !check_resources() || !check_catalog()) { + if (!ok || !check_resources() || !check_catalog(UPDATE_CATALOG)) { + /* + * We got an error, save broken point, restore old one, + * then release everything from broken pointer. + */ rtable = find_free_reload_table_entry(); /* save new, bad table */ if (rtable < 0) { - Jmsg(NULL, M_ERROR, 0, _("Please correct configuration file: %s\n"), configfile); - Jmsg(NULL, M_ERROR_TERM, 0, _("Out of reload table entries. Giving up.\n")); + Qmsg(NULL, M_ERROR, 0, _("Please correct configuration file: %s\n"), configfile); + Qmsg(NULL, M_ERROR_TERM, 0, _("Out of reload table entries. Giving up.\n")); } else { - Jmsg(NULL, M_ERROR, 0, _("Please correct configuration file: %s\n"), configfile); - Jmsg(NULL, M_ERROR, 0, _("Resetting previous configuration.\n")); - } - reload_table[rtable].res_table = save_config_resources(); - /* Now restore old resoure values */ - int num = r_last - r_first + 1; - RES **res_tab = reload_table[table].res_table; - for (int i=0; iJobType != JT_SYSTEM) { + if (jcr->getJobType() != JT_SYSTEM) { reload_table[table].job_count++; job_end_push(jcr, reload_job_end_cb, (void *)((long int)table)); njobs++; } } endeach_jcr(jcr); + /* + * Now walk through globals tables and plug them into the + * new resources. + */ + CLIENT_GLOBALS *cg; + foreach_dlist(cg, &client_globals) { + CLIENT *client; + client = GetClientResWithName(cg->name); + if (!client) { + Qmsg(NULL, M_INFO, 0, _("Client=%s not found. Assuming it was removed!!!\n"), cg->name); + } else { + client->globals = cg; /* Set globals pointer */ + } + } + STORE_GLOBALS *sg; + foreach_dlist(sg, &store_globals) { + STORE *store; + store = GetStoreResWithName(sg->name); + if (!store) { + Qmsg(NULL, M_INFO, 0, _("Storage=%s not found. Assuming it was removed!!!\n"), sg->name); + } else { + store->globals = sg; /* set globals pointer */ + Dmsg2(200, "Reload found numConcurrent=%ld for Store %s\n", + sg->NumConcurrentJobs, sg->name); + } + } + JOB_GLOBALS *jg; + foreach_dlist(jg, &job_globals) { + JOB *job; + job = GetJobResWithName(jg->name); + if (!job) { + Qmsg(NULL, M_INFO, 0, _("Job=%s not found. Assuming it was removed!!!\n"), jg->name); + } else { + job->globals = jg; /* Set globals pointer */ + } + } + SCHED_GLOBALS *schg; + foreach_dlist(schg, &sched_globals) { + SCHED *sched; + sched = GetSchedResWithName(schg->name); + if (!sched) { + Qmsg(NULL, M_INFO, 0, _("Schedule=%s not found. Assuming it was removed!!!\n"), schg->name); + } else { + sched->globals = schg; /* Set globals pointer */ + } + } } - /* Reset globals */ + /* Reset other globals */ set_working_directory(director->working_directory); FDConnectTimeout = director->FDConnectTimeout; SDConnectTimeout = director->SDConnectTimeout; @@ -521,6 +665,79 @@ bail_out: already_here = false; } +/* Cleanup and then exit */ +void terminate_dird(int sig) +{ + static bool already_here = false; + + if (already_here) { /* avoid recursive temination problems */ + bmicrosleep(2, 0); /* yield */ + exit(1); + } + already_here = true; + debug_level = 0; /* turn off debug */ + stop_watchdog(); + generate_daemon_event(NULL, "Exit"); + unload_plugins(); + if (!test_config) { + write_state_file(director->working_directory, "bacula-dir", get_first_port_host_order(director->DIRaddrs)); + if (make_pid_file) { + delete_pid_file(director->pid_directory, "bacula-dir", get_first_port_host_order(director->DIRaddrs)); + } + } + term_scheduler(); + term_job_server(); + if (runjob) { + free(runjob); + } + if (configfile != NULL) { + free(configfile); + } + if (chk_dbglvl(5)) { + print_memory_pool_stats(); + } + if (config) { + delete config; + config = NULL; + } + term_ua_server(); + term_msg(); /* terminate message handler */ + cleanup_crypto(); + + free_daemon_message_queue(); + + if (reload_table) { + free(reload_table); + } + free(res_head); + res_head = NULL; + /* + * Now walk through resource globals tables and release them + */ + CLIENT_GLOBALS *cg; + foreach_dlist(cg, &client_globals) { + free(cg->name); + if (cg->SetIPaddress) { + free(cg->SetIPaddress); + } + free(cg); + } + STORE_GLOBALS *sg; + foreach_dlist(sg, &store_globals) { + free(sg->name); + free(sg); + } + JOB_GLOBALS *jg; + foreach_dlist(jg, &job_globals) { + free(jg->name); + free(jg); + } + close_memory_pool(); /* release free memory in pool */ + lmgr_cleanup_main(); + sm_dump(false); + exit(sig); +} + /* * Make a quick check to see that we have all the * resources needed. @@ -532,6 +749,7 @@ static bool check_resources() { bool OK = true; JOB *job; + bool need_tls; LockRes(); @@ -565,19 +783,22 @@ static bool check_resources() } } - if (!director->tls_certfile && director->tls_enable) { + need_tls = director->tls_enable || director->tls_authenticate; + + if (!director->tls_certfile && need_tls) { Jmsg(NULL, M_FATAL, 0, _("\"TLS Certificate\" file not defined for Director \"%s\" in %s.\n"), director->name(), configfile); OK = false; } - if (!director->tls_keyfile && director->tls_enable) { + if (!director->tls_keyfile && need_tls) { Jmsg(NULL, M_FATAL, 0, _("\"TLS Key\" file not defined for Director \"%s\" in %s.\n"), director->name(), configfile); OK = false; } - if ((!director->tls_ca_certfile && !director->tls_ca_certdir) && director->tls_enable && director->tls_verify_peer) { + if ((!director->tls_ca_certfile && !director->tls_ca_certdir) && + need_tls && director->tls_verify_peer) { Jmsg(NULL, M_FATAL, 0, _("Neither \"TLS CA Certificate\" or \"TLS CA" " Certificate Dir\" are defined for Director \"%s\" in %s." " At least one CA certificate store is required" @@ -587,7 +808,7 @@ static bool check_resources() } /* If everything is well, attempt to initialize our per-resource TLS context */ - if (OK && (director->tls_enable || director->tls_require)) { + if (OK && (need_tls || director->tls_require)) { /* Initialize TLS context: * Args: CA certfile, CA certdir, Certfile, Keyfile, * Keyfile PEM Callback, Keyfile CB Userdata, DHfile, Verify Peer */ @@ -595,7 +816,7 @@ static bool check_resources() director->tls_ca_certdir, director->tls_certfile, director->tls_keyfile, NULL, NULL, director->tls_dhfile, director->tls_verify_peer); - + if (!director->tls_ctx) { Jmsg(NULL, M_FATAL, 0, _("Failed to initialize TLS context for Director \"%s\" in %s.\n"), director->name(), configfile); @@ -612,23 +833,15 @@ static bool check_resources() int i; if (job->jobdefs) { - /* Handle Storage alists specifically */ JOB *jobdefs = job->jobdefs; - if (jobdefs->storage && !job->storage) { - STORE *st; - job->storage = New(alist(10, not_owned_by_alist)); - foreach_alist(st, jobdefs->storage) { - job->storage->append(st); - } - } /* Handle RunScripts alists specifically */ if (jobdefs->RunScripts) { RUNSCRIPT *rs, *elt; - + if (!job->RunScripts) { job->RunScripts = New(alist(10, not_owned_by_alist)); } - + foreach_alist(rs, jobdefs->RunScripts) { elt = copy_runscript(rs); job->RunScripts->append(elt); /* we have to free it */ @@ -638,9 +851,10 @@ static bool check_resources() /* Transfer default items from JobDefs Resource */ for (i=0; job_items[i].name; i++) { char **def_svalue, **svalue; /* string value */ - int *def_ivalue, *ivalue; /* integer value */ + uint32_t *def_ivalue, *ivalue; /* integer value */ bool *def_bvalue, *bvalue; /* bool value */ int64_t *def_lvalue, *lvalue; /* 64 bit values */ + alist **def_avalue, **avalue; /* alist values */ uint32_t offset; Dmsg4(1400, "Job \"%s\", field \"%s\" bit=%d def=%d\n", @@ -684,30 +898,41 @@ static bool check_resources() * Handle alist resources */ } else if (job_items[i].handler == store_alist_res) { - if (bit_is_set(i, job->jobdefs->hdr.item_present)) { - set_bit(i, job->hdr.item_present); + void *elt; + + def_avalue = (alist **)((char *)(job->jobdefs) + offset); + avalue = (alist **)((char *)job + offset); + + *avalue = New(alist(10, not_owned_by_alist)); + + foreach_alist(elt, (*def_avalue)) { + (*avalue)->append(elt); } + set_bit(i, job->hdr.item_present); /* * Handle integer fields * Note, our store_bit does not handle bitmaped fields */ } else if (job_items[i].handler == store_bit || - job_items[i].handler == store_pint || + job_items[i].handler == store_pint32 || job_items[i].handler == store_jobtype || job_items[i].handler == store_level || - job_items[i].handler == store_pint || + job_items[i].handler == store_int32 || + job_items[i].handler == store_size32 || + job_items[i].handler == store_migtype || job_items[i].handler == store_replace) { - def_ivalue = (int *)((char *)(job->jobdefs) + offset); + def_ivalue = (uint32_t *)((char *)(job->jobdefs) + offset); Dmsg5(400, "Job \"%s\", field \"%s\" def_ivalue=%d item %d offset=%u\n", job->name(), job_items[i].name, *def_ivalue, i, offset); - ivalue = (int *)((char *)job + offset); + ivalue = (uint32_t *)((char *)job + offset); *ivalue = *def_ivalue; set_bit(i, job->hdr.item_present); /* * Handle 64 bit integer fields */ } else if (job_items[i].handler == store_time || - job_items[i].handler == store_size || + job_items[i].handler == store_size64 || + job_items[i].handler == store_speed || job_items[i].handler == store_int64) { def_lvalue = (int64_t *)((char *)(job->jobdefs) + offset); Dmsg5(400, "Job \"%s\", field \"%s\" def_lvalue=%" lld " item %d offset=%u\n", @@ -750,6 +975,16 @@ static bool check_resources() job->name()); OK = false; } + + /* Make sure the job doesn't use the Scratch Pool to start with */ + const char *name; + if (!check_pool(job->JobType, job->JobLevel, + job->pool, job->next_pool, &name)) { + Jmsg(NULL, M_FATAL, 0, + _("%s \"Scratch\" not valid in Job \"%s\".\n"), + name, job->name()); + OK = false; + } } /* End loop over Job res */ @@ -767,19 +1002,22 @@ static bool check_resources() } } - if (!cons->tls_certfile && cons->tls_enable) { + need_tls = cons->tls_enable || cons->tls_authenticate; + + if (!cons->tls_certfile && need_tls) { Jmsg(NULL, M_FATAL, 0, _("\"TLS Certificate\" file not defined for Console \"%s\" in %s.\n"), cons->name(), configfile); OK = false; } - if (!cons->tls_keyfile && cons->tls_enable) { + if (!cons->tls_keyfile && need_tls) { Jmsg(NULL, M_FATAL, 0, _("\"TLS Key\" file not defined for Console \"%s\" in %s.\n"), cons->name(), configfile); OK = false; } - if ((!cons->tls_ca_certfile && !cons->tls_ca_certdir) && cons->tls_enable && cons->tls_verify_peer) { + if ((!cons->tls_ca_certfile && !cons->tls_ca_certdir) + && need_tls && cons->tls_verify_peer) { Jmsg(NULL, M_FATAL, 0, _("Neither \"TLS CA Certificate\" or \"TLS CA" " Certificate Dir\" are defined for Console \"%s\" in %s." " At least one CA certificate store is required" @@ -788,14 +1026,14 @@ static bool check_resources() OK = false; } /* If everything is well, attempt to initialize our per-resource TLS context */ - if (OK && (cons->tls_enable || cons->tls_require)) { + if (OK && (need_tls || cons->tls_require)) { /* Initialize TLS context: * Args: CA certfile, CA certdir, Certfile, Keyfile, * Keyfile PEM Callback, Keyfile CB Userdata, DHfile, Verify Peer */ cons->tls_ctx = new_tls_context(cons->tls_ca_certfile, cons->tls_ca_certdir, cons->tls_certfile, cons->tls_keyfile, NULL, NULL, cons->tls_dhfile, cons->tls_verify_peer); - + if (!cons->tls_ctx) { Jmsg(NULL, M_FATAL, 0, _("Failed to initialize TLS context for File daemon \"%s\" in %s.\n"), cons->name(), configfile); @@ -818,8 +1056,8 @@ static bool check_resources() continue; } } - - if ((!client->tls_ca_certfile && !client->tls_ca_certdir) && client->tls_enable) { + need_tls = client->tls_enable || client->tls_authenticate; + if ((!client->tls_ca_certfile && !client->tls_ca_certdir) && need_tls) { Jmsg(NULL, M_FATAL, 0, _("Neither \"TLS CA Certificate\"" " or \"TLS CA Certificate Dir\" are defined for File daemon \"%s\" in %s.\n"), client->name(), configfile); @@ -827,7 +1065,7 @@ static bool check_resources() } /* If everything is well, attempt to initialize our per-resource TLS context */ - if (OK && (client->tls_enable || client->tls_require)) { + if (OK && (need_tls || client->tls_require)) { /* Initialize TLS context: * Args: CA certfile, CA certdir, Certfile, Keyfile, * Keyfile PEM Callback, Keyfile CB Userdata, DHfile, Verify Peer */ @@ -835,7 +1073,7 @@ static bool check_resources() client->tls_ca_certdir, client->tls_certfile, client->tls_keyfile, NULL, NULL, NULL, true); - + if (!client->tls_ctx) { Jmsg(NULL, M_FATAL, 0, _("Failed to initialize TLS context for File daemon \"%s\" in %s.\n"), client->name(), configfile); @@ -844,30 +1082,74 @@ static bool check_resources() } } + /* Loop over all pools, check PoolType */ + POOL *pool; + foreach_res(pool, R_POOL) { + if (!pool->pool_type) { + /* This case is checked by the parse engine, we should not */ + Jmsg(NULL, M_FATAL, 0, _("PoolType required in Pool resource \"%s\".\n"), pool->hdr.name); + OK = false; + continue; + } + if ((strcasecmp(pool->pool_type, NT_("backup")) != 0) && + (strcasecmp(pool->pool_type, NT_("copy")) != 0) && + (strcasecmp(pool->pool_type, NT_("cloned")) != 0) && + (strcasecmp(pool->pool_type, NT_("archive")) != 0) && + (strcasecmp(pool->pool_type, NT_("migration")) != 0) && + (strcasecmp(pool->pool_type, NT_("scratch")) != 0)) + { + Jmsg(NULL, M_FATAL, 0, _("Invalid PoolType \"%s\" in Pool resource \"%s\".\n"), pool->pool_type, pool->hdr.name); + OK = false; + } + + if (pool->NextPool && strcmp(pool->NextPool->name(), "Scratch") == 0) { + Jmsg(NULL, M_FATAL, 0, + _("NextPool \"Scratch\" not valid in Pool \"%s\".\n"), + pool->name()); + OK = false; + } + } + UnlockRes(); if (OK) { close_msg(NULL); /* close temp message handler */ init_msg(NULL, director->messages); /* open daemon message handler */ + last_reload_time = time(NULL); } return OK; } -static bool check_catalog() +/* + * In this routine, + * - we can check the connection (mode=CHECK_CONNECTION) + * - we can synchronize the catalog with the configuration (mode=UPDATE_CATALOG) + * - we can synchronize, and fix old job records (mode=UPDATE_AND_FIX) + * - we hook up the Autochange children with the parent, and + * we hook the shared autochangers together. + */ +static bool check_catalog(cat_op mode) { bool OK = true; + bool need_tls; + STORE *store, *ac_child; /* Loop over databases */ CAT *catalog; foreach_res(catalog, R_CATALOG) { - B_DB *db; + BDB *db; /* * Make sure we can open catalog, otherwise print a warning * message because the server is probably not running. */ - db = db_init_database(NULL, catalog->db_name, catalog->db_user, - catalog->db_password, catalog->db_address, - catalog->db_port, catalog->db_socket, - catalog->mult_db_connections); + db = db_init_database(NULL, catalog->db_driver, catalog->db_name, + catalog->db_user, + catalog->db_password, catalog->db_address, + catalog->db_port, catalog->db_socket, + catalog->db_ssl_mode, catalog->db_ssl_key, + catalog->db_ssl_cert, catalog->db_ssl_ca, + catalog->db_ssl_capath, catalog->db_ssl_cipher, + catalog->mult_db_connections, + catalog->disable_batch_insert); if (!db || !db_open_database(NULL, db)) { Pmsg2(000, _("Could not open Catalog \"%s\", database \"%s\".\n"), catalog->name(), catalog->db_name); @@ -882,34 +1164,89 @@ static bool check_catalog() continue; } + /* Display a message if the db max_connections is too low */ + if (!db_check_max_connections(NULL, db, director->MaxConcurrentJobs)) { + Pmsg1(000, "Warning, settings problem for Catalog=%s\n", catalog->name()); + Pmsg1(000, "%s", db_strerror(db)); + } + + /* we are in testing mode, so don't touch anything in the catalog */ + if (mode == CHECK_CONNECTION) { + if (db) db_close_database(NULL, db); + continue; + } + /* Loop over all pools, defining/updating them in each database */ POOL *pool; foreach_res(pool, R_POOL) { - create_pool(NULL, db, pool, POOL_OP_UPDATE); /* update request */ + /* + * If the Pool has a catalog resource create the pool only + * in that catalog. + */ + if (!pool->catalog || pool->catalog == catalog) { + create_pool(NULL, db, pool, POOL_OP_UPDATE); /* update request */ + } } - /* Loop over all pools for updating RecyclePool */ + /* Once they are created, we can loop over them again, updating + * references (RecyclePool) + */ foreach_res(pool, R_POOL) { - update_pool_recyclepool(NULL, db, pool); + /* + * If the Pool has a catalog resource update the pool only + * in that catalog. + */ + if (!pool->catalog || pool->catalog == catalog) { + update_pool_references(NULL, db, pool); + } + } + + /* Ensure basic client record is in DB */ + CLIENT *client; + foreach_res(client, R_CLIENT) { + CLIENT_DBR cr; + /* Create clients only if they use the current catalog */ + if (client->catalog != catalog) { + Dmsg3(500, "Skip client=%s with cat=%s not catalog=%s\n", + client->name(), client->catalog->name(), catalog->name()); + continue; + } + Dmsg2(500, "create cat=%s for client=%s\n", + client->catalog->name(), client->name()); + memset(&cr, 0, sizeof(cr)); + bstrncpy(cr.Name, client->name(), sizeof(cr.Name)); + + db_create_client_record(NULL, db, &cr); } - STORE *store; + /* Ensure basic storage record is in DB */ foreach_res(store, R_STORAGE) { STORAGE_DBR sr; - MEDIATYPE_DBR mr; + MEDIATYPE_DBR mtr; + memset(&sr, 0, sizeof(sr)); + memset(&mtr, 0, sizeof(mtr)); if (store->media_type) { - bstrncpy(mr.MediaType, store->media_type, sizeof(mr.MediaType)); - mr.ReadOnly = 0; - db_create_mediatype_record(NULL, db, &mr); + bstrncpy(mtr.MediaType, store->media_type, sizeof(mtr.MediaType)); + mtr.ReadOnly = 0; + db_create_mediatype_record(NULL, db, &mtr); } else { - mr.MediaTypeId = 0; + mtr.MediaTypeId = 0; } bstrncpy(sr.Name, store->name(), sizeof(sr.Name)); sr.AutoChanger = store->autochanger; - db_create_storage_record(NULL, db, &sr); + if (!db_create_storage_record(NULL, db, &sr)) { + Jmsg(NULL, M_FATAL, 0, _("Could not create storage record for %s\n"), + store->name()); + OK = false; + } store->StorageId = sr.StorageId; /* set storage Id */ if (!sr.created) { /* if not created, update it */ - db_update_storage_record(NULL, db, &sr); + sr.AutoChanger = store->autochanger; + if (!db_update_storage_record(NULL, db, &sr)) { + Jmsg(NULL, M_FATAL, 0, _("Could not update storage record for %s\n"), + store->name()); + OK = false; + } } /* tls_require implies tls_enable */ @@ -920,9 +1257,11 @@ static bool check_catalog() Jmsg(NULL, M_FATAL, 0, _("TLS required but not configured in Bacula.\n")); OK = false; } - } + } + + need_tls = store->tls_enable || store->tls_authenticate; - if ((!store->tls_ca_certfile && !store->tls_ca_certdir) && store->tls_enable) { + if ((!store->tls_ca_certfile && !store->tls_ca_certdir) && need_tls) { Jmsg(NULL, M_FATAL, 0, _("Neither \"TLS CA Certificate\"" " or \"TLS CA Certificate Dir\" are defined for Storage \"%s\" in %s.\n"), store->name(), configfile); @@ -930,14 +1269,14 @@ static bool check_catalog() } /* If everything is well, attempt to initialize our per-resource TLS context */ - if (OK && (store->tls_enable || store->tls_require)) { + if (OK && (need_tls || store->tls_require)) { /* Initialize TLS context: * Args: CA certfile, CA certdir, Certfile, Keyfile, * Keyfile PEM Callback, Keyfile CB Userdata, DHfile, Verify Peer */ store->tls_ctx = new_tls_context(store->tls_ca_certfile, store->tls_ca_certdir, store->tls_certfile, store->tls_keyfile, NULL, NULL, NULL, true); - + if (!store->tls_ctx) { Jmsg(NULL, M_FATAL, 0, _("Failed to initialize TLS context for Storage \"%s\" in %s.\n"), store->name(), configfile); @@ -946,6 +1285,38 @@ static bool check_catalog() } } + /* Link up all the children for each changer */ + foreach_res(store, R_STORAGE) { + char sid[50]; + if (store->changer == store) { /* we are a real Autochanger */ + store->ac_group = get_pool_memory(PM_FNAME); + store->ac_group[0] = 0; + pm_strcat(store->ac_group, edit_int64(store->StorageId, sid)); + /* Now look for children who point to this storage */ + foreach_res(ac_child, R_STORAGE) { + if (ac_child != store && ac_child->changer == store) { + /* Found a child -- add StorageId */ + pm_strcat(store->ac_group, ","); + pm_strcat(store->ac_group, edit_int64(ac_child->StorageId, sid)); + } + } + } + } + + /* Link up all the shared storage devices */ + foreach_res(store, R_STORAGE) { + if (store->ac_group) { /* we are a real Autochanger */ + /* Now look for Shared Storage who point to this storage */ + foreach_res(ac_child, R_STORAGE) { + if (ac_child->shared_storage == store && ac_child->ac_group && + ac_child->shared_storage != ac_child) { + pm_strcat(store->ac_group, ","); + pm_strcat(store->ac_group, ac_child->ac_group); + } + } + } + } + /* Loop over all counters, defining them in each database */ /* Set default value in all counters */ COUNTER *counter; @@ -972,9 +1343,92 @@ static bool check_catalog() counter->CurrentValue = counter->MinValue; /* default value */ } } - db_close_database(NULL, db); + /* cleanup old job records */ + if (mode == UPDATE_AND_FIX) { + db_sql_query(db, cleanup_created_job, NULL, NULL); + db_sql_query(db, cleanup_running_job, NULL, NULL); + } + + /* Set SQL engine name in global for debugging */ + set_db_engine_name(db_get_engine_name(db)); + if (db) db_close_database(NULL, db); } - /* Set type in global for debugging */ - set_db_type(db_get_type()); return OK; } + +static void cleanup_old_files() +{ + DIR* dp; + struct dirent *entry, *result; + int rc, name_max; + int my_name_len = strlen(my_name); + int len = strlen(director->working_directory); + POOLMEM *cleanup = get_pool_memory(PM_MESSAGE); + POOLMEM *basename = get_pool_memory(PM_MESSAGE); + regex_t preg1; + char prbuf[500]; + const int nmatch = 30; + regmatch_t pmatch[nmatch]; + berrno be; + + /* Exclude spaces and look for .mail, .tmp or .restore.xx.bsr files */ + const char *pat1 = "^[^ ]+\\.(restore\\.[^ ]+\\.bsr|mail|tmp)$"; + + /* Setup working directory prefix */ + pm_strcpy(basename, director->working_directory); + if (len > 0 && !IsPathSeparator(director->working_directory[len-1])) { + pm_strcat(basename, "/"); + } + + /* Compile regex expressions */ + rc = regcomp(&preg1, pat1, REG_EXTENDED); + if (rc != 0) { + regerror(rc, &preg1, prbuf, sizeof(prbuf)); + Pmsg2(000, _("Could not compile regex pattern \"%s\" ERR=%s\n"), + pat1, prbuf); + goto get_out2; + } + + name_max = pathconf(".", _PC_NAME_MAX); + if (name_max < 1024) { + name_max = 1024; + } + + if (!(dp = opendir(director->working_directory))) { + berrno be; + Pmsg2(000, "Failed to open working dir %s for cleanup: ERR=%s\n", + director->working_directory, be.bstrerror()); + goto get_out1; + return; + } + + entry = (struct dirent *)malloc(sizeof(struct dirent) + name_max + 1000); + while (1) { + if ((readdir_r(dp, entry, &result) != 0) || (result == NULL)) { + break; + } + /* Exclude any name with ., .., not my_name or containing a space */ + if (strcmp(result->d_name, ".") == 0 || strcmp(result->d_name, "..") == 0 || + strncmp(result->d_name, my_name, my_name_len) != 0) { + Dmsg1(500, "Skipped: %s\n", result->d_name); + continue; + } + + /* Unlink files that match regexes */ + if (regexec(&preg1, result->d_name, nmatch, pmatch, 0) == 0) { + pm_strcpy(cleanup, basename); + pm_strcat(cleanup, result->d_name); + Dmsg1(100, "Unlink: %s\n", cleanup); + unlink(cleanup); + } + } + + free(entry); + closedir(dp); +/* Be careful to free up the correct resources */ +get_out1: + regfree(&preg1); +get_out2: + free_pool_memory(cleanup); + free_pool_memory(basename); +}