#include "bacula.h"
#include "jcr.h"
-extern void timeout_handler(int sig);
+/* External variables we reference */
+extern time_t watchdog_time;
+
+/* Forward referenced functions */
+static void timeout_handler(int sig);
+static void jcr_timeout_check(watchdog_t *self);
struct s_last_job last_job; /* last job run by this daemon */
-dlist *last_jobs;
+dlist *last_jobs = NULL;
#define MAX_LAST_JOBS 10
static JCR *jobs = NULL; /* pointer to JCR chain */
-/* Mutex for locking various jcr chains while updating */
-static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
+static brwlock_t lock; /* lock for last jobs and JCR chain */
void init_last_jobs_list()
{
- struct s_last_job *job_entry;
- last_jobs = new dlist(job_entry, &job_entry->link);
- memset(&last_job, 0, sizeof(last_job));
+ int errstat;
+ struct s_last_job *job_entry = NULL;
+ if (!last_jobs) {
+ last_jobs = new dlist(job_entry, &job_entry->link);
+ memset(&last_job, 0, sizeof(last_job));
+ }
+ if ((errstat=rwl_init(&lock)) != 0) {
+ Emsg1(M_ABORT, 0, _("Unable to initialize jcr_chain lock. ERR=%s\n"),
+ strerror(errstat));
+ }
+
}
void term_last_jobs_list()
{
- for (void *je=NULL; (je=last_jobs->next(je)); ) {
- free(je);
+ char *je;
+ if (last_jobs) {
+ foreach_dlist(je, last_jobs) {
+ free(je);
+ }
+ delete last_jobs;
+ last_jobs = NULL;
}
- delete last_jobs;
+ rwl_destroy(&lock);
}
void lock_last_jobs_list()
{
- P(mutex);
+ /* Use jcr chain mutex */
+ lock_jcr_chain();
}
void unlock_last_jobs_list()
{
- V(mutex);
+ /* Use jcr chain mutex */
+ unlock_jcr_chain();
}
/*
JCR *new_jcr(int size, JCR_free_HANDLER *daemon_free_jcr)
{
JCR *jcr;
+ MQUEUE_ITEM *item = NULL;
struct sigaction sigtimer;
Dmsg0(200, "Enter new_jcr\n");
jcr = (JCR *)malloc(size);
memset(jcr, 0, size);
+ jcr->msg_queue = new dlist(item, &item->link);
jcr->my_thread_id = pthread_self();
jcr->sched_time = time(NULL);
jcr->daemon_free_jcr = daemon_free_jcr; /* plug daemon free routine */
sigfillset(&sigtimer.sa_mask);
sigaction(TIMEOUT_SIGNAL, &sigtimer, NULL);
- P(mutex);
+ lock_jcr_chain();
jcr->prev = NULL;
jcr->next = jobs;
if (jobs) {
jobs->prev = jcr;
}
jobs = jcr;
- V(mutex);
+ unlock_jcr_chain();
return jcr;
}
pthread_mutex_destroy(&jcr->mutex);
close_msg(jcr); /* close messages for this job */
+ delete jcr->msg_queue;
/* do this after closing messages */
if (jcr->client_name) {
* Global routine to free a jcr
*/
#ifdef DEBUG
-void b_free_jcr(char *file, int line, JCR *jcr)
+void b_free_jcr(const char *file, int line, JCR *jcr)
{
Dmsg3(200, "Enter free_jcr 0x%x from %s:%d\n", jcr, file, line);
#endif
struct s_last_job *je;
- P(mutex);
+ lock_jcr_chain();
jcr->use_count--; /* decrement use count */
- Dmsg3(200, "Dec jcr 0x%x use_count=%d jobid=%d\n", jcr, jcr->use_count, jcr->JobId);
+ Dmsg3(200, "Dec free_jcr 0x%x use_count=%d jobid=%d\n", jcr, jcr->use_count, jcr->JobId);
if (jcr->use_count > 0) { /* if in use */
- V(mutex);
- Dmsg2(200, "jcr 0x%x use_count=%d\n", jcr, jcr->use_count);
+ unlock_jcr_chain();
+ Dmsg2(200, "free_jcr 0x%x use_count=%d\n", jcr, jcr->use_count);
return;
}
remove_jcr(jcr);
if (last_job.JobId > 0) {
je = (struct s_last_job *)malloc(sizeof(struct s_last_job));
memcpy((char *)je, (char *)&last_job, sizeof(last_job));
+ if (!last_jobs) {
+ init_last_jobs_list();
+ }
last_jobs->append(je);
if (last_jobs->size() > MAX_LAST_JOBS) {
last_jobs->remove(last_jobs->first());
last_job.JobId = 0; /* zap last job */
}
close_msg(NULL); /* flush any daemon messages */
- V(mutex);
+ unlock_jcr_chain();
Dmsg0(200, "Exit free_jcr\n");
}
void free_locked_jcr(JCR *jcr)
{
jcr->use_count--; /* decrement use count */
- Dmsg2(200, "Dec jcr 0x%x use_count=%d\n", jcr, jcr->use_count);
+ Dmsg2(200, "Dec free_locked_jcr 0x%x use_count=%d\n", jcr, jcr->use_count);
if (jcr->use_count > 0) { /* if in use */
return;
}
{
JCR *jcr;
- P(mutex);
+ lock_jcr_chain(); /* lock chain */
for (jcr = jobs; jcr; jcr=jcr->next) {
if (jcr->JobId == JobId) {
+ P(jcr->mutex);
jcr->use_count++;
- Dmsg2(200, "Inc jcr 0x%x use_count=%d\n", jcr, jcr->use_count);
+ V(jcr->mutex);
+ Dmsg2(200, "Inc get_jcr 0x%x use_count=%d\n", jcr, jcr->use_count);
break;
}
}
- V(mutex);
+ unlock_jcr_chain();
return jcr;
}
{
JCR *jcr;
- P(mutex);
+ lock_jcr_chain();
for (jcr = jobs; jcr; jcr=jcr->next) {
if (jcr->VolSessionId == SessionId &&
jcr->VolSessionTime == SessionTime) {
+ P(jcr->mutex);
jcr->use_count++;
- Dmsg2(200, "Inc jcr 0x%x use_count=%d\n", jcr, jcr->use_count);
+ V(jcr->mutex);
+ Dmsg2(200, "Inc get_jcr 0x%x use_count=%d\n", jcr, jcr->use_count);
break;
}
}
- V(mutex);
+ unlock_jcr_chain();
return jcr;
}
if (!Job) {
return NULL;
}
- P(mutex);
+ lock_jcr_chain();
len = strlen(Job);
for (jcr = jobs; jcr; jcr=jcr->next) {
if (strncmp(Job, jcr->Job, len) == 0) {
+ P(jcr->mutex);
jcr->use_count++;
- Dmsg2(200, "Inc jcr 0x%x use_count=%d\n", jcr, jcr->use_count);
+ V(jcr->mutex);
+ Dmsg2(200, "Inc get_jcr 0x%x use_count=%d\n", jcr, jcr->use_count);
break;
}
}
- V(mutex);
+ unlock_jcr_chain();
return jcr;
}
if (!Job) {
return NULL;
}
- P(mutex);
+ lock_jcr_chain();
for (jcr = jobs; jcr; jcr=jcr->next) {
if (strcmp(jcr->Job, Job) == 0) {
+ P(jcr->mutex);
jcr->use_count++;
- Dmsg2(200, "Inc jcr 0x%x use_count=%d\n", jcr, jcr->use_count);
+ V(jcr->mutex);
+ Dmsg2(200, "Inc get_jcr 0x%x use_count=%d\n", jcr, jcr->use_count);
break;
}
}
- V(mutex);
+ unlock_jcr_chain();
return jcr;
}
*/
void lock_jcr_chain()
{
- P(mutex);
+ int errstat;
+ if ((errstat=rwl_writelock(&lock)) != 0) {
+ Emsg1(M_ABORT, 0, "rwl_writelock failure. ERR=%s\n",
+ strerror(errstat));
+ }
}
/*
*/
void unlock_jcr_chain()
{
- V(mutex);
+ int errstat;
+ if ((errstat=rwl_writeunlock(&lock)) != 0) {
+ Emsg1(M_ABORT, 0, "rwl_writeunlock failure. ERR=%s\n",
+ strerror(errstat));
+ }
}
-JCR *get_next_jcr(JCR *jcr)
+JCR *get_next_jcr(JCR *prev_jcr)
{
- JCR *rjcr;
+ JCR *jcr;
- if (jcr == NULL) {
- rjcr = jobs;
+ if (prev_jcr == NULL) {
+ jcr = jobs;
} else {
- rjcr = jcr->next;
+ jcr = prev_jcr->next;
}
- if (rjcr) {
- rjcr->use_count++;
- Dmsg1(200, "Inc jcr use_count=%d\n", rjcr->use_count);
+ if (jcr) {
+ P(jcr->mutex);
+ jcr->use_count++;
+ V(jcr->mutex);
+ Dmsg2(200, "Inc get_next_jcr 0x%x use_count=%d\n", jcr, jcr->use_count);
}
- return rjcr;
+ return jcr;
+}
+
+bool init_jcr_subsystem(void)
+{
+ watchdog_t *wd = new_watchdog();
+
+ wd->one_shot = false;
+ wd->interval = 30; /* FIXME: should be configurable somewhere, even
+ if only with a #define */
+ wd->callback = jcr_timeout_check;
+
+ register_watchdog(wd);
+
+ return true;
+}
+
+static void jcr_timeout_check(watchdog_t *self)
+{
+ JCR *jcr;
+ BSOCK *fd;
+ time_t timer_start;
+
+ Dmsg0(400, "Start JCR timeout checks\n");
+
+ /* Walk through all JCRs checking if any one is
+ * blocked for more than specified max time.
+ */
+ lock_jcr_chain();
+ foreach_jcr(jcr) {
+ free_locked_jcr(jcr); /* OK to free now cuz chain is locked */
+ if (jcr->JobId == 0) {
+ continue;
+ }
+ fd = jcr->store_bsock;
+ if (fd) {
+ timer_start = fd->timer_start;
+ if (timer_start && (watchdog_time - timer_start) > fd->timeout) {
+ fd->timer_start = 0; /* turn off timer */
+ fd->timed_out = TRUE;
+ Jmsg(jcr, M_ERROR, 0, _(
+"Watchdog sending kill after %d secs to thread stalled reading Storage daemon.\n"),
+ watchdog_time - timer_start);
+ pthread_kill(jcr->my_thread_id, TIMEOUT_SIGNAL);
+ }
+ }
+ fd = jcr->file_bsock;
+ if (fd) {
+ timer_start = fd->timer_start;
+ if (timer_start && (watchdog_time - timer_start) > fd->timeout) {
+ fd->timer_start = 0; /* turn off timer */
+ fd->timed_out = TRUE;
+ Jmsg(jcr, M_ERROR, 0, _(
+"Watchdog sending kill after %d secs to thread stalled reading File daemon.\n"),
+ watchdog_time - timer_start);
+ pthread_kill(jcr->my_thread_id, TIMEOUT_SIGNAL);
+ }
+ }
+ fd = jcr->dir_bsock;
+ if (fd) {
+ timer_start = fd->timer_start;
+ if (timer_start && (watchdog_time - timer_start) > fd->timeout) {
+ fd->timer_start = 0; /* turn off timer */
+ fd->timed_out = TRUE;
+ Jmsg(jcr, M_ERROR, 0, _(
+"Watchdog sending kill after %d secs to thread stalled reading Director.\n"),
+ watchdog_time - timer_start);
+ pthread_kill(jcr->my_thread_id, TIMEOUT_SIGNAL);
+ }
+ }
+
+ }
+ unlock_jcr_chain();
+
+ Dmsg0(200, "Finished JCR timeout checks\n");
+}
+
+/*
+ * Timeout signal comes here
+ */
+static void timeout_handler(int sig)
+{
+ return; /* thus interrupting the function */
}