#include "bacula.h"
#include "jcr.h"
-extern void timeout_handler(int sig);
+/* External variables we reference */
+extern time_t watchdog_time;
-struct s_last_job last_job; /* last job run by this daemon */
+/* Forward referenced functions */
+static void timeout_handler(int sig);
+static void jcr_timeout_check(watchdog_t *self);
+
+int num_jobs_run;
+dlist *last_jobs = NULL;
+#define MAX_LAST_JOBS 15
static JCR *jobs = NULL; /* pointer to JCR chain */
-static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
+static brwlock_t lock; /* lock for last jobs and JCR chain */
+
+void init_last_jobs_list()
+{
+ int errstat;
+ struct s_last_job *job_entry = NULL;
+ if (!last_jobs) {
+ last_jobs = new dlist(job_entry, &job_entry->link);
+ if ((errstat=rwl_init(&lock)) != 0) {
+ Emsg1(M_ABORT, 0, _("Unable to initialize jcr_chain lock. ERR=%s\n"),
+ strerror(errstat));
+ }
+ }
+
+}
+
+void term_last_jobs_list()
+{
+ struct s_last_job *je;
+ if (last_jobs) {
+ foreach_dlist(je, last_jobs) {
+ free(je);
+ }
+ delete last_jobs;
+ last_jobs = NULL;
+ rwl_destroy(&lock);
+ }
+}
+
+void read_last_jobs_list(int fd, uint64_t addr)
+{
+ struct s_last_job *je, job;
+ uint32_t num;
+
+ Dmsg1(010, "read_last_jobs seek to %d\n", (int)addr);
+ if (addr == 0 || lseek(fd, addr, SEEK_SET) < 0) {
+ return;
+ }
+ if (read(fd, &num, sizeof(num)) != sizeof(num)) {
+ return;
+ }
+ Dmsg1(010, "Read num_items=%d\n", num);
+ if (num > 4 * MAX_LAST_JOBS) { /* sanity check */
+ return;
+ }
+ for ( ; num; num--) {
+ if (read(fd, &job, sizeof(job)) != sizeof(job)) {
+ Dmsg1(000, "Read job entry. ERR=%s\n", strerror(errno));
+ return;
+ }
+ if (job.JobId > 0) {
+ je = (struct s_last_job *)malloc(sizeof(struct s_last_job));
+ memcpy((char *)je, (char *)&job, sizeof(job));
+ if (!last_jobs) {
+ init_last_jobs_list();
+ }
+ last_jobs->append(je);
+ if (last_jobs->size() > MAX_LAST_JOBS) {
+ last_jobs->remove(last_jobs->first());
+ }
+ }
+ }
+}
+
+uint64_t write_last_jobs_list(int fd, uint64_t addr)
+{
+ struct s_last_job *je;
+ uint32_t num;
+
+ Dmsg1(010, "write_last_jobs seek to %d\n", (int)addr);
+ if (lseek(fd, addr, SEEK_SET) < 0) {
+ return 0;
+ }
+ if (last_jobs) {
+ /* First record is number of entires */
+ num = last_jobs->size();
+ if (write(fd, &num, sizeof(num)) != sizeof(num)) {
+ Dmsg1(000, "Error writing num_items: ERR=%s\n", strerror(errno));
+ return 0;
+ }
+ foreach_dlist(je, last_jobs) {
+ if (write(fd, je, sizeof(struct s_last_job)) != sizeof(struct s_last_job)) {
+ Dmsg1(000, "Error writing job: ERR=%s\n", strerror(errno));
+ return 0;
+ }
+ }
+ }
+ /* Return current address */
+ ssize_t stat = lseek(fd, 0, SEEK_CUR);
+ if (stat < 0) {
+ stat = 0;
+ }
+ return stat;
+
+}
+
+void lock_last_jobs_list()
+{
+ /* Use jcr chain mutex */
+ lock_jcr_chain();
+}
+
+void unlock_last_jobs_list()
+{
+ /* Use jcr chain mutex */
+ unlock_jcr_chain();
+}
/*
* Create a Job Control Record and link it into JCR chain
JCR *new_jcr(int size, JCR_free_HANDLER *daemon_free_jcr)
{
JCR *jcr;
+ MQUEUE_ITEM *item = NULL;
struct sigaction sigtimer;
Dmsg0(200, "Enter new_jcr\n");
jcr = (JCR *)malloc(size);
memset(jcr, 0, size);
+ jcr->msg_queue = new dlist(item, &item->link);
jcr->my_thread_id = pthread_self();
jcr->sched_time = time(NULL);
jcr->daemon_free_jcr = daemon_free_jcr; /* plug daemon free routine */
sigfillset(&sigtimer.sa_mask);
sigaction(TIMEOUT_SIGNAL, &sigtimer, NULL);
- P(mutex);
+ lock_jcr_chain();
jcr->prev = NULL;
jcr->next = jobs;
if (jobs) {
jobs->prev = jcr;
}
jobs = jcr;
- V(mutex);
+ unlock_jcr_chain();
return jcr;
}
}
/*
- * Free stuff common to all JCRs
+ * Free stuff common to all JCRs. N.B. Be careful to include only
+ * generic stuff in the common part of the jcr.
*/
static void free_common_jcr(JCR *jcr)
{
+ struct s_last_job *je, last_job;
+
/* Keep some statistics */
switch (jcr->JobType) {
- case JT_BACKUP:
- case JT_VERIFY:
- case JT_RESTORE:
- last_job.NumJobs++;
- last_job.JobType = jcr->JobType;
- last_job.JobId = jcr->JobId;
- last_job.VolSessionId = jcr->VolSessionId;
- last_job.VolSessionTime = jcr->VolSessionTime;
- strcpy(last_job.Job, jcr->Job);
- last_job.JobFiles = jcr->JobFiles;
- last_job.JobBytes = jcr->JobBytes;
- last_job.JobStatus = jcr->JobStatus;
- last_job.start_time = jcr->start_time;
- last_job.end_time = time(NULL);
- break;
- default:
- break;
+ case JT_BACKUP:
+ case JT_VERIFY:
+ case JT_RESTORE:
+ case JT_ADMIN:
+ num_jobs_run++;
+ last_job.JobType = jcr->JobType;
+ last_job.JobId = jcr->JobId;
+ last_job.VolSessionId = jcr->VolSessionId;
+ last_job.VolSessionTime = jcr->VolSessionTime;
+ bstrncpy(last_job.Job, jcr->Job, sizeof(last_job.Job));
+ last_job.JobFiles = jcr->JobFiles;
+ last_job.JobBytes = jcr->JobBytes;
+ last_job.JobStatus = jcr->JobStatus;
+ last_job.JobLevel = jcr->JobLevel;
+ last_job.start_time = jcr->start_time;
+ last_job.end_time = time(NULL);
+ /* Keep list of last jobs, but not Console where JobId==0 */
+ if (last_job.JobId > 0) {
+ je = (struct s_last_job *)malloc(sizeof(struct s_last_job));
+ memcpy((char *)je, (char *)&last_job, sizeof(last_job));
+ if (!last_jobs) {
+ init_last_jobs_list();
+ }
+ last_jobs->append(je);
+ if (last_jobs->size() > MAX_LAST_JOBS) {
+ last_jobs->remove(last_jobs->first());
+ }
+ }
+ break;
+ default:
+ break;
}
pthread_mutex_destroy(&jcr->mutex);
close_msg(jcr); /* close messages for this job */
+ delete jcr->msg_queue;
/* do this after closing messages */
if (jcr->client_name) {
}
if (jcr->sd_auth_key) {
- Dmsg0(200, "Free JCR sd_auth_key\n");
free(jcr->sd_auth_key);
jcr->sd_auth_key = NULL;
}
free_pool_memory(jcr->errmsg);
jcr->errmsg = NULL;
}
+ if (jcr->where) {
+ free(jcr->where);
+ jcr->where = NULL;
+ }
+ if (jcr->cached_path) {
+ free_pool_memory(jcr->cached_path);
+ jcr->cached_path = NULL;
+ jcr->cached_pnl = 0;
+ }
+ free_getuser_cache();
+ free_getgroup_cache();
free(jcr);
}
* Global routine to free a jcr
*/
#ifdef DEBUG
-void b_free_jcr(char *file, int line, JCR *jcr)
+void b_free_jcr(const char *file, int line, JCR *jcr)
{
Dmsg3(200, "Enter free_jcr 0x%x from %s:%d\n", jcr, file, line);
void free_jcr(JCR *jcr)
{
+
Dmsg1(200, "Enter free_jcr 0x%x\n", jcr);
#endif
- P(mutex);
+ lock_jcr_chain();
jcr->use_count--; /* decrement use count */
- Dmsg2(200, "Dec jcr 0x%x use_count=%d\n", jcr, jcr->use_count);
+ if (jcr->use_count < 0) {
+ Emsg2(M_ERROR, 0, _("JCR use_count=%d JobId=%d\n"),
+ jcr->use_count, jcr->JobId);
+ }
+ Dmsg3(200, "Dec free_jcr 0x%x use_count=%d jobid=%d\n", jcr, jcr->use_count, jcr->JobId);
if (jcr->use_count > 0) { /* if in use */
- V(mutex);
- Dmsg2(200, "jcr 0x%x use_count=%d\n", jcr, jcr->use_count);
+ unlock_jcr_chain();
+ Dmsg2(200, "free_jcr 0x%x use_count=%d\n", jcr, jcr->use_count);
return;
}
remove_jcr(jcr);
- V(mutex);
+ Dmsg1(200, "End job=%d\n", jcr->JobId);
if (jcr->daemon_free_jcr) {
jcr->daemon_free_jcr(jcr); /* call daemon free routine */
}
+
free_common_jcr(jcr);
- P(mutex);
close_msg(NULL); /* flush any daemon messages */
- V(mutex);
+ unlock_jcr_chain();
Dmsg0(200, "Exit free_jcr\n");
}
void free_locked_jcr(JCR *jcr)
{
jcr->use_count--; /* decrement use count */
- Dmsg2(200, "Dec jcr 0x%x use_count=%d\n", jcr, jcr->use_count);
+ Dmsg2(200, "Dec free_locked_jcr 0x%x use_count=%d\n", jcr, jcr->use_count);
if (jcr->use_count > 0) { /* if in use */
return;
}
{
JCR *jcr;
- P(mutex);
+ lock_jcr_chain(); /* lock chain */
for (jcr = jobs; jcr; jcr=jcr->next) {
if (jcr->JobId == JobId) {
+ P(jcr->mutex);
jcr->use_count++;
- Dmsg2(200, "Inc jcr 0x%x use_count=%d\n", jcr, jcr->use_count);
+ V(jcr->mutex);
+ Dmsg2(200, "Inc get_jcr 0x%x use_count=%d\n", jcr, jcr->use_count);
break;
}
}
- V(mutex);
+ unlock_jcr_chain();
return jcr;
}
{
JCR *jcr;
- P(mutex);
+ lock_jcr_chain();
for (jcr = jobs; jcr; jcr=jcr->next) {
if (jcr->VolSessionId == SessionId &&
jcr->VolSessionTime == SessionTime) {
+ P(jcr->mutex);
jcr->use_count++;
- Dmsg2(200, "Inc jcr 0x%x use_count=%d\n", jcr, jcr->use_count);
+ V(jcr->mutex);
+ Dmsg2(200, "Inc get_jcr 0x%x use_count=%d\n", jcr, jcr->use_count);
break;
}
}
- V(mutex);
+ unlock_jcr_chain();
return jcr;
}
JCR *jcr;
int len;
- P(mutex);
+ if (!Job) {
+ return NULL;
+ }
+ lock_jcr_chain();
len = strlen(Job);
for (jcr = jobs; jcr; jcr=jcr->next) {
if (strncmp(Job, jcr->Job, len) == 0) {
+ P(jcr->mutex);
jcr->use_count++;
- Dmsg2(200, "Inc jcr 0x%x use_count=%d\n", jcr, jcr->use_count);
+ V(jcr->mutex);
+ Dmsg2(200, "Inc get_jcr 0x%x use_count=%d\n", jcr, jcr->use_count);
break;
}
}
- V(mutex);
+ unlock_jcr_chain();
return jcr;
}
{
JCR *jcr;
- P(mutex);
+ if (!Job) {
+ return NULL;
+ }
+ lock_jcr_chain();
for (jcr = jobs; jcr; jcr=jcr->next) {
if (strcmp(jcr->Job, Job) == 0) {
+ P(jcr->mutex);
jcr->use_count++;
- Dmsg2(200, "Inc jcr 0x%x use_count=%d\n", jcr, jcr->use_count);
+ V(jcr->mutex);
+ Dmsg2(200, "Inc get_jcr 0x%x use_count=%d\n", jcr, jcr->use_count);
break;
}
}
- V(mutex);
+ unlock_jcr_chain();
return jcr;
}
*/
void lock_jcr_chain()
{
- P(mutex);
+ int errstat;
+ if ((errstat=rwl_writelock(&lock)) != 0) {
+ Emsg1(M_ABORT, 0, "rwl_writelock failure. ERR=%s\n",
+ strerror(errstat));
+ }
}
/*
*/
void unlock_jcr_chain()
{
- V(mutex);
+ int errstat;
+ if ((errstat=rwl_writeunlock(&lock)) != 0) {
+ Emsg1(M_ABORT, 0, "rwl_writeunlock failure. ERR=%s\n",
+ strerror(errstat));
+ }
}
-JCR *get_next_jcr(JCR *jcr)
+JCR *get_next_jcr(JCR *prev_jcr)
{
- JCR *rjcr;
+ JCR *jcr;
- if (jcr == NULL) {
- rjcr = jobs;
+ if (prev_jcr == NULL) {
+ jcr = jobs;
} else {
- rjcr = jcr->next;
+ jcr = prev_jcr->next;
}
- if (rjcr) {
- rjcr->use_count++;
- Dmsg1(200, "Inc jcr use_count=%d\n", rjcr->use_count);
+ if (jcr) {
+ P(jcr->mutex);
+ jcr->use_count++;
+ V(jcr->mutex);
+ Dmsg2(200, "Inc get_next_jcr 0x%x use_count=%d\n", jcr, jcr->use_count);
}
- return rjcr;
+ return jcr;
+}
+
+bool init_jcr_subsystem(void)
+{
+ watchdog_t *wd = new_watchdog();
+
+ wd->one_shot = false;
+ wd->interval = 30; /* FIXME: should be configurable somewhere, even
+ if only with a #define */
+ wd->callback = jcr_timeout_check;
+
+ register_watchdog(wd);
+
+ return true;
+}
+
+static void jcr_timeout_check(watchdog_t *self)
+{
+ JCR *jcr;
+ BSOCK *fd;
+ time_t timer_start;
+
+ Dmsg0(400, "Start JCR timeout checks\n");
+
+ /* Walk through all JCRs checking if any one is
+ * blocked for more than specified max time.
+ */
+ lock_jcr_chain();
+ foreach_jcr(jcr) {
+ free_locked_jcr(jcr); /* OK to free now cuz chain is locked */
+ if (jcr->JobId == 0) {
+ continue;
+ }
+ fd = jcr->store_bsock;
+ if (fd) {
+ timer_start = fd->timer_start;
+ if (timer_start && (watchdog_time - timer_start) > fd->timeout) {
+ fd->timer_start = 0; /* turn off timer */
+ fd->timed_out = TRUE;
+ Jmsg(jcr, M_ERROR, 0, _(
+"Watchdog sending kill after %d secs to thread stalled reading Storage daemon.\n"),
+ watchdog_time - timer_start);
+ pthread_kill(jcr->my_thread_id, TIMEOUT_SIGNAL);
+ }
+ }
+ fd = jcr->file_bsock;
+ if (fd) {
+ timer_start = fd->timer_start;
+ if (timer_start && (watchdog_time - timer_start) > fd->timeout) {
+ fd->timer_start = 0; /* turn off timer */
+ fd->timed_out = TRUE;
+ Jmsg(jcr, M_ERROR, 0, _(
+"Watchdog sending kill after %d secs to thread stalled reading File daemon.\n"),
+ watchdog_time - timer_start);
+ pthread_kill(jcr->my_thread_id, TIMEOUT_SIGNAL);
+ }
+ }
+ fd = jcr->dir_bsock;
+ if (fd) {
+ timer_start = fd->timer_start;
+ if (timer_start && (watchdog_time - timer_start) > fd->timeout) {
+ fd->timer_start = 0; /* turn off timer */
+ fd->timed_out = TRUE;
+ Jmsg(jcr, M_ERROR, 0, _(
+"Watchdog sending kill after %d secs to thread stalled reading Director.\n"),
+ watchdog_time - timer_start);
+ pthread_kill(jcr->my_thread_id, TIMEOUT_SIGNAL);
+ }
+ }
+
+ }
+ unlock_jcr_chain();
+
+ Dmsg0(200, "Finished JCR timeout checks\n");
+}
+
+/*
+ * Timeout signal comes here
+ */
+static void timeout_handler(int sig)
+{
+ return; /* thus interrupting the function */
}
projects / bacula / bacula / blobdiff