3 * Bacula Director Job processing routines
5 * Kern Sibbald, October MM
10 Copyright (C) 2000-2003 Kern Sibbald and John Walker
12 This program is free software; you can redistribute it and/or
13 modify it under the terms of the GNU General Public License as
14 published by the Free Software Foundation; either version 2 of
15 the License, or (at your option) any later version.
17 This program is distributed in the hope that it will be useful,
18 but WITHOUT ANY WARRANTY; without even the implied warranty of
19 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 General Public License for more details.
22 You should have received a copy of the GNU General Public
23 License along with this program; if not, write to the Free
24 Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
32 /* Forward referenced subroutines */
33 static void *job_thread(void *arg);
34 static char *edit_run_codes(JCR *jcr, char *omsg, char *imsg);
35 static void release_resource_locks(JCR *jcr);
36 static int acquire_resource_locks(JCR *jcr);
38 static void backoff_resource_locks(JCR *jcr, int count);
41 /* Exported subroutines */
42 void run_job(JCR *jcr);
45 /* Imported subroutines */
46 extern void term_scheduler();
47 extern void term_ua_server();
48 extern int do_backup(JCR *jcr);
49 extern int do_restore(JCR *jcr);
50 extern int do_verify(JCR *jcr);
51 extern void backup_cleanup(void);
54 static semlock_t job_lock;
55 static pthread_mutex_t mutex;
56 static pthread_cond_t resource_wait;
57 static int waiting = 0; /* count of waiting threads */
59 /* Queue of jobs to be run */
60 workq_t job_wq; /* our job work queue */
63 void init_job_server(int max_workers)
67 if ((stat = sem_init(&job_lock, max_workers)) != 0) {
68 Emsg1(M_ABORT, 0, _("Could not init job lock: ERR=%s\n"), strerror(stat));
70 if ((stat = pthread_mutex_init(&mutex, NULL)) != 0) {
71 Emsg1(M_ABORT, 0, _("Could not init resource mutex: ERR=%s\n"), strerror(stat));
73 if ((stat = pthread_cond_init(&resource_wait, NULL)) != 0) {
74 Emsg1(M_ABORT, 0, _("Could not init resource wait: ERR=%s\n"), strerror(stat));
78 if ((stat = workq_init(&job_wq, max_workers, job_thread)) != 0) {
79 Emsg1(M_ABORT, 0, _("Could not init job work queue: ERR=%s\n"), strerror(stat));
86 * Run a job -- typically called by the scheduler, but may also
87 * be called by the UA (Console program).
90 void run_job(JCR *jcr)
96 workq_ele_t *work_item;
99 sm_check(__FILE__, __LINE__, True);
100 init_msg(jcr, jcr->messages);
101 create_unique_job_name(jcr, jcr->job->hdr.name);
102 jcr->jr.SchedTime = jcr->sched_time;
103 jcr->jr.StartTime = jcr->start_time;
104 jcr->jr.Type = jcr->JobType;
105 jcr->jr.Level = jcr->JobLevel;
106 jcr->jr.JobStatus = jcr->JobStatus;
107 strcpy(jcr->jr.Name, jcr->job->hdr.name);
108 strcpy(jcr->jr.Job, jcr->Job);
110 /* Initialize termination condition variable */
111 if ((errstat = pthread_cond_init(&jcr->term_wait, NULL)) != 0) {
112 Jmsg1(jcr, M_FATAL, 0, _("Unable to init job cond variable: ERR=%s\n"), strerror(errstat));
113 set_jcr_job_status(jcr, JS_ErrorTerminated);
121 Dmsg0(50, "Open database\n");
122 jcr->db=db_init_database(jcr, jcr->catalog->db_name, jcr->catalog->db_user,
123 jcr->catalog->db_password, jcr->catalog->db_address,
124 jcr->catalog->db_port, jcr->catalog->db_socket);
125 if (!db_open_database(jcr, jcr->db)) {
126 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db));
127 db_close_database(jcr, jcr->db);
128 set_jcr_job_status(jcr, JS_ErrorTerminated);
132 Dmsg0(50, "DB opened\n");
137 jcr->jr.JobStatus = jcr->JobStatus;
138 if (!db_create_job_record(jcr, jcr->db, &jcr->jr)) {
139 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db));
140 db_close_database(jcr, jcr->db);
141 set_jcr_job_status(jcr, JS_ErrorTerminated);
145 jcr->JobId = jcr->jr.JobId;
146 ASSERT(jcr->jr.JobId > 0);
148 Dmsg4(30, "Created job record JobId=%d Name=%s Type=%c Level=%c\n",
149 jcr->JobId, jcr->Job, jcr->jr.Type, jcr->jr.Level);
150 Dmsg0(200, "Add jrc to work queue\n");
153 if ((stat = pthread_create(&tid, NULL, job_thread, (void *)jcr)) != 0) {
154 Emsg1(M_ABORT, 0, _("Unable to create job thread: ERR=%s\n"), strerror(stat));
157 /* Queue the job to be run */
158 if ((stat = workq_add(&job_wq, (void *)jcr, &work_item, 0)) != 0) {
159 Emsg1(M_ABORT, 0, _("Could not add job to work queue: ERR=%s\n"), strerror(stat));
161 jcr->work_item = work_item;
163 Dmsg0(200, "Done run_job()\n");
167 * This is the engine called by workq_add() when we were pulled
168 * from the work queue.
169 * At this point, we are running in our own thread
171 static void *job_thread(void *arg)
173 JCR *jcr = (JCR *)arg;
175 pthread_detach(pthread_self());
176 sm_check(__FILE__, __LINE__, True);
178 if (!acquire_resource_locks(jcr)) {
179 set_jcr_job_status(jcr, JS_Cancelled);
182 Dmsg0(200, "=====Start Job=========\n");
183 jcr->start_time = time(NULL); /* set the real start time */
184 Dmsg2(200, "jcr->JobStatus=%d %c\n", jcr->JobStatus, (char)jcr->JobStatus);
185 if (job_cancelled(jcr)) {
186 update_job_end_record(jcr);
187 } else if (jcr->job->MaxStartDelay != 0 && jcr->job->MaxStartDelay <
188 (utime_t)(jcr->start_time - jcr->sched_time)) {
189 Jmsg(jcr, M_FATAL, 0, _("Job cancelled because max start delay time exceeded.\n"));
190 set_jcr_job_status(jcr, JS_Cancelled);
191 update_job_end_record(jcr);
195 set_jcr_job_status(jcr, JS_Running);
197 if (jcr->job->RunBeforeJob) {
198 POOLMEM *before = get_pool_memory(PM_FNAME);
201 before = edit_run_codes(jcr, before, jcr->job->RunBeforeJob);
202 status = run_program(before, 0, NULL);
203 free_pool_memory(before);
205 switch (jcr->JobType) {
208 if (jcr->JobStatus == JS_Terminated) {
214 if (jcr->JobStatus == JS_Terminated) {
220 if (jcr->JobStatus == JS_Terminated) {
227 set_jcr_job_status(jcr, JS_Terminated);
230 Pmsg1(0, "Unimplemented job type: %d\n", jcr->JobType);
233 if (jcr->job->RunAfterJob) {
234 POOLMEM *after = get_pool_memory(PM_FNAME);
237 after = edit_run_codes(jcr, after, jcr->job->RunAfterJob);
238 status = run_program(after, 0, NULL);
239 free_pool_memory(after);
242 release_resource_locks(jcr);
243 Dmsg0(50, "Before free jcr\n");
245 Dmsg0(50, "======== End Job ==========\n");
246 sm_check(__FILE__, __LINE__, True);
251 * Acquire the resources needed. These locks limit the
252 * number of jobs by each resource. We have limits on
253 * Jobs, Clients, Storage, and total jobs.
255 static int acquire_resource_locks(JCR *jcr)
260 if (jcr->store->sem.valid != SEMLOCK_VALID) {
261 if ((stat = sem_init(&jcr->store->sem, jcr->store->MaxConcurrentJobs)) != 0) {
262 Emsg1(M_ABORT, 0, _("Could not init Storage semaphore: ERR=%s\n"), strerror(stat));
265 if (jcr->client->sem.valid != SEMLOCK_VALID) {
266 if ((stat = sem_init(&jcr->client->sem, jcr->client->MaxConcurrentJobs)) != 0) {
267 Emsg1(M_ABORT, 0, _("Could not init Client semaphore: ERR=%s\n"), strerror(stat));
270 if (jcr->job->sem.valid != SEMLOCK_VALID) {
271 if ((stat = sem_init(&jcr->job->sem, jcr->job->MaxConcurrentJobs)) != 0) {
272 Emsg1(M_ABORT, 0, _("Could not init Job semaphore: ERR=%s\n"), strerror(stat));
277 /* Acquire semaphore */
278 set_jcr_job_status(jcr, JS_WaitJobRes);
279 if ((stat = sem_lock(&jcr->job->sem)) != 0) {
280 Emsg1(M_ABORT, 0, _("Could not acquire Job max jobs lock: ERR=%s\n"), strerror(stat));
282 set_jcr_job_status(jcr, JS_WaitClientRes);
283 if ((stat = sem_trylock(&jcr->client->sem)) != 0) {
285 backoff_resource_locks(jcr, 1);
288 Emsg1(M_ABORT, 0, _("Could not acquire Client max jobs lock: ERR=%s\n"), strerror(stat));
291 set_jcr_job_status(jcr, JS_WaitStoreRes);
292 if ((stat = sem_trylock(&jcr->store->sem)) != 0) {
294 backoff_resource_locks(jcr, 2);
297 Emsg1(M_ABORT, 0, _("Could not acquire Storage max jobs lock: ERR=%s\n"), strerror(stat));
300 set_jcr_job_status(jcr, JS_WaitMaxJobs);
301 if ((stat = sem_trylock(&job_lock)) != 0) {
303 backoff_resource_locks(jcr, 3);
306 Emsg1(M_ABORT, 0, _("Could not acquire max jobs lock: ERR=%s\n"), strerror(stat));
314 * Wait for a resource to be released either by backoff or
315 * by a job terminating.
318 pthread_cond_wait(&resource_wait, &mutex);
329 * We could not get all the resource locks because
330 * too many jobs are running, so release any locks
331 * we did acquire, giving others a chance to use them
334 static void backoff_resource_locks(JCR *jcr, int count)
339 sem_unlock(&jcr->store->sem);
340 /* Fall through wanted */
342 sem_unlock(&jcr->client->sem);
343 /* Fall through wanted */
345 sem_unlock(&jcr->job->sem);
349 * Since we released a lock, if there are any threads
350 * waiting, wake them up so that they can try again.
353 pthread_cond_broadcast(&resource_wait);
360 * This is called at the end of the job to release
361 * any resource limits on the number of jobs. If
362 * there are any other jobs waiting, we wake them
363 * up so that they can try again.
365 static void release_resource_locks(JCR *jcr)
369 sem_unlock(&jcr->store->sem);
370 sem_unlock(&jcr->client->sem);
371 sem_unlock(&jcr->job->sem);
372 sem_unlock(&job_lock);
374 pthread_cond_broadcast(&resource_wait);
381 * Get or create a Client record for this Job
383 int get_or_create_client_record(JCR *jcr)
387 memset(&cr, 0, sizeof(cr));
388 strcpy(cr.Name, jcr->client->hdr.name);
389 cr.AutoPrune = jcr->client->AutoPrune;
390 cr.FileRetention = jcr->client->FileRetention;
391 cr.JobRetention = jcr->client->JobRetention;
392 if (jcr->client_name) {
393 free_pool_memory(jcr->client_name);
395 jcr->client_name = get_memory(strlen(jcr->client->hdr.name) + 1);
396 strcpy(jcr->client_name, jcr->client->hdr.name);
397 if (!db_create_client_record(jcr, jcr->db, &cr)) {
398 Jmsg(jcr, M_FATAL, 0, _("Could not create Client record. %s"),
399 db_strerror(jcr->db));
402 jcr->jr.ClientId = cr.ClientId;
404 if (jcr->client_uname) {
405 free_pool_memory(jcr->client_uname);
407 jcr->client_uname = get_memory(strlen(cr.Uname) + 1);
408 strcpy(jcr->client_uname, cr.Uname);
410 Dmsg2(100, "Created Client %s record %d\n", jcr->client->hdr.name,
417 * Write status and such in DB
419 void update_job_end_record(JCR *jcr)
421 if (jcr->jr.EndTime == 0) {
422 jcr->jr.EndTime = time(NULL);
424 jcr->end_time = jcr->jr.EndTime;
425 jcr->jr.JobId = jcr->JobId;
426 jcr->jr.JobStatus = jcr->JobStatus;
427 jcr->jr.JobFiles = jcr->JobFiles;
428 jcr->jr.JobBytes = jcr->JobBytes;
429 jcr->jr.VolSessionId = jcr->VolSessionId;
430 jcr->jr.VolSessionTime = jcr->VolSessionTime;
431 if (!db_update_job_end_record(jcr, jcr->db, &jcr->jr)) {
432 Jmsg(jcr, M_WARNING, 0, _("Error updating job record. %s"),
433 db_strerror(jcr->db));
438 * Takes base_name and appends (unique) current
439 * date and time to form unique job name.
441 * Returns: unique job name in jcr->Job
442 * date/time in jcr->start_time
444 void create_unique_job_name(JCR *jcr, char *base_name)
446 /* Job start mutex */
447 static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
448 static time_t last_start_time = 0;
451 char dt[MAX_TIME_LENGTH];
452 char name[MAX_NAME_LENGTH];
455 /* Guarantee unique start time -- maximum one per second, and
456 * thus unique Job Name
458 P(mutex); /* lock creation of jobs */
460 while (now == last_start_time) {
464 last_start_time = now;
465 V(mutex); /* allow creation of jobs */
466 jcr->start_time = now;
467 /* Form Unique JobName */
468 localtime_r(&now, &tm);
469 /* Use only characters that are permitted in Windows filenames */
470 strftime(dt, sizeof(dt), "%Y-%m-%d_%H.%M.%S", &tm);
471 bstrncpy(name, base_name, sizeof(name));
472 name[sizeof(name)-22] = 0; /* truncate if too long */
473 sprintf(jcr->Job, "%s.%s", name, dt); /* add date & time */
474 /* Convert spaces into underscores */
475 for (p=jcr->Job; *p; p++) {
483 * Free the Job Control Record if no one is still using it.
484 * Called from main free_jcr() routine in src/lib/jcr.c so
485 * that we can do our Director specific cleanup of the jcr.
487 void dird_free_jcr(JCR *jcr)
489 Dmsg0(200, "Start dird free_jcr\n");
491 if (jcr->file_bsock) {
492 Dmsg0(200, "Close File bsock\n");
493 bnet_close(jcr->file_bsock);
495 if (jcr->store_bsock) {
496 Dmsg0(200, "Close Store bsock\n");
497 bnet_close(jcr->store_bsock);
500 Dmsg0(200, "Free JCR fname\n");
501 free_pool_memory(jcr->fname);
504 Dmsg0(200, "Free JCR stime\n");
505 free_pool_memory(jcr->stime);
508 Dmsg0(200, "Close DB\n");
509 db_close_database(jcr, jcr->db);
511 if (jcr->RestoreWhere) {
512 free(jcr->RestoreWhere);
514 if (jcr->RestoreBootstrap) {
515 free(jcr->RestoreBootstrap);
517 Dmsg0(200, "End dird free_jcr\n");
521 * Set some defaults in the JCR necessary to
522 * run. These items are pulled from the job
523 * definition as defaults, but can be overridden
524 * later either by the Run record in the Schedule resource,
525 * or by the Console program.
527 void set_jcr_defaults(JCR *jcr, JOB *job)
530 jcr->JobType = job->JobType;
531 jcr->JobLevel = job->level;
532 jcr->store = job->storage;
533 jcr->client = job->client;
534 if (jcr->client_name) {
535 free_pool_memory(jcr->client_name);
537 jcr->client_name = get_memory(strlen(jcr->client->hdr.name) + 1);
538 strcpy(jcr->client_name, jcr->client->hdr.name);
539 jcr->pool = job->pool;
540 jcr->catalog = job->client->catalog;
541 jcr->fileset = job->fileset;
542 jcr->messages = job->messages;
543 if (jcr->RestoreBootstrap) {
544 free(jcr->RestoreBootstrap);
546 /* This can be overridden by Console program */
547 if (job->RestoreBootstrap) {
548 jcr->RestoreBootstrap = bstrdup(job->RestoreBootstrap);
550 /* If no default level given, set one */
551 if (jcr->JobLevel == 0) {
552 switch (jcr->JobType) {
554 jcr->JobLevel = L_VERIFY_CATALOG;
557 jcr->JobLevel = L_INCREMENTAL;
566 * Edit codes into Run command
569 * %d = Director's name
577 * omsg = edited output message
578 * imsg = input string containing edit codes (%x)
581 static char *edit_run_codes(JCR *jcr, char *omsg, char *imsg)
588 Dmsg1(200, "edit_run_codes: %s\n", imsg);
589 for (p=imsg; *p; p++) {
596 str = jcr->client_name;
605 str = job_status_to_str(jcr->JobStatus);
608 sprintf(add, "%d", jcr->JobId);
615 str = job_level_to_str(jcr->JobLevel);
618 str = jcr->job->hdr.name;
621 str = job_type_to_str(jcr->JobType);
635 Dmsg1(200, "add_str %s\n", str);
636 pm_strcat(&omsg, (char *)str);
637 Dmsg1(200, "omsg=%s\n", omsg);