2 Bacula® - The Network Backup Solution
4 Copyright (C) 2003-2010 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of Kern Sibbald.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
29 * Bacula job queue routines.
31 * This code consists of three queues, the waiting_jobs
32 * queue, where jobs are initially queued, the ready_jobs
33 * queue, where jobs are placed when all the resources are
34 * allocated and they can immediately be run, and the
35 * running queue where jobs are placed when they are
38 * Kern Sibbald, July MMIII
41 * This code was adapted from the Bacula workq, which was
42 * adapted from "Programming with POSIX Threads", by
52 /* Forward referenced functions */
53 extern "C" void *jobq_server(void *arg);
54 extern "C" void *sched_wait(void *arg);
56 static int start_server(jobq_t *jq);
57 static bool acquire_resources(JCR *jcr);
58 static bool reschedule_job(JCR *jcr, jobq_t *jq, jobq_item_t *je);
59 static void dec_write_store(JCR *jcr);
62 * Initialize a job queue
64 * Returns: 0 on success
67 int jobq_init(jobq_t *jq, int threads, void *(*engine)(void *arg))
70 jobq_item_t *item = NULL;
72 if ((stat = pthread_attr_init(&jq->attr)) != 0) {
74 Jmsg1(NULL, M_ERROR, 0, _("pthread_attr_init: ERR=%s\n"), be.bstrerror(stat));
77 if ((stat = pthread_attr_setdetachstate(&jq->attr, PTHREAD_CREATE_DETACHED)) != 0) {
78 pthread_attr_destroy(&jq->attr);
81 if ((stat = pthread_mutex_init(&jq->mutex, NULL)) != 0) {
83 Jmsg1(NULL, M_ERROR, 0, _("pthread_mutex_init: ERR=%s\n"), be.bstrerror(stat));
84 pthread_attr_destroy(&jq->attr);
87 if ((stat = pthread_cond_init(&jq->work, NULL)) != 0) {
89 Jmsg1(NULL, M_ERROR, 0, _("pthread_cond_init: ERR=%s\n"), be.bstrerror(stat));
90 pthread_mutex_destroy(&jq->mutex);
91 pthread_attr_destroy(&jq->attr);
95 jq->max_workers = threads; /* max threads to create */
96 jq->num_workers = 0; /* no threads yet */
97 jq->idle_workers = 0; /* no idle threads */
98 jq->engine = engine; /* routine to run */
99 jq->valid = JOBQ_VALID;
100 /* Initialize the job queues */
101 jq->waiting_jobs = New(dlist(item, &item->link));
102 jq->running_jobs = New(dlist(item, &item->link));
103 jq->ready_jobs = New(dlist(item, &item->link));
108 * Destroy the job queue
110 * Returns: 0 on success
113 int jobq_destroy(jobq_t *jq)
115 int stat, stat1, stat2;
117 if (jq->valid != JOBQ_VALID) {
121 jq->valid = 0; /* prevent any more operations */
124 * If any threads are active, wake them
126 if (jq->num_workers > 0) {
128 if (jq->idle_workers) {
129 if ((stat = pthread_cond_broadcast(&jq->work)) != 0) {
131 Jmsg1(NULL, M_ERROR, 0, _("pthread_cond_broadcast: ERR=%s\n"), be.bstrerror(stat));
136 while (jq->num_workers > 0) {
137 if ((stat = pthread_cond_wait(&jq->work, &jq->mutex)) != 0) {
139 Jmsg1(NULL, M_ERROR, 0, _("pthread_cond_wait: ERR=%s\n"), be.bstrerror(stat));
146 stat = pthread_mutex_destroy(&jq->mutex);
147 stat1 = pthread_cond_destroy(&jq->work);
148 stat2 = pthread_attr_destroy(&jq->attr);
149 delete jq->waiting_jobs;
150 delete jq->running_jobs;
151 delete jq->ready_jobs;
152 return (stat != 0 ? stat : (stat1 != 0 ? stat1 : stat2));
161 * Wait until schedule time arrives before starting. Normally
162 * this routine is only used for jobs started from the console
163 * for which the user explicitly specified a start time. Otherwise
164 * most jobs are put into the job queue only when their
165 * scheduled time arives.
168 void *sched_wait(void *arg)
170 JCR *jcr = ((wait_pkt *)arg)->jcr;
171 jobq_t *jq = ((wait_pkt *)arg)->jq;
174 Dmsg0(2300, "Enter sched_wait.\n");
176 time_t wtime = jcr->sched_time - time(NULL);
177 set_jcr_job_status(jcr, JS_WaitStartTime);
178 /* Wait until scheduled time arrives */
180 Jmsg(jcr, M_INFO, 0, _("Job %s waiting %d seconds for scheduled start time.\n"),
183 /* Check every 30 seconds if canceled */
185 Dmsg3(2300, "Waiting on sched time, jobid=%d secs=%d use=%d\n",
186 jcr->JobId, wtime, jcr->use_count());
190 bmicrosleep(wtime, 0);
191 if (job_canceled(jcr)) {
194 wtime = jcr->sched_time - time(NULL);
196 Dmsg1(200, "resched use=%d\n", jcr->use_count());
198 free_jcr(jcr); /* we are done with jcr */
199 Dmsg0(2300, "Exit sched_wait\n");
204 * Add a job to the queue
205 * jq is a queue that was created with jobq_init
207 int jobq_add(jobq_t *jq, JCR *jcr)
210 jobq_item_t *item, *li;
211 bool inserted = false;
212 time_t wtime = jcr->sched_time - time(NULL);
216 if (!jcr->term_wait_inited) {
217 /* Initialize termination condition variable */
218 if ((stat = pthread_cond_init(&jcr->term_wait, NULL)) != 0) {
220 Jmsg1(jcr, M_FATAL, 0, _("Unable to init job cond variable: ERR=%s\n"), be.bstrerror(stat));
223 jcr->term_wait_inited = true;
226 Dmsg3(2300, "jobq_add jobid=%d jcr=0x%x use_count=%d\n", jcr->JobId, jcr, jcr->use_count());
227 if (jq->valid != JOBQ_VALID) {
228 Jmsg0(jcr, M_ERROR, 0, "Jobq_add queue not initialized.\n");
232 jcr->inc_use_count(); /* mark jcr in use by us */
233 Dmsg3(2300, "jobq_add jobid=%d jcr=0x%x use_count=%d\n", jcr->JobId, jcr, jcr->use_count());
234 if (!job_canceled(jcr) && wtime > 0) {
235 set_thread_concurrency(jq->max_workers + 2);
236 sched_pkt = (wait_pkt *)malloc(sizeof(wait_pkt));
237 sched_pkt->jcr = jcr;
239 stat = pthread_create(&id, &jq->attr, sched_wait, (void *)sched_pkt);
240 if (stat != 0) { /* thread not created */
242 Jmsg1(jcr, M_ERROR, 0, _("pthread_thread_create: ERR=%s\n"), be.bstrerror(stat));
249 if ((item = (jobq_item_t *)malloc(sizeof(jobq_item_t))) == NULL) {
250 free_jcr(jcr); /* release jcr */
255 /* While waiting in a queue this job is not attached to a thread */
256 set_jcr_in_tsd(INVALID_JCR);
257 if (job_canceled(jcr)) {
258 /* Add job to ready queue so that it is canceled quickly */
259 jq->ready_jobs->prepend(item);
260 Dmsg1(2300, "Prepended job=%d to ready queue\n", jcr->JobId);
262 /* Add this job to the wait queue in priority sorted order */
263 foreach_dlist(li, jq->waiting_jobs) {
264 Dmsg2(2300, "waiting item jobid=%d priority=%d\n",
265 li->jcr->JobId, li->jcr->JobPriority);
266 if (li->jcr->JobPriority > jcr->JobPriority) {
267 jq->waiting_jobs->insert_before(item, li);
268 Dmsg2(2300, "insert_before jobid=%d before waiting job=%d\n",
269 li->jcr->JobId, jcr->JobId);
274 /* If not jobs in wait queue, append it */
276 jq->waiting_jobs->append(item);
277 Dmsg1(2300, "Appended item jobid=%d to waiting queue\n", jcr->JobId);
281 /* Ensure that at least one server looks at the queue. */
282 stat = start_server(jq);
285 Dmsg0(2300, "Return jobq_add\n");
290 * Remove a job from the job queue. Used only by cancel_job().
291 * jq is a queue that was created with jobq_init
292 * work_item is an element of work
294 * Note, it is "removed" from the job queue.
295 * If you want to cancel it, you need to provide some external means
296 * of doing so (e.g. pthread_kill()).
298 int jobq_remove(jobq_t *jq, JCR *jcr)
304 Dmsg2(2300, "jobq_remove jobid=%d jcr=0x%x\n", jcr->JobId, jcr);
305 if (jq->valid != JOBQ_VALID) {
310 foreach_dlist(item, jq->waiting_jobs) {
311 if (jcr == item->jcr) {
318 Dmsg2(2300, "jobq_remove jobid=%d jcr=0x%x not in wait queue\n", jcr->JobId, jcr);
322 /* Move item to be the first on the list */
323 jq->waiting_jobs->remove(item);
324 jq->ready_jobs->prepend(item);
325 Dmsg2(2300, "jobq_remove jobid=%d jcr=0x%x moved to ready queue\n", jcr->JobId, jcr);
327 stat = start_server(jq);
330 Dmsg0(2300, "Return jobq_remove\n");
336 * Start the server thread if it isn't already running
338 static int start_server(jobq_t *jq)
344 * if any threads are idle, wake one.
345 * Actually we do a broadcast because on /lib/tls
346 * these signals seem to get lost from time to time.
348 if (jq->idle_workers > 0) {
349 Dmsg0(2300, "Signal worker to wake up\n");
350 if ((stat = pthread_cond_broadcast(&jq->work)) != 0) {
352 Jmsg1(NULL, M_ERROR, 0, _("pthread_cond_signal: ERR=%s\n"), be.bstrerror(stat));
355 } else if (jq->num_workers < jq->max_workers) {
356 Dmsg0(2300, "Create worker thread\n");
357 /* No idle threads so create a new one */
358 set_thread_concurrency(jq->max_workers + 1);
360 if ((stat = pthread_create(&id, &jq->attr, jobq_server, (void *)jq)) != 0) {
363 Jmsg1(NULL, M_ERROR, 0, _("pthread_create: ERR=%s\n"), be.bstrerror(stat));
372 * This is the worker thread that serves the job queue.
373 * When all the resources are acquired for the job,
374 * it will call the user's engine.
377 void *jobq_server(void *arg)
379 struct timespec timeout;
380 jobq_t *jq = (jobq_t *)arg;
381 jobq_item_t *je; /* job entry in queue */
383 bool timedout = false;
386 set_jcr_in_tsd(INVALID_JCR);
387 Dmsg0(2300, "Start jobq_server\n");
394 Dmsg0(2300, "Top of for loop\n");
395 if (!work && !jq->quit) {
396 gettimeofday(&tv, &tz);
398 timeout.tv_sec = tv.tv_sec + 4;
402 * Wait 4 seconds, then if no more work, exit
404 Dmsg0(2300, "pthread_cond_timedwait()\n");
405 stat = pthread_cond_timedwait(&jq->work, &jq->mutex, &timeout);
406 if (stat == ETIMEDOUT) {
407 Dmsg0(2300, "timedwait timedout.\n");
410 } else if (stat != 0) {
411 /* This shouldn't happen */
412 Dmsg0(2300, "This shouldn't happen\n");
421 * If anything is in the ready queue, run it
423 Dmsg0(2300, "Checking ready queue.\n");
424 while (!jq->ready_jobs->empty() && !jq->quit) {
426 je = (jobq_item_t *)jq->ready_jobs->first();
428 jq->ready_jobs->remove(je);
429 if (!jq->ready_jobs->empty()) {
430 Dmsg0(2300, "ready queue not empty start server\n");
431 if (start_server(jq) != 0) {
437 jq->running_jobs->append(je);
439 /* Attach jcr to this thread while we run the job */
441 Dmsg1(2300, "Took jobid=%d from ready and appended to run\n", jcr->JobId);
443 /* Release job queue lock */
446 /* Call user's routine here */
447 Dmsg3(2300, "Calling user engine for jobid=%d use=%d stat=%c\n", jcr->JobId,
448 jcr->use_count(), jcr->JobStatus);
451 /* Job finished detach from thread */
452 remove_jcr_from_tsd(je->jcr);
454 Dmsg2(2300, "Back from user engine jobid=%d use=%d.\n", jcr->JobId,
457 /* Reacquire job queue lock */
459 Dmsg0(200, "Done lock mutex after running job. Release locks.\n");
460 jq->running_jobs->remove(je);
462 * Release locks if acquired. Note, they will not have
463 * been acquired for jobs canceled before they were
464 * put into the ready queue.
466 if (jcr->acquired_resource_locks) {
468 dec_write_store(jcr);
469 jcr->client->NumConcurrentJobs--;
470 jcr->job->NumConcurrentJobs--;
471 jcr->acquired_resource_locks = false;
474 if (reschedule_job(jcr, jq, je)) {
475 continue; /* go look for more work */
478 /* Clean up and release old jcr */
479 Dmsg2(2300, "====== Termination job=%d use_cnt=%d\n", jcr->JobId, jcr->use_count());
480 jcr->SDJobStatus = 0;
481 V(jq->mutex); /* release internal lock */
483 free(je); /* release job entry */
484 P(jq->mutex); /* reacquire job queue lock */
487 * If any job in the wait queue can be run,
488 * move it to the ready queue
490 Dmsg0(2300, "Done check ready, now check wait queue.\n");
491 if (!jq->waiting_jobs->empty() && !jq->quit) {
493 bool running_allow_mix = false;
494 je = (jobq_item_t *)jq->waiting_jobs->first();
495 jobq_item_t *re = (jobq_item_t *)jq->running_jobs->first();
497 Priority = re->jcr->JobPriority;
498 Dmsg2(2300, "JobId %d is running. Look for pri=%d\n",
499 re->jcr->JobId, Priority);
500 running_allow_mix = true;
502 Dmsg2(2300, "JobId %d is also running with %s\n",
504 re->jcr->job->allow_mixed_priority ? "mix" : "no mix");
505 if (!re->jcr->job->allow_mixed_priority) {
506 running_allow_mix = false;
509 re = (jobq_item_t *)jq->running_jobs->next(re);
511 Dmsg1(2300, "The running job(s) %s mixing priorities.\n",
512 running_allow_mix ? "allow" : "don't allow");
514 Priority = je->jcr->JobPriority;
515 Dmsg1(2300, "No job running. Look for Job pri=%d\n", Priority);
518 * Walk down the list of waiting jobs and attempt
519 * to acquire the resources it needs.
522 /* je is current job item on the queue, jn is the next one */
524 jobq_item_t *jn = (jobq_item_t *)jq->waiting_jobs->next(je);
526 Dmsg4(2300, "Examining Job=%d JobPri=%d want Pri=%d (%s)\n",
527 jcr->JobId, jcr->JobPriority, Priority,
528 jcr->job->allow_mixed_priority ? "mix" : "no mix");
530 /* Take only jobs of correct Priority */
531 if (!(jcr->JobPriority == Priority
532 || (jcr->JobPriority < Priority &&
533 jcr->job->allow_mixed_priority && running_allow_mix))) {
534 set_jcr_job_status(jcr, JS_WaitPriority);
538 if (!acquire_resources(jcr)) {
539 /* If resource conflict, job is canceled */
540 if (!job_canceled(jcr)) {
541 je = jn; /* point to next waiting job */
547 * Got all locks, now remove it from wait queue and append it
548 * to the ready queue. Note, we may also get here if the
549 * job was canceled. Once it is "run", it will quickly
552 jq->waiting_jobs->remove(je);
553 jq->ready_jobs->append(je);
554 Dmsg1(2300, "moved JobId=%d from wait to ready queue\n", je->jcr->JobId);
555 je = jn; /* Point to next waiting job */
560 Dmsg0(2300, "Done checking wait queue.\n");
562 * If no more ready work and we are asked to quit, then do it
564 if (jq->ready_jobs->empty() && jq->quit) {
566 if (jq->num_workers == 0) {
567 Dmsg0(2300, "Wake up destroy routine\n");
568 /* Wake up destroy routine if he is waiting */
569 pthread_cond_broadcast(&jq->work);
573 Dmsg0(2300, "Check for work request\n");
575 * If no more work requests, and we waited long enough, quit
577 Dmsg2(2300, "timedout=%d read empty=%d\n", timedout,
578 jq->ready_jobs->empty());
579 if (jq->ready_jobs->empty() && timedout) {
580 Dmsg0(2300, "break big loop\n");
585 work = !jq->ready_jobs->empty() || !jq->waiting_jobs->empty();
588 * If a job is waiting on a Resource, don't consume all
589 * the CPU time looping looking for work, and even more
590 * important, release the lock so that a job that has
591 * terminated can give us the resource.
594 bmicrosleep(2, 0); /* pause for 2 seconds */
596 /* Recompute work as something may have changed in last 2 secs */
597 work = !jq->ready_jobs->empty() || !jq->waiting_jobs->empty();
599 Dmsg1(2300, "Loop again. work=%d\n", work);
600 } /* end of big for loop */
602 Dmsg0(200, "unlock mutex\n");
604 Dmsg0(2300, "End jobq_server\n");
609 * Returns true if cleanup done and we should look for more work
611 static bool reschedule_job(JCR *jcr, jobq_t *jq, jobq_item_t *je)
614 * Reschedule the job if necessary and requested
616 if (jcr->job->RescheduleOnError &&
617 jcr->JobStatus != JS_Terminated &&
618 jcr->JobStatus != JS_Canceled &&
619 jcr->getJobType() == JT_BACKUP &&
620 (jcr->job->RescheduleTimes == 0 ||
621 jcr->reschedule_count < jcr->job->RescheduleTimes)) {
622 char dt[50], dt2[50];
625 * Reschedule this job by cleaning it up, but
626 * reuse the same JobId if possible.
628 time_t now = time(NULL);
629 jcr->reschedule_count++;
630 jcr->sched_time = now + jcr->job->RescheduleInterval;
631 bstrftime(dt, sizeof(dt), now);
632 bstrftime(dt2, sizeof(dt2), jcr->sched_time);
633 Dmsg4(2300, "Rescheduled Job %s to re-run in %d seconds.(now=%u,then=%u)\n", jcr->Job,
634 (int)jcr->job->RescheduleInterval, now, jcr->sched_time);
635 Jmsg(jcr, M_INFO, 0, _("Rescheduled Job %s at %s to re-run in %d seconds (%s).\n"),
636 jcr->Job, dt, (int)jcr->job->RescheduleInterval, dt2);
637 dird_free_jcr_pointers(jcr); /* partial cleanup old stuff */
639 set_jcr_job_status(jcr, JS_WaitStartTime);
640 jcr->SDJobStatus = 0;
641 if (!allow_duplicate_job(jcr)) {
644 if (jcr->JobBytes == 0) {
645 Dmsg2(2300, "Requeue job=%d use=%d\n", jcr->JobId, jcr->use_count());
647 jobq_add(jq, jcr); /* queue the job to run again */
649 free_jcr(jcr); /* release jcr */
650 free(je); /* free the job entry */
651 return true; /* we already cleaned up */
654 * Something was actually backed up, so we cannot reuse
655 * the old JobId or there will be database record
656 * conflicts. We now create a new job, copying the
657 * appropriate fields.
659 JCR *njcr = new_jcr(sizeof(JCR), dird_free_jcr);
660 set_jcr_defaults(njcr, jcr->job);
661 njcr->reschedule_count = jcr->reschedule_count;
662 njcr->sched_time = jcr->sched_time;
663 njcr->set_JobLevel(jcr->getJobLevel());
664 njcr->pool = jcr->pool;
665 njcr->run_pool_override = jcr->run_pool_override;
666 njcr->full_pool = jcr->full_pool;
667 njcr->run_full_pool_override = jcr->run_full_pool_override;
668 njcr->inc_pool = jcr->inc_pool;
669 njcr->run_inc_pool_override = jcr->run_inc_pool_override;
670 njcr->diff_pool = jcr->diff_pool;
671 njcr->JobStatus = -1;
672 set_jcr_job_status(njcr, jcr->JobStatus);
674 copy_rstorage(njcr, jcr->rstorage, _("previous Job"));
679 copy_wstorage(njcr, jcr->wstorage, _("previous Job"));
683 njcr->messages = jcr->messages;
684 njcr->spool_data = jcr->spool_data;
685 njcr->write_part_after_job = jcr->write_part_after_job;
686 Dmsg0(2300, "Call to run new job\n");
688 run_job(njcr); /* This creates a "new" job */
689 free_jcr(njcr); /* release "new" jcr */
691 Dmsg0(2300, "Back from running new job.\n");
697 * See if we can acquire all the necessary resources for the job (JCR)
699 * Returns: true if successful
700 * false if resource failure
702 static bool acquire_resources(JCR *jcr)
704 bool skip_this_jcr = false;
706 jcr->acquired_resource_locks = false;
708 * Turning this code off is likely to cause some deadlocks,
709 * but we do not really have enough information here to
710 * know if this is really a deadlock (it may be a dual drive
711 * autochanger), and in principle, the SD reservation system
712 * should detect these deadlocks, so push the work off on it.
715 if (jcr->rstore && jcr->rstore == jcr->wstore) { /* possible deadlock */
716 Jmsg(jcr, M_FATAL, 0, _("Job canceled. Attempt to read and write same device.\n"
717 " Read storage \"%s\" (From %s) -- Write storage \"%s\" (From %s)\n"),
718 jcr->rstore->name(), jcr->rstore_source, jcr->wstore->name(), jcr->wstore_source);
719 set_jcr_job_status(jcr, JS_Canceled);
724 Dmsg1(200, "Rstore=%s\n", jcr->rstore->name());
725 if (!inc_read_store(jcr)) {
726 Dmsg1(200, "Fail rncj=%d\n", jcr->rstore->NumConcurrentJobs);
727 set_jcr_job_status(jcr, JS_WaitStoreRes);
733 Dmsg1(200, "Wstore=%s\n", jcr->wstore->name());
734 if (jcr->wstore->NumConcurrentJobs < jcr->wstore->MaxConcurrentJobs) {
735 jcr->wstore->NumConcurrentJobs++;
736 Dmsg1(200, "Inc wncj=%d\n", jcr->wstore->NumConcurrentJobs);
737 } else if (jcr->rstore) {
739 skip_this_jcr = true;
741 Dmsg1(200, "Fail wncj=%d\n", jcr->wstore->NumConcurrentJobs);
742 skip_this_jcr = true;
746 set_jcr_job_status(jcr, JS_WaitStoreRes);
750 if (jcr->client->NumConcurrentJobs < jcr->client->MaxConcurrentJobs) {
751 jcr->client->NumConcurrentJobs++;
753 /* Back out previous locks */
754 dec_write_store(jcr);
756 set_jcr_job_status(jcr, JS_WaitClientRes);
759 if (jcr->job->NumConcurrentJobs < jcr->job->MaxConcurrentJobs) {
760 jcr->job->NumConcurrentJobs++;
762 /* Back out previous locks */
763 dec_write_store(jcr);
765 jcr->client->NumConcurrentJobs--;
766 set_jcr_job_status(jcr, JS_WaitJobRes);
770 jcr->acquired_resource_locks = true;
774 static pthread_mutex_t rstore_mutex = PTHREAD_MUTEX_INITIALIZER;
777 * Note: inc_read_store() and dec_read_store() are
778 * called from select_rstore() in src/dird/restore.c
780 bool inc_read_store(JCR *jcr)
783 if (jcr->rstore->NumConcurrentJobs < jcr->rstore->MaxConcurrentJobs) {
784 jcr->rstore->NumConcurrentReadJobs++;
785 jcr->rstore->NumConcurrentJobs++;
786 Dmsg1(200, "Inc rncj=%d\n", jcr->rstore->NumConcurrentJobs);
794 void dec_read_store(JCR *jcr)
798 jcr->rstore->NumConcurrentReadJobs--; /* back out rstore */
799 jcr->rstore->NumConcurrentJobs--; /* back out rstore */
800 Dmsg1(200, "Dec rncj=%d\n", jcr->rstore->NumConcurrentJobs);
802 ASSERT(jcr->rstore->NumConcurrentReadJobs >= 0);
803 ASSERT(jcr->rstore->NumConcurrentJobs >= 0);
807 static void dec_write_store(JCR *jcr)
810 jcr->wstore->NumConcurrentJobs--;
811 Dmsg1(200, "Dec wncj=%d\n", jcr->wstore->NumConcurrentJobs);
812 ASSERT(jcr->wstore->NumConcurrentJobs >= 0);