From 19ec33747123174276b5dd8b0f5702c02e90465d Mon Sep 17 00:00:00 2001 From: Kern Sibbald Date: Sat, 19 Feb 2005 15:29:48 +0000 Subject: [PATCH] - Add back JobId index for MySQL as default -- speeds up pruning. - Add more database fields and fix the update scripts to include the new items. - Pass actual level to FD so that ClientRun editing can reflect correct level -- ditto for job status. This makes the DIR incompatible with older clients! - Move jobq.c acquire resources to static subroutine so that the code logic becomes clearer. This is in preparation for actually using the new Device resources. - Fix some lower case problems in sql_cmds.c reported by Debian. - Correct a seg fault in the SD reported by a user. Occurred only when a high debug level was set. - Modify init_dev() in dev.c to take JCR as first arg so that proper error messages can be reported in next item. - Modify the query and use device SD commands to attempt to open the device if it could not previously be opened. - Correct error message for Could not reserve device. - Correct some minor details with Autochanger resource in SD. git-svn-id: https://bacula.svn.sourceforge.net/svnroot/bacula/trunk@1833 91ce42f0-d328-0410-95d8-f526ca767f89 --- bacula/src/cats/make_mysql_tables.in | 9 +- bacula/src/cats/make_postgresql_tables.in | 9 +- bacula/src/cats/make_sqlite3_tables.in | 4 + bacula/src/cats/make_sqlite_tables.in | 4 + bacula/src/cats/update_mysql_tables.in | 8 +- bacula/src/cats/update_postgresql_tables.in | 12 ++ bacula/src/cats/update_sqlite3_tables.in | 58 +++++- bacula/src/cats/update_sqlite_tables.in | 21 ++- bacula/src/dird/backup.c | 2 +- bacula/src/dird/fd_cmds.c | 25 ++- bacula/src/dird/jobq.c | 172 +++++++++--------- bacula/src/dird/sql_cmds.c | 4 +- bacula/src/filed/job.c | 31 +--- bacula/src/lib/address_conf.c | 14 +- bacula/src/lib/address_conf.h | 2 +- bacula/src/lib/scan.c | 2 +- bacula/src/stored/butil.c | 2 +- bacula/src/stored/dev.c | 26 +-- bacula/src/stored/job.c | 14 +- bacula/src/stored/protos.h | 2 +- bacula/src/stored/stored.c | 2 +- bacula/src/stored/stored_conf.c | 192 ++++++++++---------- bacula/src/version.h | 6 +- 23 files changed, 371 insertions(+), 250 deletions(-) diff --git a/bacula/src/cats/make_mysql_tables.in b/bacula/src/cats/make_mysql_tables.in index 8877617421..6f7d0b6d68 100644 --- a/bacula/src/cats/make_mysql_tables.in +++ b/bacula/src/cats/make_mysql_tables.in @@ -32,6 +32,7 @@ CREATE TABLE File ( LStat TINYBLOB NOT NULL, MD5 TINYBLOB NOT NULL, PRIMARY KEY(FileId), + INDEX (JobId), INDEX (FilenameId, PathId) ); @@ -40,13 +41,10 @@ CREATE TABLE File ( # to the above File table if your Verifies are # too slow. # -# INDEX (JobId), # INDEX (PathId), # INDEX (FilenameId), # INDEX (JobId, PathId, FilenameId) # -# Adding an index on JobId can speed up pruning -# CREATE TABLE Job ( JobId INTEGER UNSIGNED NOT NULL AUTO_INCREMENT, @@ -127,6 +125,7 @@ CREATE TABLE Media ( MaxVolFiles INTEGER UNSIGNED NOT NULL DEFAULT 0, MaxVolBytes BIGINT UNSIGNED NOT NULL DEFAULT 0, InChanger TINYINT NOT NULL DEFAULT 0, + StorageId INTEGER UNSIGNED DEFAULT 0 REFERENCES Storage, MediaAddressing TINYINT NOT NULL DEFAULT 0, VolReadTime BIGINT UNSIGNED NOT NULL DEFAULT 0, VolWriteTime BIGINT UNSIGNED NOT NULL DEFAULT 0, @@ -192,6 +191,10 @@ CREATE TABLE Pool ( Enabled TINYINT DEFAULT 1, ScratchPoolId INTEGER UNSIGNED DEFAULT 0 REFERENCES Pool, RecyclePoolId INTEGER UNSIGNED DEFAULT 0 REFERENCES Pool, + NextPoolId INTEGER UNSIGNED DEFAULT 0 REFERENCES Pool, + MigrationHighBytes BIGINT UNSIGNED DEFAULT 0, + MigrationLowBytes BIGINT UNSIGNED DEFAULT 0, + MigrationTime BIGINT UNSIGNED DEFAULT 0, UNIQUE (Name(128)), PRIMARY KEY (PoolId) ); diff --git a/bacula/src/cats/make_postgresql_tables.in b/bacula/src/cats/make_postgresql_tables.in index 93cfb786e0..97ab280633 100644 --- a/bacula/src/cats/make_postgresql_tables.in +++ b/bacula/src/cats/make_postgresql_tables.in @@ -135,6 +135,7 @@ CREATE TABLE media maxvolfiles integer not null default 0, maxvolbytes bigint not null default 0, inchanger smallint not null default 0, + StorageId integer default 0, mediaaddressing smallint not null default 0, volreadtime bigint not null default 0, volwritetime bigint not null default 0, @@ -200,8 +201,12 @@ CREATE TABLE pool labeltype integer not null default 0, labelformat text not null, enabled smallint not null default 1, - scratchpoolid integer, - recyclepoolid integer, + scratchpoolid integer default 0, + recyclepoolid integer default 0, + NextPoolId integer default 0, + MigrationHighBytes BIGINT DEFAULT 0, + MigrationLowBytes BIGINT DEFAULT 0, + MigrationTime BIGINT DEFAULT 0, primary key (poolid) ); diff --git a/bacula/src/cats/make_sqlite3_tables.in b/bacula/src/cats/make_sqlite3_tables.in index fdaee10675..7bd3bd4b97 100644 --- a/bacula/src/cats/make_sqlite3_tables.in +++ b/bacula/src/cats/make_sqlite3_tables.in @@ -191,6 +191,10 @@ CREATE TABLE Pool ( Enabled TINYINT DEFAULT 1, ScratchPoolId INTEGER UNSIGNED REFERENCES Pool DEFAULT 0, RecyclePoolId INTEGER UNSIGNED REFERENCES Pool DEFAULT 0, + NextPoolId INTEGER UNSIGNED REFERENCES Pool DEFAULT 0, + MigrationHighBytes BIGINT UNSIGNED DEFAULT 0, + MigrationLowBytes BIGINT UNSIGNED DEFAULT 0, + MigrationTime BIGINT UNSIGNED DEFAULT 0, UNIQUE (Name), PRIMARY KEY (PoolId) ); diff --git a/bacula/src/cats/make_sqlite_tables.in b/bacula/src/cats/make_sqlite_tables.in index fdaee10675..7bd3bd4b97 100644 --- a/bacula/src/cats/make_sqlite_tables.in +++ b/bacula/src/cats/make_sqlite_tables.in @@ -191,6 +191,10 @@ CREATE TABLE Pool ( Enabled TINYINT DEFAULT 1, ScratchPoolId INTEGER UNSIGNED REFERENCES Pool DEFAULT 0, RecyclePoolId INTEGER UNSIGNED REFERENCES Pool DEFAULT 0, + NextPoolId INTEGER UNSIGNED REFERENCES Pool DEFAULT 0, + MigrationHighBytes BIGINT UNSIGNED DEFAULT 0, + MigrationLowBytes BIGINT UNSIGNED DEFAULT 0, + MigrationTime BIGINT UNSIGNED DEFAULT 0, UNIQUE (Name), PRIMARY KEY (PoolId) ); diff --git a/bacula/src/cats/update_mysql_tables.in b/bacula/src/cats/update_mysql_tables.in index d088d83469..aeb5cde418 100755 --- a/bacula/src/cats/update_mysql_tables.in +++ b/bacula/src/cats/update_mysql_tables.in @@ -13,9 +13,15 @@ if $bindir/mysql $* -f <JobLevel, jcr->stime); } +static void send_since_time(JCR *jcr) +{ + BSOCK *fd = jcr->file_bsock; + utime_t stime; + char ed1[50]; + + stime = str_to_utime(jcr->stime); + bnet_fsend(fd, levelcmd, "since_utime ", edit_uint64(stime, ed1), 0); + while (bget_dirmsg(fd) >= 0) { /* allow him to poll us to sync clocks */ + Jmsg(jcr, M_INFO, 0, "%s\n", fd->msg); + } +} + /* * Send level command to FD. @@ -194,8 +207,6 @@ void get_level_since_time(JCR *jcr, char *since, int since_len) int send_level_command(JCR *jcr) { BSOCK *fd = jcr->file_bsock; - utime_t stime; - char ed1[50]; /* * Send Level command to File daemon */ @@ -209,12 +220,12 @@ int send_level_command(JCR *jcr) bnet_fsend(fd, levelcmd, "full", " ", 0); break; case L_DIFFERENTIAL: + bnet_fsend(fd, levelcmd, "differential", " ", 0); + send_since_time(jcr); + break; case L_INCREMENTAL: - stime = str_to_utime(jcr->stime); - bnet_fsend(fd, levelcmd, "since_utime ", edit_uint64(stime, ed1), 0); - while (bget_dirmsg(fd) >= 0) { /* allow him to poll us to sync clocks */ - Jmsg(jcr, M_INFO, 0, "%s\n", fd->msg); - } + bnet_fsend(fd, levelcmd, "incremental", " ", 0); + send_since_time(jcr); break; case L_SINCE: default: diff --git a/bacula/src/dird/jobq.c b/bacula/src/dird/jobq.c index 1a9954c8c3..1345e30398 100755 --- a/bacula/src/dird/jobq.c +++ b/bacula/src/dird/jobq.c @@ -46,7 +46,8 @@ extern JCR *jobs; extern "C" void *jobq_server(void *arg); extern "C" void *sched_wait(void *arg); -static int start_server(jobq_t *jq); +static int start_server(jobq_t *jq); +static bool acquire_resources(JCR *jcr); @@ -530,7 +531,7 @@ void *jobq_server(void *arg) * move it to the ready queue */ Dmsg0(2300, "Done check ready, now check wait queue.\n"); - while (!jq->waiting_jobs->empty() && !jq->quit) { + if (!jq->waiting_jobs->empty() && !jq->quit) { int Priority; je = (jobq_item_t *)jq->waiting_jobs->first(); jobq_item_t *re = (jobq_item_t *)jq->running_jobs->first(); @@ -548,79 +549,22 @@ void *jobq_server(void *arg) for ( ; je; ) { /* je is current job item on the queue, jn is the next one */ JCR *jcr = je->jcr; - bool skip_this_jcr = false; jobq_item_t *jn = (jobq_item_t *)jq->waiting_jobs->next(je); + Dmsg3(2300, "Examining Job=%d JobPri=%d want Pri=%d\n", jcr->JobId, jcr->JobPriority, Priority); + /* Take only jobs of correct Priority */ if (jcr->JobPriority != Priority) { set_jcr_job_status(jcr, JS_WaitPriority); break; } - if (jcr->JobType == JT_RESTORE || jcr->JobType == JT_VERIFY) { - /* Let only one Restore/verify job run at a time regardless of MaxConcurrentJobs */ - if (jcr->store->NumConcurrentJobs == 0) { - jcr->store->NumConcurrentJobs = 1; - } else { - set_jcr_job_status(jcr, JS_WaitStoreRes); - je = jn; /* point to next waiting job */ - continue; - } - /* We are not doing a Restore or Verify */ - } else if (jcr->store->NumConcurrentJobs == 0 && - jcr->store->NumConcurrentJobs < jcr->store->MaxConcurrentJobs) { - /* Simple case, first job */ - jcr->store->NumConcurrentJobs = 1; - } else if (jcr->store->NumConcurrentJobs < jcr->store->MaxConcurrentJobs) { - /* - * At this point, we already have at least one Job running - * for this Storage daemon, so we must ensure that there - * is no Volume conflict. In general, it should be OK, if - * all Jobs pull from the same Pool, so we check the Pools. - */ - JCR *njcr; - lock_jcr_chain(); - for (njcr=jobs; njcr; njcr=njcr->next) { - if (njcr->JobId == 0 || njcr == jcr) { - continue; - } - if (njcr->pool != jcr->pool) { - skip_this_jcr = true; - break; - } - } - unlock_jcr_chain(); - if (!skip_this_jcr) { - jcr->store->NumConcurrentJobs++; - } - } else { - skip_this_jcr = true; - } - if (skip_this_jcr) { - set_jcr_job_status(jcr, JS_WaitStoreRes); - je = jn; /* point to next waiting job */ - continue; - } - if (jcr->client->NumConcurrentJobs < jcr->client->MaxConcurrentJobs) { - jcr->client->NumConcurrentJobs++; - } else { - /* Back out previous locks */ - jcr->store->NumConcurrentJobs--; - set_jcr_job_status(jcr, JS_WaitClientRes); - je = jn; /* point to next waiting job */ - continue; - } - if (jcr->job->NumConcurrentJobs < jcr->job->MaxConcurrentJobs) { - jcr->job->NumConcurrentJobs++; - } else { - /* Back out previous locks */ - jcr->store->NumConcurrentJobs--; - jcr->client->NumConcurrentJobs--; - set_jcr_job_status(jcr, JS_WaitJobRes); - je = jn; /* Point to next waiting job */ + if (!acquire_resources(jcr)) { + je = jn; /* point to next waiting job */ continue; } + /* Got all locks, now remove it from wait queue and append it * to the ready queue */ @@ -630,8 +574,9 @@ void *jobq_server(void *arg) Dmsg1(2300, "moved JobId=%d from wait to ready queue\n", je->jcr->JobId); je = jn; /* Point to next waiting job */ } /* end for loop */ - break; - } /* end while loop */ + + } /* end if */ + Dmsg0(2300, "Done checking wait queue.\n"); /* * If no more ready work and we are asked to quit, then do it @@ -665,19 +610,9 @@ void *jobq_server(void *arg) * important, release the lock so that a job that has * terminated can give us the resource. */ - if ((stat = pthread_mutex_unlock(&jq->mutex)) != 0) { - berrno be; - Jmsg1(NULL, M_ERROR, 0, "pthread_mutex_unlock: ERR=%s\n", be.strerror(stat)); - jq->num_workers--; - return NULL; - } + V(jq->mutex); bmicrosleep(2, 0); /* pause for 2 seconds */ - if ((stat = pthread_mutex_lock(&jq->mutex)) != 0) { - berrno be; - Jmsg1(NULL, M_ERROR, 0, "pthread_mutex_lock: ERR=%s\n", be.strerror(stat)); - jq->num_workers--; - return NULL; - } + P(jq->mutex); /* Recompute work as something may have changed in last 2 secs */ work = !jq->ready_jobs->empty() || !jq->waiting_jobs->empty(); } @@ -685,10 +620,83 @@ void *jobq_server(void *arg) } /* end of big for loop */ Dmsg0(200, "unlock mutex\n"); - if ((stat = pthread_mutex_unlock(&jq->mutex)) != 0) { - berrno be; - Jmsg1(NULL, M_ERROR, 0, "pthread_mutex_unlock: ERR=%s\n", be.strerror(stat)); - } + V(jq->mutex); Dmsg0(2300, "End jobq_server\n"); return NULL; } + +/* + * See if we can acquire all the necessary resources for the job (JCR) + * + * Returns: true if successful + * false if resource failure + */ +static bool acquire_resources(JCR *jcr) +{ + bool skip_this_jcr = false; + + if (jcr->JobType == JT_RESTORE || jcr->JobType == JT_VERIFY) { + /* + * Let only one Restore/verify job run at a time regardless + * of MaxConcurrentJobs. + */ + if (jcr->store->NumConcurrentJobs == 0) { + jcr->store->NumConcurrentJobs = 1; + } else { + set_jcr_job_status(jcr, JS_WaitStoreRes); + return false; + } + /* We are not doing a Restore or Verify */ + } else if (jcr->store->NumConcurrentJobs == 0 && + jcr->store->NumConcurrentJobs < jcr->store->MaxConcurrentJobs) { + /* Simple case, first job */ + jcr->store->NumConcurrentJobs = 1; + } else if (jcr->store->NumConcurrentJobs < jcr->store->MaxConcurrentJobs) { + /* + * At this point, we already have at least one Job running + * for this Storage daemon, so we must ensure that there + * is no Volume conflict. In general, it should be OK, if + * all Jobs pull from the same Pool, so we check the Pools. + */ + JCR *njcr; + lock_jcr_chain(); + for (njcr=jobs; njcr; njcr=njcr->next) { + if (njcr->JobId == 0 || njcr == jcr) { + continue; + } + if (njcr->pool != jcr->pool) { + skip_this_jcr = true; + break; + } + } + unlock_jcr_chain(); + if (!skip_this_jcr) { + jcr->store->NumConcurrentJobs++; + } + } else { + skip_this_jcr = true; + } + if (skip_this_jcr) { + set_jcr_job_status(jcr, JS_WaitStoreRes); + return false; + } + + if (jcr->client->NumConcurrentJobs < jcr->client->MaxConcurrentJobs) { + jcr->client->NumConcurrentJobs++; + } else { + /* Back out previous locks */ + jcr->store->NumConcurrentJobs--; + set_jcr_job_status(jcr, JS_WaitClientRes); + return false; + } + if (jcr->job->NumConcurrentJobs < jcr->job->MaxConcurrentJobs) { + jcr->job->NumConcurrentJobs++; + } else { + /* Back out previous locks */ + jcr->store->NumConcurrentJobs--; + jcr->client->NumConcurrentJobs--; + set_jcr_job_status(jcr, JS_WaitJobRes); + return false; + } + return true; +} diff --git a/bacula/src/dird/sql_cmds.c b/bacula/src/dird/sql_cmds.c index b33bbd24d1..a4913ab887 100644 --- a/bacula/src/dird/sql_cmds.c +++ b/bacula/src/dird/sql_cmds.c @@ -147,7 +147,7 @@ const char *select_verify_del = const char *select_restore_del = "SELECT DISTINCT DelCandidates.JobId " "FROM Job,DelCandidates " - "WHERE (Job.JobTdate<%s AND delCandidates.JobStatus!='T') OR " + "WHERE (Job.JobTdate<%s AND DelCandidates.JobStatus!='T') OR " "(Job.JobTDate>%s " "AND Job.ClientId=%u " "AND Job.Type='R')"; @@ -158,7 +158,7 @@ const char *select_restore_del = const char *select_admin_del = "SELECT DISTINCT DelCandidates.JobId " "FROM Job,DelCandidates " - "WHERE (Job.JobTdate<%s AND delCandidates.JobStatus!='T') OR " + "WHERE (Job.JobTdate<%s AND DelCandidates.JobStatus!='T') OR " "(Job.JobTDate>%s " "AND Job.ClientId=%u " "AND Job.Type='D')"; diff --git a/bacula/src/filed/job.c b/bacula/src/filed/job.c index 6690d949c8..8b055faa22 100644 --- a/bacula/src/filed/job.c +++ b/bacula/src/filed/job.c @@ -1110,8 +1110,6 @@ static int level_cmd(JCR *jcr) { BSOCK *dir = jcr->dir_bsock; POOLMEM *level, *buf = NULL; - struct tm tm; - time_t mtime; int mtime_only; level = get_memory(dir->msglen+1); @@ -1125,26 +1123,14 @@ static int level_cmd(JCR *jcr) /* Full backup requested? */ } else if (strcmp(level, "full") == 0) { jcr->JobLevel = L_FULL; - /* - * Backup requested since