X-Git-Url: https://git.sur5r.net/?a=blobdiff_plain;f=bacula%2Fsrc%2Fdird%2Fmigrate.c;h=43abf952588cd664818842b28d4b1fd677de5674;hb=c38dc738eeca4a6b0a8ecdeee4829efab6e99416;hp=38f51f31a848e71ebfc0d1aff3bbc7f08d014eec;hpb=09c8c88482741e8f3e906be9159fa68015a9e3ec;p=bacula%2Fbacula diff --git a/bacula/src/dird/migrate.c b/bacula/src/dird/migrate.c index 38f51f31a8..43abf95258 100644 --- a/bacula/src/dird/migrate.c +++ b/bacula/src/dird/migrate.c @@ -1,7 +1,36 @@ +/* + Bacula® - The Network Backup Solution + + Copyright (C) 2004-2010 Free Software Foundation Europe e.V. + + The main author of Bacula is Kern Sibbald, with contributions from + many others, a complete list can be found in the file AUTHORS. + This program is Free Software; you can redistribute it and/or + modify it under the terms of version three of the GNU Affero General Public + License as published by the Free Software Foundation and included + in the file LICENSE. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + 02110-1301, USA. + + Bacula® is a registered trademark of Kern Sibbald. + The licensor of Bacula is the Free Software Foundation Europe + (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich, + Switzerland, email:ftf@fsfeurope.org. +*/ /* * * Bacula Director -- migrate.c -- responsible for doing - * migration jobs. + * migration and copy jobs. + * + * Also handles Copy jobs (March MMVIII) * * Kern Sibbald, September MMIV * @@ -12,21 +41,6 @@ * to do the backup. * When the Storage daemon finishes the job, update the DB. * - * Version $Id$ - */ -/* - Copyright (C) 2004-2006 Kern Sibbald - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - version 2 as amended with additional clauses defined in the - file LICENSE in the main source directory. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - the file LICENSE for additional details. - */ #include "bacula.h" @@ -38,111 +52,145 @@ #include #endif -static const int dbglevel = 100; +static const int dbglevel = 10; -static char OKbootstrap[] = "3000 OK bootstrap\n"; -static bool get_job_to_migrate(JCR *jcr); +static int getJob_to_migrate(JCR *jcr); struct idpkt; static bool regex_find_jobids(JCR *jcr, idpkt *ids, const char *query1, const char *query2, const char *type); static bool find_mediaid_then_jobids(JCR *jcr, idpkt *ids, const char *query1, const char *type); static bool find_jobids_from_mediaid_list(JCR *jcr, idpkt *ids, const char *type); +static bool find_jobids_of_pool_uncopied_jobs(JCR *jcr, idpkt *ids); static void start_migration_job(JCR *jcr); static int get_next_dbid_from_list(char **p, DBId_t *DBId); +static bool set_migration_next_pool(JCR *jcr, POOL **pool); /* * Called here before the job is run to do the job - * specific setup. + * specific setup. Note, one of the important things to + * complete in this init code is to make the definitive + * choice of input and output storage devices. This is + * because immediately after the init, the job is queued + * in the jobq.c code, and it checks that all the resources + * (storage resources in particular) are available, so these + * must all be properly defined. + * + * previous_jr refers to the job DB record of the Job that is + * going to be migrated. + * prev_job refers to the job resource of the Job that is + * going to be migrated. + * jcr is the jcr for the current "migration" job. It is a + * control job that is put in the DB as a migration job, which + * means that this job migrated a previous job to a new job. + * No Volume or File data is associated with this control + * job. + * mig_jcr refers to the newly migrated job that is run by + * the current jcr. It is a backup job that moves (migrates) the + * data written for the previous_jr into the new pool. This + * job (mig_jcr) becomes the new backup job that replaces + * the original backup job. Note, this jcr is not really run. It + * is simply attached to the current jcr. It will show up in + * the Director's status output, but not in the SD or FD, both of + * which deal only with the current migration job (i.e. jcr). */ bool do_migration_init(JCR *jcr) { - /* If we find a job or jobs to migrate it is previous_jr.JobId */ - if (!get_job_to_migrate(jcr)) { + POOL *pool = NULL; + JOB *job, *prev_job; + JCR *mig_jcr; /* newly migrated job */ + int count; + + + apply_pool_overrides(jcr); + + if (!allow_duplicate_job(jcr)) { return false; } - if (jcr->previous_jr.JobId == 0) { - return true; /* no work */ + jcr->jr.PoolId = get_or_create_pool_record(jcr, jcr->pool->name()); + if (jcr->jr.PoolId == 0) { + Dmsg1(dbglevel, "JobId=%d no PoolId\n", (int)jcr->JobId); + Jmsg(jcr, M_FATAL, 0, _("Could not get or create a Pool record.\n")); + return false; } + /* + * Note, at this point, pool is the pool for this job. We + * transfer it to rpool (read pool), and a bit later, + * pool will be changed to point to the write pool, + * which comes from pool->NextPool. + */ + jcr->rpool = jcr->pool; /* save read pool */ + pm_strcpy(jcr->rpool_source, jcr->pool_source); + + + Dmsg2(dbglevel, "Read pool=%s (From %s)\n", jcr->rpool->name(), jcr->rpool_source); if (!get_or_create_fileset_record(jcr)) { + Dmsg1(dbglevel, "JobId=%d no FileSet\n", (int)jcr->JobId); + Jmsg(jcr, M_FATAL, 0, _("Could not get or create the FileSet record.\n")); return false; } - apply_pool_overrides(jcr); - - jcr->jr.PoolId = get_or_create_pool_record(jcr, jcr->pool->hdr.name); - if (jcr->jr.PoolId == 0) { + /* If we find a job or jobs to migrate it is previous_jr.JobId */ + count = getJob_to_migrate(jcr); + if (count < 0) { return false; } + if (count == 0) { + set_migration_next_pool(jcr, &pool); + return true; /* no work */ + } - /* If pool storage specified, use it instead of job storage */ - copy_wstorage(jcr, jcr->pool->storage, _("Pool resource")); + Dmsg1(dbglevel, "Back from getJob_to_migrate JobId=%d\n", (int)jcr->JobId); - if (!jcr->wstorage) { - Jmsg(jcr, M_FATAL, 0, _("No Storage specification found in Job or Pool.\n")); - return false; + if (jcr->previous_jr.JobId == 0) { + Dmsg1(dbglevel, "JobId=%d no previous JobId\n", (int)jcr->JobId); + Jmsg(jcr, M_INFO, 0, _("No previous Job found to %s.\n"), jcr->get_ActionName(0)); + set_migration_next_pool(jcr, &pool); + return true; /* no work */ } - create_restore_bootstrap_file(jcr); - return true; -} - -/* - * Do a Migration of a previous job - * - * Returns: false on failure - * true on success - */ -bool do_migration(JCR *jcr) -{ - POOL_DBR pr; - POOL *pool; - char ed1[100]; - BSOCK *sd; - JOB *job, *prev_job; - JCR *mig_jcr; /* newly migrated job */ + if (create_restore_bootstrap_file(jcr) < 0) { + Jmsg(jcr, M_FATAL, 0, _("Create bootstrap file failed.\n")); + return false; + } - /* - * previous_jr refers to the job DB record of the Job that is - * going to be migrated. - * prev_job refers to the job resource of the Job that is - * going to be migrated. - * jcr is the jcr for the current "migration" job. It is a - * control job that is put in the DB as a migration job, which - * means that this job migrated a previous job to a new job. - * No Volume or File data is associated with this control - * job. - * mig_jcr refers to the newly migrated job that is run by - * the current jcr. It is a backup job that moves (migrates) the - * data written for the previous_jr into the new pool. This - * job (mig_jcr) becomes the new backup job that replaces - * the original backup job. - */ if (jcr->previous_jr.JobId == 0 || jcr->ExpectedFiles == 0) { - set_jcr_job_status(jcr, JS_Terminated); - migration_cleanup(jcr, jcr->JobStatus); + jcr->setJobStatus(JS_Terminated); + Dmsg1(dbglevel, "JobId=%d expected files == 0\n", (int)jcr->JobId); + if (jcr->previous_jr.JobId == 0) { + Jmsg(jcr, M_INFO, 0, _("No previous Job found to %s.\n"), jcr->get_ActionName(0)); + } else { + Jmsg(jcr, M_INFO, 0, _("Previous Job has no data to %s.\n"), jcr->get_ActionName(0)); + } + set_migration_next_pool(jcr, &pool); return true; /* no work */ } - Dmsg4(dbglevel, "Previous: Name=%s JobId=%d Type=%c Level=%c\n", - jcr->previous_jr.Name, jcr->previous_jr.JobId, - jcr->previous_jr.JobType, jcr->previous_jr.JobLevel); - Dmsg4(dbglevel, "Current: Name=%s JobId=%d Type=%c Level=%c\n", - jcr->jr.Name, jcr->jr.JobId, + Dmsg5(dbglevel, "JobId=%d: Current: Name=%s JobId=%d Type=%c Level=%c\n", + (int)jcr->JobId, + jcr->jr.Name, (int)jcr->jr.JobId, jcr->jr.JobType, jcr->jr.JobLevel); LockRes(); job = (JOB *)GetResWithName(R_JOB, jcr->jr.Name); prev_job = (JOB *)GetResWithName(R_JOB, jcr->previous_jr.Name); UnlockRes(); - if (!job || !prev_job) { + if (!job) { + Jmsg(jcr, M_FATAL, 0, _("Job resource not found for \"%s\".\n"), jcr->jr.Name); return false; } + if (!prev_job) { + Jmsg(jcr, M_FATAL, 0, _("Previous Job resource not found for \"%s\".\n"), + jcr->previous_jr.Name); + return false; + } + + jcr->spool_data = job->spool_data; /* turn on spooling if requested in job */ - /* Create a migation jcr */ + /* Create a migration jcr */ mig_jcr = jcr->mig_jcr = new_jcr(sizeof(JCR), dird_free_jcr); memcpy(&mig_jcr->previous_jr, &jcr->previous_jr, sizeof(mig_jcr->previous_jr)); @@ -152,26 +200,59 @@ bool do_migration(JCR *jcr) */ set_jcr_defaults(mig_jcr, prev_job); if (!setup_job(mig_jcr)) { + Jmsg(jcr, M_FATAL, 0, _("setup job failed.\n")); return false; } /* Now reset the job record from the previous job */ memcpy(&mig_jcr->jr, &jcr->previous_jr, sizeof(mig_jcr->jr)); - /* Update the jr to reflect the new values of PoolId, FileSetId, and JobId. */ + /* Update the jr to reflect the new values of PoolId and JobId. */ mig_jcr->jr.PoolId = jcr->jr.PoolId; - mig_jcr->jr.FileSetId = jcr->jr.FileSetId; mig_jcr->jr.JobId = mig_jcr->JobId; + /* Don't let WatchDog checks Max*Time value on this Job */ + mig_jcr->no_maxtime = true; + + /* + * Don't check for duplicates on migration and copy jobs + */ + mig_jcr->job->IgnoreDuplicateJobChecking = true; + Dmsg4(dbglevel, "mig_jcr: Name=%s JobId=%d Type=%c Level=%c\n", - mig_jcr->jr.Name, mig_jcr->jr.JobId, + mig_jcr->jr.Name, (int)mig_jcr->jr.JobId, mig_jcr->jr.JobType, mig_jcr->jr.JobLevel); + if (set_migration_next_pool(jcr, &pool)) { + /* If pool storage specified, use it for restore */ + copy_rstorage(mig_jcr, pool->storage, _("Pool resource")); + copy_rstorage(jcr, pool->storage, _("Pool resource")); + + mig_jcr->pool = jcr->pool; + mig_jcr->jr.PoolId = jcr->jr.PoolId; + } + + return true; +} + + +/* + * set_migration_next_pool() called by do_migration_init() + * at differents stages. + * The idea here is tofactorize the NextPool's search code and + * to permit do_migration_init() to return with NextPool set in jcr struct. + */ +static bool set_migration_next_pool(JCR *jcr, POOL **retpool) +{ + POOL_DBR pr; + POOL *pool; + char ed1[100]; + /* * Get the PoolId used with the original job. Then * find the pool name from the database record. */ memset(&pr, 0, sizeof(pr)); - pr.PoolId = mig_jcr->previous_jr.PoolId; + pr.PoolId = jcr->jr.PoolId; if (!db_get_pool_record(jcr, jcr->db, &pr)) { Jmsg(jcr, M_FATAL, 0, _("Pool for JobId %s not in database. ERR=%s\n"), edit_int64(pr.PoolId, ed1), db_strerror(jcr->db)); @@ -179,61 +260,81 @@ bool do_migration(JCR *jcr) } /* Get the pool resource corresponding to the original job */ pool = (POOL *)GetResWithName(R_POOL, pr.Name); + *retpool = pool; if (!pool) { Jmsg(jcr, M_FATAL, 0, _("Pool resource \"%s\" not found.\n"), pr.Name); return false; } - /* If pool storage specified, use it for restore */ - copy_rstorage(mig_jcr, pool->storage, _("Pool resource")); - copy_rstorage(jcr, pool->storage, _("Pool resource")); - /* * If the original backup pool has a NextPool, make sure a * record exists in the database. Note, in this case, we * will be migrating from pool to pool->NextPool. */ if (pool->NextPool) { - jcr->jr.PoolId = get_or_create_pool_record(jcr, pool->NextPool->hdr.name); + jcr->jr.PoolId = get_or_create_pool_record(jcr, pool->NextPool->name()); if (jcr->jr.PoolId == 0) { return false; } - /* - * put the "NextPool" resource pointer in our jcr so that we - * can pull the Storage reference from it. - */ - mig_jcr->pool = jcr->pool = pool->NextPool; - mig_jcr->jr.PoolId = jcr->jr.PoolId; - pm_strcpy(jcr->pool_source, _("NextPool in Pool resource")); } + if (!set_migration_wstorage(jcr, pool)) { + return false; + } + jcr->pool = pool->NextPool; + pm_strcpy(jcr->pool_source, _("Job Pool's NextPool resource")); - /* If pool storage specified, use it instead of job storage for backup */ - copy_wstorage(jcr, jcr->pool->storage, _("Pool resource")); + Dmsg2(dbglevel, "Write pool=%s read rpool=%s\n", jcr->pool->name(), jcr->rpool->name()); - /* Print Job Start message */ - Jmsg(jcr, M_INFO, 0, _("Start Migration JobId %s, Job=%s\n"), - edit_uint64(jcr->JobId, ed1), jcr->Job); + return true; +} - set_jcr_job_status(jcr, JS_Running); - set_jcr_job_status(mig_jcr, JS_Running); - Dmsg2(dbglevel, "JobId=%d JobLevel=%c\n", jcr->jr.JobId, jcr->jr.JobLevel); - /* Update job start record for this migration control job */ - if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) { - Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db)); - return false; - } +/* + * Do a Migration of a previous job + * + * Returns: false on failure + * true on success + */ +bool do_migration(JCR *jcr) +{ + char ed1[100]; + BSOCK *sd; + JCR *mig_jcr = jcr->mig_jcr; /* newly migrated job */ - Dmsg4(dbglevel, "mig_jcr: Name=%s JobId=%d Type=%c Level=%c\n", - mig_jcr->jr.Name, mig_jcr->jr.JobId, - mig_jcr->jr.JobType, mig_jcr->jr.JobLevel); + /* + * If mig_jcr is NULL, there is nothing to do for this job, + * so set a normal status, cleanup and return OK. + */ + if (!mig_jcr) { + jcr->setJobStatus(JS_Terminated); + migration_cleanup(jcr, jcr->JobStatus); + return true; + } - /* Update job start record for the real migration backup job */ - if (!db_update_job_start_record(mig_jcr, mig_jcr->db, &mig_jcr->jr)) { - Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(mig_jcr->db)); - return false; + if (!db_get_job_record(jcr, jcr->db, &jcr->previous_jr)) { + Jmsg(jcr, M_FATAL, 0, _("Could not get job record for JobId %s to %s. ERR=%s"), + edit_int64(jcr->previous_jr.JobId, ed1), + jcr->get_ActionName(0), + db_strerror(jcr->db)); + jcr->setJobStatus(JS_Terminated); + migration_cleanup(jcr, jcr->JobStatus); + return true; + } + /* Make sure this job was not already migrated */ + if (jcr->previous_jr.JobType != JT_BACKUP && + jcr->previous_jr.JobType != JT_JOB_COPY) { + Jmsg(jcr, M_INFO, 0, _("JobId %s already %s probably by another Job. %s stopped.\n"), + edit_int64(jcr->previous_jr.JobId, ed1), + jcr->get_ActionName(1), + jcr->get_OperationName()); + jcr->setJobStatus(JS_Terminated); + migration_cleanup(jcr, jcr->JobStatus); + return true; } + /* Print Job Start message */ + Jmsg(jcr, M_INFO, 0, _("Start %s JobId %s, Job=%s\n"), + jcr->get_OperationName(), edit_uint64(jcr->JobId, ed1), jcr->Job); /* * Open a message channel connection with the Storage @@ -242,8 +343,8 @@ bool do_migration(JCR *jcr) * */ Dmsg0(110, "Open connection with storage daemon\n"); - set_jcr_job_status(jcr, JS_WaitSD); - set_jcr_job_status(mig_jcr, JS_WaitSD); + jcr->setJobStatus(JS_WaitSD); + mig_jcr->setJobStatus(JS_WaitSD); /* * Start conversation with Storage daemon */ @@ -257,17 +358,57 @@ bool do_migration(JCR *jcr) Dmsg2(dbglevel, "Read store=%s, write store=%s\n", ((STORE *)jcr->rstorage->first())->name(), ((STORE *)jcr->wstorage->first())->name()); - if (!start_storage_daemon_job(jcr, jcr->rstorage, jcr->wstorage)) { + + if (!start_storage_daemon_job(jcr, jcr->rstorage, jcr->wstorage, /*send_bsr*/true)) { return false; } Dmsg0(150, "Storage daemon connection OK\n"); - if (!send_bootstrap_file(jcr, sd) || - !response(jcr, sd, OKbootstrap, "Bootstrap", DISPLAY_ERROR)) { + + /* + * We re-update the job start record so that the start + * time is set after the run before job. This avoids + * that any files created by the run before job will + * be saved twice. They will be backed up in the current + * job, but not in the next one unless they are changed. + * Without this, they will be backed up in this job and + * in the next job run because in that case, their date + * is after the start of this run. + */ + jcr->start_time = time(NULL); + jcr->jr.StartTime = jcr->start_time; + jcr->jr.JobTDate = jcr->start_time; + jcr->setJobStatus(JS_Running); + + /* Update job start record for this migration control job */ + if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) { + Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db)); + return false; + } + + + mig_jcr->start_time = time(NULL); + mig_jcr->jr.StartTime = mig_jcr->start_time; + mig_jcr->jr.JobTDate = mig_jcr->start_time; + mig_jcr->setJobStatus(JS_Running); + + /* Update job start record for the real migration backup job */ + if (!db_update_job_start_record(mig_jcr, mig_jcr->db, &mig_jcr->jr)) { + Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(mig_jcr->db)); return false; } - if (!bnet_fsend(sd, "run")) { + Dmsg4(dbglevel, "mig_jcr: Name=%s JobId=%d Type=%c Level=%c\n", + mig_jcr->jr.Name, (int)mig_jcr->jr.JobId, + mig_jcr->jr.JobType, mig_jcr->jr.JobLevel); + + + /* + * Start the job prior to starting the message thread below + * to avoid two threads from using the BSOCK structure at + * the same time. + */ + if (!sd->fsend("run")) { return false; } @@ -279,23 +420,20 @@ bool do_migration(JCR *jcr) } - set_jcr_job_status(jcr, JS_Running); - set_jcr_job_status(mig_jcr, JS_Running); + jcr->setJobStatus(JS_Running); + mig_jcr->setJobStatus(JS_Running); /* Pickup Job termination data */ - /* Note, the SD stores in jcr->JobFiles/ReadBytes/JobBytes/Errors */ + /* Note, the SD stores in jcr->JobFiles/ReadBytes/JobBytes/JobErrors */ wait_for_storage_daemon_termination(jcr); - - set_jcr_job_status(jcr, jcr->SDJobStatus); + jcr->setJobStatus(jcr->SDJobStatus); + db_write_batch_file_records(jcr); /* used by bulk batch file insert */ if (jcr->JobStatus != JS_Terminated) { return false; } + migration_cleanup(jcr, jcr->JobStatus); - if (mig_jcr) { - UAContext *ua = new_ua_context(jcr); - purge_files_from_job(ua, jcr->previous_jr.JobId); - free_ua_context(ua); - } + return true; } @@ -304,21 +442,57 @@ struct idpkt { uint32_t count; }; -/* - * Callback handler make list of DB Ids - */ -static int dbid_handler(void *ctx, int num_fields, char **row) +/* Add an item to the list if it is unique */ +static void add_unique_id(idpkt *ids, char *item) { - idpkt *ids = (idpkt *)ctx; - - Dmsg3(dbglevel, "count=%d Ids=%p %s\n", ids->count, ids->list, ids->list); + const int maxlen = 30; + char id[maxlen+1]; + char *q = ids->list; + + /* Walk through current list to see if each item is the same as item */ + for ( ; *q; ) { + id[0] = 0; + for (int i=0; icount == 0) { ids->list[0] = 0; } else { pm_strcat(ids->list, ","); } - pm_strcat(ids->list, row[0]); + pm_strcat(ids->list, item); ids->count++; +// Dmsg3(0, "add_uniq count=%d Ids=%p %s\n", ids->count, ids->list, ids->list); + return; +} + +/* + * Callback handler make list of DB Ids + */ +static int unique_dbid_handler(void *ctx, int num_fields, char **row) +{ + idpkt *ids = (idpkt *)ctx; + + /* Sanity check */ + if (!row || !row[0]) { + Dmsg0(dbglevel, "dbid_hdlr error empty row\n"); + return 1; /* stop calling us */ + } + + add_unique_id(ids, row[0]); + Dmsg3(dbglevel, "dbid_hdlr count=%d Ids=%p %s\n", ids->count, ids->list, ids->list); return 0; } @@ -344,7 +518,7 @@ static int unique_name_handler(void *ctx, int num_fields, char **row) memset(new_item, 0, sizeof(uitem)); new_item->item = bstrdup(row[0]); - Dmsg1(dbglevel, "Item=%s\n", row[0]); + Dmsg1(dbglevel, "Unique_name_hdlr Item=%s\n", row[0]); item = (uitem *)list->binary_insert((void *)new_item, item_compare); if (item != new_item) { /* already in list */ free(new_item->item); @@ -373,9 +547,10 @@ const char *sql_client = /* Get JobIds from regex'ed Client names */ const char *sql_jobids_from_client = - "SELECT DISTINCT Job.JobId,Job.StartTime FROM Job,Pool" + "SELECT DISTINCT Job.JobId,Job.StartTime FROM Job,Pool,Client" " WHERE Client.Name='%s' AND Pool.Name='%s' AND Job.PoolId=Pool.PoolId" - " AND Job.ClientId=Client.ClientId " + " AND Job.ClientId=Client.ClientId AND Job.Type IN ('B','C')" + " AND Job.JobStatus IN ('T','W')" " ORDER by Job.StartTime"; /* Get Volume names in Pool */ @@ -388,39 +563,44 @@ const char *sql_vol = const char *sql_jobids_from_vol = "SELECT DISTINCT Job.JobId,Job.StartTime FROM Media,JobMedia,Job" " WHERE Media.VolumeName='%s' AND Media.MediaId=JobMedia.MediaId" - " AND JobMedia.JobId=Job.JobId" + " AND JobMedia.JobId=Job.JobId AND Job.Type IN ('B','C')" + " AND Job.JobStatus IN ('T','W') AND Media.Enabled=1" " ORDER by Job.StartTime"; - const char *sql_smallest_vol = - "SELECT MediaId FROM Media,Pool WHERE" - " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND" + "SELECT Media.MediaId FROM Media,Pool,JobMedia WHERE" + " Media.MediaId in (SELECT DISTINCT MediaId from JobMedia) AND" + " Media.VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND" " Media.PoolId=Pool.PoolId AND Pool.Name='%s'" " ORDER BY VolBytes ASC LIMIT 1"; const char *sql_oldest_vol = - "SELECT MediaId FROM Media,Pool WHERE" - " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND" + "SELECT Media.MediaId FROM Media,Pool,JobMedia WHERE" + " Media.MediaId in (SELECT DISTINCT MediaId from JobMedia) AND" + " Media.VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND" " Media.PoolId=Pool.PoolId AND Pool.Name='%s'" " ORDER BY LastWritten ASC LIMIT 1"; /* Get JobIds when we have selected MediaId */ const char *sql_jobids_from_mediaid = "SELECT DISTINCT Job.JobId,Job.StartTime FROM JobMedia,Job" - " WHERE JobMedia.JobId=Job.JobId AND JobMedia.MediaId=%s" + " WHERE JobMedia.JobId=Job.JobId AND JobMedia.MediaId IN (%s)" + " AND Job.Type IN ('B','C') AND Job.JobStatus IN ('T','W')" " ORDER by Job.StartTime"; -/* Get tne number of bytes in the pool */ +/* Get the number of bytes in the pool */ const char *sql_pool_bytes = - "SELECT SUM(VolBytes) FROM Media,Pool WHERE" + "SELECT SUM(JobBytes) FROM Job WHERE JobId IN" + " (SELECT DISTINCT Job.JobId from Pool,Job,Media,JobMedia WHERE" + " Pool.Name='%s' AND Media.PoolId=Pool.PoolId AND" " VolStatus in ('Full','Used','Error','Append') AND Media.Enabled=1 AND" - " Media.PoolId=Pool.PoolId AND Pool.Name='%s'"; + " Job.Type IN ('B','C') AND Job.JobStatus IN ('T','W') AND" + " JobMedia.JobId=Job.JobId AND Job.PoolId=Media.PoolId)"; -/* Get tne number of bytes in the Jobs */ +/* Get the number of bytes in the Jobs */ const char *sql_job_bytes = "SELECT SUM(JobBytes) FROM Job WHERE JobId IN (%s)"; - /* Get Media Ids in Pool */ const char *sql_mediaids = "SELECT MediaId FROM Media,Pool WHERE" @@ -429,12 +609,25 @@ const char *sql_mediaids = /* Get JobIds in Pool longer than specified time */ const char *sql_pool_time = - "SELECT DISTINCT Job.JobId from Pool,Job,Media,JobMedia WHERE" + "SELECT DISTINCT Job.JobId FROM Pool,Job,Media,JobMedia WHERE" " Pool.Name='%s' AND Media.PoolId=Pool.PoolId AND" - " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND" + " VolStatus IN ('Full','Used','Error') AND Media.Enabled=1 AND" + " Job.Type IN ('B','C') AND Job.JobStatus IN ('T','W') AND" " JobMedia.JobId=Job.JobId AND Job.PoolId=Media.PoolId" " AND Job.RealEndTime<='%s'"; +/* Get JobIds from successfully completed backup jobs which have not been copied before */ +const char *sql_jobids_of_pool_uncopied_jobs = + "SELECT DISTINCT Job.JobId,Job.StartTime FROM Job,Pool" + " WHERE Pool.Name = '%s' AND Pool.PoolId = Job.PoolId" + " AND Job.Type = 'B' AND Job.JobStatus IN ('T','W')" + " AND Job.jobBytes > 0" + " AND Job.JobId NOT IN" + " (SELECT PriorJobId FROM Job WHERE" + " Type IN ('B','C') AND Job.JobStatus IN ('T','W')" + " AND PriorJobId != 0)" + " ORDER by Job.StartTime"; + /* * const char *sql_ujobid = * "SELECT DISTINCT Job.Job from Client,Pool,Media,Job,JobMedia " @@ -442,8 +635,6 @@ const char *sql_pool_time = * " JobMedia.JobId=Job.JobId AND Job.PoolId=Media.PoolId"; */ - - /* * * This is the central piece of code that finds a job or jobs @@ -457,24 +648,25 @@ const char *sql_pool_time = * one starting a new job with MigrationJobId set to that JobId, and * finally, it returns the last JobId to the caller. * - * Returns: false on error - * true if OK and jcr->previous_jr filled in + * Returns: -1 on error + * 0 if no jobs to migrate + * 1 if OK and jcr->previous_jr filled in */ -static bool get_job_to_migrate(JCR *jcr) +static int getJob_to_migrate(JCR *jcr) { - char ed1[30]; + char ed1[30], ed2[30]; POOL_MEM query(PM_MESSAGE); JobId_t JobId; - DBId_t MediaId = 0; + DBId_t DBId = 0; int stat; char *p; idpkt ids, mid, jids; db_int64_ctx ctx; int64_t pool_bytes; - bool ok; time_t ttime; struct tm tm; char dt[MAX_TIME_LENGTH]; + int count = 0; ids.list = get_pool_memory(PM_MESSAGE); ids.list[0] = 0; @@ -486,7 +678,6 @@ static bool get_job_to_migrate(JCR *jcr) jids.list[0] = 0; jids.count = 0; - /* * If MigrateJobId is set, then we migrate only that Job, * otherwise, we go through the full selection of jobs to @@ -494,8 +685,7 @@ static bool get_job_to_migrate(JCR *jcr) */ if (jcr->MigrateJobId != 0) { Dmsg1(dbglevel, "At Job start previous jobid=%u\n", jcr->MigrateJobId); - edit_uint64(jcr->MigrateJobId, ids.list); - ids.count = 1; + JobId = jcr->MigrateJobId; } else { switch (jcr->job->selection_type) { case MT_JOB: @@ -515,12 +705,12 @@ static bool get_job_to_migrate(JCR *jcr) break; case MT_SQLQUERY: if (!jcr->job->selection_pattern) { - Jmsg(jcr, M_FATAL, 0, _("No Migration SQL selection pattern specified.\n")); + Jmsg(jcr, M_FATAL, 0, _("No %s SQL selection pattern specified.\n"), jcr->get_OperationName()); goto bail_out; } Dmsg1(dbglevel, "SQL=%s\n", jcr->job->selection_pattern); if (!db_sql_query(jcr->db, jcr->job->selection_pattern, - dbid_handler, (void *)&ids)) { + unique_dbid_handler, (void *)&ids)) { Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db)); goto bail_out; @@ -536,171 +726,190 @@ static bool get_job_to_migrate(JCR *jcr) goto bail_out; } break; - case MT_POOL_OCCUPANCY: ctx.count = 0; /* Find count of bytes in pool */ - Mmsg(query, sql_pool_bytes, jcr->pool->hdr.name); + Mmsg(query, sql_pool_bytes, jcr->rpool->name()); if (!db_sql_query(jcr->db, query.c_str(), db_int64_handler, (void *)&ctx)) { Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db)); goto bail_out; } if (ctx.count == 0) { - Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n")); + Jmsg(jcr, M_INFO, 0, _("No Volumes found to %s.\n"), jcr->get_ActionName(0)); goto ok_out; } pool_bytes = ctx.value; - Dmsg2(dbglevel, "highbytes=%d pool=%d\n", (int)jcr->pool->MigrationHighBytes, - (int)pool_bytes); - if (pool_bytes < (int64_t)jcr->pool->MigrationHighBytes) { - Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n")); + Dmsg2(dbglevel, "highbytes=%lld pool=%lld\n", jcr->rpool->MigrationHighBytes, + pool_bytes); + if (pool_bytes < (int64_t)jcr->rpool->MigrationHighBytes) { + Jmsg(jcr, M_INFO, 0, _("No Volumes found to %s.\n"), jcr->get_ActionName(0)); goto ok_out; } Dmsg0(dbglevel, "We should do Occupation migration.\n"); ids.count = 0; /* Find a list of MediaIds that could be migrated */ - Mmsg(query, sql_mediaids, jcr->pool->hdr.name); -// Dmsg1(dbglevel, "query=%s\n", query.c_str()); - if (!db_sql_query(jcr->db, query.c_str(), dbid_handler, (void *)&ids)) { + Mmsg(query, sql_mediaids, jcr->rpool->name()); + Dmsg1(dbglevel, "query=%s\n", query.c_str()); + if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)&ids)) { Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db)); goto bail_out; } if (ids.count == 0) { - Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n")); + Jmsg(jcr, M_INFO, 0, _("No Volumes found to %s.\n"), jcr->get_ActionName(0)); goto ok_out; } Dmsg2(dbglevel, "Pool Occupancy ids=%d MediaIds=%s\n", ids.count, ids.list); - /* - * Now loop over MediaIds getting more JobIds to migrate until - * we reduce the pool occupancy below the low water mark. - */ + if (!find_jobids_from_mediaid_list(jcr, &ids, "Volume")) { + goto bail_out; + } + /* ids == list of jobs */ p = ids.list; for (int i=0; i < (int)ids.count; i++) { - stat = get_next_dbid_from_list(&p, &MediaId); - Dmsg2(dbglevel, "get_next_dbid stat=%d MediaId=%u\n", stat, MediaId); + stat = get_next_dbid_from_list(&p, &DBId); + Dmsg2(dbglevel, "get_next_dbid stat=%d JobId=%u\n", stat, (uint32_t)DBId); if (stat < 0) { - Jmsg(jcr, M_FATAL, 0, _("Invalid MediaId found.\n")); + Jmsg(jcr, M_FATAL, 0, _("Invalid JobId found.\n")); goto bail_out; } else if (stat == 0) { break; } + mid.count = 1; - Mmsg(mid.list, "%s", edit_int64(MediaId, ed1)); - ok = find_jobids_from_mediaid_list(jcr, &mid, "Volumes"); - if (!ok) { - continue; - } - if (i != 0) { + Mmsg(mid.list, "%s", edit_int64(DBId, ed1)); + if (jids.count > 0) { pm_strcat(jids.list, ","); } pm_strcat(jids.list, mid.list); jids.count += mid.count; - /* Now get the count of bytes added */ - ctx.count = 0; /* Find count of bytes from Jobs */ Mmsg(query, sql_job_bytes, mid.list); + Dmsg1(dbglevel, "Jobbytes query: %s\n", query.c_str()); if (!db_sql_query(jcr->db, query.c_str(), db_int64_handler, (void *)&ctx)) { Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db)); goto bail_out; } pool_bytes -= ctx.value; - Dmsg1(dbglevel, "Job bytes=%d\n", (int)ctx.value); - Dmsg2(dbglevel, "lowbytes=%d pool=%d\n", (int)jcr->pool->MigrationLowBytes, - (int)pool_bytes); - if (pool_bytes <= (int64_t)jcr->pool->MigrationLowBytes) { + Dmsg2(dbglevel, "Total %s Job bytes=%s\n", jcr->get_ActionName(0), edit_int64_with_commas(ctx.value, ed1)); + Dmsg2(dbglevel, "lowbytes=%s poolafter=%s\n", + edit_int64_with_commas(jcr->rpool->MigrationLowBytes, ed1), + edit_int64_with_commas(pool_bytes, ed2)); + if (pool_bytes <= (int64_t)jcr->rpool->MigrationLowBytes) { Dmsg0(dbglevel, "We should be done.\n"); break; } - } - Dmsg2(dbglevel, "Pool Occupancy ids=%d JobIds=%s\n", jids.count, jids.list); - + /* Transfer jids to ids, where the jobs list is expected */ + ids.count = jids.count; + pm_strcpy(ids.list, jids.list); + Dmsg2(dbglevel, "Pool Occupancy ids=%d JobIds=%s\n", ids.count, ids.list); break; - case MT_POOL_TIME: - ttime = time(NULL) - (time_t)jcr->pool->MigrationTime; + ttime = time(NULL) - (time_t)jcr->rpool->MigrationTime; (void)localtime_r(&ttime, &tm); strftime(dt, sizeof(dt), "%Y-%m-%d %H:%M:%S", &tm); ids.count = 0; - Mmsg(query, sql_pool_time, jcr->pool->hdr.name, dt); -// Dmsg1(000, "query=%s\n", query.c_str()); - if (!db_sql_query(jcr->db, query.c_str(), dbid_handler, (void *)&ids)) { + Mmsg(query, sql_pool_time, jcr->rpool->name(), dt); + Dmsg1(dbglevel, "query=%s\n", query.c_str()); + if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)&ids)) { Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db)); goto bail_out; } if (ids.count == 0) { - Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n")); + Jmsg(jcr, M_INFO, 0, _("No Volumes found to %s.\n"), jcr->get_ActionName(0)); goto ok_out; } Dmsg2(dbglevel, "PoolTime ids=%d JobIds=%s\n", ids.count, ids.list); break; - + case MT_POOL_UNCOPIED_JOBS: + if (!find_jobids_of_pool_uncopied_jobs(jcr, &ids)) { + goto bail_out; + } + break; default: - Jmsg(jcr, M_FATAL, 0, _("Unknown Migration Selection Type.\n")); + Jmsg(jcr, M_FATAL, 0, _("Unknown %s Selection Type.\n"), jcr->get_OperationName()); goto bail_out; } - } - /* - * Loop over all jobids except the last one, sending - * them to start_migration_job(), which will start a job - * for each of them. For the last JobId, we handle it below. - */ - p = ids.list; - for (int i=1; i < (int)ids.count; i++) { + /* + * Loop over all jobids except the last one, sending + * them to start_migration_job(), which will start a job + * for each of them. For the last JobId, we handle it below. + */ + p = ids.list; + if (ids.count == 0) { + Jmsg(jcr, M_INFO, 0, _("No JobIds found to %s.\n"), jcr->get_ActionName(0)); + goto ok_out; + } + + Jmsg(jcr, M_INFO, 0, _("The following %u JobId%s chosen to be %s: %s\n"), + ids.count, (ids.count < 2) ? _(" was") : _("s were"), + jcr->get_ActionName(1), ids.list); + + Dmsg2(dbglevel, "Before loop count=%d ids=%s\n", ids.count, ids.list); + for (int i=1; i < (int)ids.count; i++) { + JobId = 0; + stat = get_next_jobid_from_list(&p, &JobId); + Dmsg3(dbglevel, "getJobid_no=%d stat=%d JobId=%u\n", i, stat, JobId); + if (stat < 0) { + Jmsg(jcr, M_FATAL, 0, _("Invalid JobId found.\n")); + goto bail_out; + } else if (stat == 0) { + Jmsg(jcr, M_INFO, 0, _("No JobIds found to %s.\n"), jcr->get_ActionName(0)); + goto ok_out; + } + jcr->MigrateJobId = JobId; + start_migration_job(jcr); + Dmsg0(dbglevel, "Back from start_migration_job\n"); + } + + /* Now get the last JobId and handle it in the current job */ JobId = 0; stat = get_next_jobid_from_list(&p, &JobId); - Dmsg2(dbglevel, "get_next_jobid stat=%d JobId=%u\n", stat, JobId); - jcr->MigrateJobId = JobId; - start_migration_job(jcr); + Dmsg2(dbglevel, "Last get_next_jobid stat=%d JobId=%u\n", stat, (int)JobId); if (stat < 0) { Jmsg(jcr, M_FATAL, 0, _("Invalid JobId found.\n")); goto bail_out; } else if (stat == 0) { - Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n")); + Jmsg(jcr, M_INFO, 0, _("No JobIds found to %s.\n"), jcr->get_ActionName(0)); goto ok_out; } } - - /* Now get the last JobId and handle it in the current job */ - JobId = 0; - stat = get_next_jobid_from_list(&p, &JobId); - Dmsg2(dbglevel, "Last get_next_jobid stat=%d JobId=%u\n", stat, JobId); - if (stat < 0) { - Jmsg(jcr, M_FATAL, 0, _("Invalid JobId found.\n")); - goto bail_out; - } else if (stat == 0) { - Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n")); - goto ok_out; - } jcr->previous_jr.JobId = JobId; - Dmsg1(100, "Previous jobid=%d\n", jcr->previous_jr.JobId); + Dmsg1(dbglevel, "Previous jobid=%d\n", (int)jcr->previous_jr.JobId); if (!db_get_job_record(jcr, jcr->db, &jcr->previous_jr)) { - Jmsg(jcr, M_FATAL, 0, _("Could not get job record for JobId %s to migrate. ERR=%s"), + Jmsg(jcr, M_FATAL, 0, _("Could not get job record for JobId %s to %s. ERR=%s"), edit_int64(jcr->previous_jr.JobId, ed1), + jcr->get_ActionName(0), db_strerror(jcr->db)); goto bail_out; } - Jmsg(jcr, M_INFO, 0, _("Migration using JobId=%d Job=%s\n"), - jcr->previous_jr.JobId, jcr->previous_jr.Job); + + Jmsg(jcr, M_INFO, 0, _("%s using JobId=%s Job=%s\n"), + jcr->get_OperationName(), + edit_int64(jcr->previous_jr.JobId, ed1), jcr->previous_jr.Job); + Dmsg4(dbglevel, "%s JobId=%d using JobId=%s Job=%s\n", + jcr->get_OperationName(), + jcr->JobId, + edit_int64(jcr->previous_jr.JobId, ed1), jcr->previous_jr.Job); + count = 1; ok_out: - free_pool_memory(ids.list); - free_pool_memory(mid.list); - free_pool_memory(jids.list); - return true; + goto out; bail_out: + count = -1; + +out: free_pool_memory(ids.list); free_pool_memory(mid.list); free_pool_memory(jids.list); - return false; + return count; } static void start_migration_job(JCR *jcr) @@ -708,15 +917,15 @@ static void start_migration_job(JCR *jcr) UAContext *ua = new_ua_context(jcr); char ed1[50]; ua->batch = true; - Mmsg(ua->cmd, "run %s jobid=%s", jcr->job->hdr.name, + Mmsg(ua->cmd, "run job=\"%s\" jobid=%s ignoreduplicatecheck=yes", jcr->job->name(), edit_uint64(jcr->MigrateJobId, ed1)); - Dmsg1(dbglevel, "=============== Migration cmd=%s\n", ua->cmd); + Dmsg2(dbglevel, "=============== %s cmd=%s\n", jcr->get_OperationName(), ua->cmd); parse_ua_args(ua); /* parse command */ - int stat = run_cmd(ua, ua->cmd); - if (stat == 0) { + JobId_t jobid = run_cmd(ua, ua->cmd); + if (jobid == 0) { Jmsg(jcr, M_ERROR, 0, _("Could not start migration job.\n")); } else { - Jmsg(jcr, M_INFO, 0, _("Migration JobId %d started.\n"), stat); + Jmsg(jcr, M_INFO, 0, _("%s JobId %d started.\n"), jcr->get_OperationName(), (int)jobid); } free_ua_context(ua); } @@ -729,20 +938,20 @@ static bool find_mediaid_then_jobids(JCR *jcr, idpkt *ids, const char *query1, ids->count = 0; /* Basic query for MediaId */ - Mmsg(query, query1, jcr->pool->hdr.name); - if (!db_sql_query(jcr->db, query.c_str(), dbid_handler, (void *)ids)) { + Mmsg(query, query1, jcr->rpool->name()); + if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) { Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db)); goto bail_out; } if (ids->count == 0) { - Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type); - } - if (ids->count != 1) { - Jmsg(jcr, M_FATAL, 0, _("SQL logic error. Count should be 1 but is %d\n"), - ids->count); + Jmsg(jcr, M_INFO, 0, _("No %s found to %s.\n"), type, jcr->get_ActionName(0)); + ok = true; /* Not an error */ + goto bail_out; + } else if (ids->count != 1) { + Jmsg(jcr, M_FATAL, 0, _("SQL error. Expected 1 MediaId got %d\n"), ids->count); goto bail_out; } - Dmsg1(dbglevel, "Smallest Vol Jobids=%s\n", ids->list); + Dmsg2(dbglevel, "%s MediaIds=%s\n", type, ids->list); ok = find_jobids_from_mediaid_list(jcr, ids, type); @@ -750,6 +959,12 @@ bail_out: return ok; } +/* + * This routine returns: + * false if an error occurred + * true otherwise + * ids.count number of jobids found (may be zero) + */ static bool find_jobids_from_mediaid_list(JCR *jcr, idpkt *ids, const char *type) { bool ok = false; @@ -757,14 +972,47 @@ static bool find_jobids_from_mediaid_list(JCR *jcr, idpkt *ids, const char *type Mmsg(query, sql_jobids_from_mediaid, ids->list); ids->count = 0; - if (!db_sql_query(jcr->db, query.c_str(), dbid_handler, (void *)ids)) { + if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) { Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db)); goto bail_out; } if (ids->count == 0) { - Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type); + Jmsg(jcr, M_INFO, 0, _("No %ss found to %s.\n"), type, jcr->get_ActionName(0)); } ok = true; + +bail_out: + return ok; +} + +/* + * This routine returns: + * false if an error occurred + * true otherwise + * ids.count number of jobids found (may be zero) + */ +static bool find_jobids_of_pool_uncopied_jobs(JCR *jcr, idpkt *ids) +{ + bool ok = false; + POOL_MEM query(PM_MESSAGE); + + /* Only a copy job is allowed */ + if (jcr->getJobType() != JT_COPY) { + Jmsg(jcr, M_FATAL, 0, + _("Selection Type 'pooluncopiedjobs' only applies to Copy Jobs")); + goto bail_out; + } + + Dmsg1(dbglevel, "copy selection pattern=%s\n", jcr->rpool->name()); + Mmsg(query, sql_jobids_of_pool_uncopied_jobs, jcr->rpool->name()); + Dmsg1(dbglevel, "get uncopied jobs query=%s\n", query.c_str()); + if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) { + Jmsg(jcr, M_FATAL, 0, + _("SQL to get uncopied jobs failed. ERR=%s\n"), db_strerror(jcr->db)); + goto bail_out; + } + ok = true; + bail_out: return ok; } @@ -783,51 +1031,65 @@ static bool regex_find_jobids(JCR *jcr, idpkt *ids, const char *query1, item_chain = New(dlist(item, &item->link)); if (!jcr->job->selection_pattern) { - Jmsg(jcr, M_FATAL, 0, _("No Migration %s selection pattern specified.\n"), - type); - goto bail_out; - } - Dmsg1(dbglevel, "regex=%s\n", jcr->job->selection_pattern); - /* Compile regex expression */ - rc = regcomp(&preg, jcr->job->selection_pattern, REG_EXTENDED); - if (rc != 0) { - regerror(rc, &preg, prbuf, sizeof(prbuf)); - Jmsg(jcr, M_FATAL, 0, _("Could not compile regex pattern \"%s\" ERR=%s\n"), - jcr->job->selection_pattern, prbuf); + Jmsg(jcr, M_FATAL, 0, _("No %s %s selection pattern specified.\n"), + jcr->get_OperationName(), type); goto bail_out; } + Dmsg1(dbglevel, "regex-sel-pattern=%s\n", jcr->job->selection_pattern); /* Basic query for names */ - Mmsg(query, query1, jcr->pool->hdr.name); - Dmsg1(dbglevel, "query1=%s\n", query.c_str()); + Mmsg(query, query1, jcr->rpool->name()); + Dmsg1(dbglevel, "get name query1=%s\n", query.c_str()); if (!db_sql_query(jcr->db, query.c_str(), unique_name_handler, (void *)item_chain)) { Jmsg(jcr, M_FATAL, 0, _("SQL to get %s failed. ERR=%s\n"), type, db_strerror(jcr->db)); goto bail_out; } - /* Now apply the regex to the names and remove any item not matched */ - foreach_dlist(item, item_chain) { - const int nmatch = 30; - regmatch_t pmatch[nmatch]; + Dmsg1(dbglevel, "query1 returned %d names\n", item_chain->size()); + if (item_chain->size() == 0) { + Jmsg(jcr, M_INFO, 0, _("Query of Pool \"%s\" returned no Jobs to %s.\n"), + jcr->rpool->name(), jcr->get_ActionName(0)); + ok = true; + goto bail_out; /* skip regex match */ + } else { + /* Compile regex expression */ + rc = regcomp(&preg, jcr->job->selection_pattern, REG_EXTENDED); + if (rc != 0) { + regerror(rc, &preg, prbuf, sizeof(prbuf)); + Jmsg(jcr, M_FATAL, 0, _("Could not compile regex pattern \"%s\" ERR=%s\n"), + jcr->job->selection_pattern, prbuf); + goto bail_out; + } + /* Now apply the regex to the names and remove any item not matched */ + foreach_dlist(item, item_chain) { + const int nmatch = 30; + regmatch_t pmatch[nmatch]; + if (last_item) { + Dmsg1(dbglevel, "Remove item %s\n", last_item->item); + free(last_item->item); + item_chain->remove(last_item); + } + Dmsg1(dbglevel, "get name Item=%s\n", item->item); + rc = regexec(&preg, item->item, nmatch, pmatch, 0); + if (rc == 0) { + last_item = NULL; /* keep this one */ + } else { + last_item = item; + } + } if (last_item) { - Dmsg1(dbglevel, "Remove item %s\n", last_item->item); free(last_item->item); + Dmsg1(dbglevel, "Remove item %s\n", last_item->item); item_chain->remove(last_item); } - Dmsg1(dbglevel, "Item=%s\n", item->item); - rc = regexec(&preg, item->item, nmatch, pmatch, 0); - if (rc == 0) { - last_item = NULL; /* keep this one */ - } else { - last_item = item; - } + regfree(&preg); } - if (last_item) { - free(last_item->item); - Dmsg1(dbglevel, "Remove item %s\n", last_item->item); - item_chain->remove(last_item); + if (item_chain->size() == 0) { + Jmsg(jcr, M_INFO, 0, _("Regex pattern matched no Jobs to %s.\n"), jcr->get_ActionName(0)); + ok = true; + goto bail_out; /* skip regex match */ } - regfree(&preg); + /* * At this point, we have a list of items in item_chain * that have been matched by the regex, so now we need @@ -836,26 +1098,28 @@ static bool regex_find_jobids(JCR *jcr, idpkt *ids, const char *query1, ids->count = 0; foreach_dlist(item, item_chain) { Dmsg2(dbglevel, "Got %s: %s\n", type, item->item); - Mmsg(query, query2, item->item, jcr->pool->hdr.name); - Dmsg1(dbglevel, "query2=%s\n", query.c_str()); - if (!db_sql_query(jcr->db, query.c_str(), dbid_handler, (void *)ids)) { + Mmsg(query, query2, item->item, jcr->rpool->name()); + Dmsg1(dbglevel, "get id from name query2=%s\n", query.c_str()); + if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) { Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db)); goto bail_out; } } if (ids->count == 0) { - Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type); + Jmsg(jcr, M_INFO, 0, _("No %ss found to %s.\n"), type, jcr->get_ActionName(0)); } ok = true; + bail_out: Dmsg2(dbglevel, "Count=%d Jobids=%s\n", ids->count, ids->list); + foreach_dlist(item, item_chain) { + free(item->item); + } delete item_chain; - Dmsg0(dbglevel, "After delete item_chain\n"); return ok; } - /* * Release resources allocated during backup. */ @@ -866,7 +1130,7 @@ void migration_cleanup(JCR *jcr, int TermCode) char ec6[50], ec7[50], ec8[50]; char term_code[100], sd_term_msg[100]; const char *term_msg; - int msg_type; + int msg_type = M_INFO; MEDIA_DBR mr; double kbps; utime_t RunTime; @@ -874,16 +1138,19 @@ void migration_cleanup(JCR *jcr, int TermCode) POOL_MEM query(PM_MESSAGE); Dmsg2(100, "Enter migrate_cleanup %d %c\n", TermCode, TermCode); - dequeue_messages(jcr); /* display any queued messages */ + update_job_end(jcr, TermCode); memset(&mr, 0, sizeof(mr)); - set_jcr_job_status(jcr, TermCode); - update_job_end_record(jcr); /* update database */ /* * Check if we actually did something. * mig_jcr is jcr of the newly migrated job. */ if (mig_jcr) { + char old_jobid[50], new_jobid[50]; + + edit_uint64(jcr->previous_jr.JobId, old_jobid); + edit_uint64(mig_jcr->jr.JobId, new_jobid); + mig_jcr->JobFiles = jcr->JobFiles = jcr->SDJobFiles; mig_jcr->JobBytes = jcr->JobBytes = jcr->SDJobBytes; mig_jcr->VolSessionId = jcr->VolSessionId; @@ -891,42 +1158,70 @@ void migration_cleanup(JCR *jcr, int TermCode) mig_jcr->jr.RealEndTime = 0; mig_jcr->jr.PriorJobId = jcr->previous_jr.JobId; - set_jcr_job_status(mig_jcr, TermCode); - - - update_job_end_record(mig_jcr); + update_job_end(mig_jcr, TermCode); /* Update final items to set them to the previous job's values */ Mmsg(query, "UPDATE Job SET StartTime='%s',EndTime='%s'," "JobTDate=%s WHERE JobId=%s", jcr->previous_jr.cStartTime, jcr->previous_jr.cEndTime, edit_uint64(jcr->previous_jr.JobTDate, ec1), - edit_uint64(mig_jcr->jr.JobId, ec2)); + new_jobid); db_sql_query(mig_jcr->db, query.c_str(), NULL, NULL); - /* Now marke the previous job as migrated */ - Mmsg(query, "UPDATE Job SET Type='%c' WHERE JobId=%s", - (char)JT_MIGRATED_JOB, edit_uint64(jcr->previous_jr.JobId, ec1)); - db_sql_query(mig_jcr->db, query.c_str(), NULL, NULL); + /* + * If we terminated a migration normally: + * - mark the previous job as migrated + * - move any Log records to the new JobId + * - Purge the File records from the previous job + */ + if (jcr->getJobType() == JT_MIGRATE && jcr->JobStatus == JS_Terminated) { + Mmsg(query, "UPDATE Job SET Type='%c' WHERE JobId=%s", + (char)JT_MIGRATED_JOB, old_jobid); + db_sql_query(mig_jcr->db, query.c_str(), NULL, NULL); + UAContext *ua = new_ua_context(jcr); + /* Move JobLog to new JobId */ + Mmsg(query, "UPDATE Log SET JobId=%s WHERE JobId=%s", + new_jobid, old_jobid); + db_sql_query(mig_jcr->db, query.c_str(), NULL, NULL); + + if (jcr->job->PurgeMigrateJob) { + /* Purge old Job record */ + purge_jobs_from_catalog(ua, old_jobid); + } else { + /* Purge all old file records, but leave Job record */ + purge_files_from_jobs(ua, old_jobid); + } + + free_ua_context(ua); + } + + /* + * If we terminated a Copy (rather than a Migration) normally: + * - copy any Log records to the new JobId + * - set type="Job Copy" for the new job + */ + if (jcr->getJobType() == JT_COPY && jcr->JobStatus == JS_Terminated) { + /* Copy JobLog to new JobId */ + Mmsg(query, "INSERT INTO Log (JobId, Time, LogText ) " + "SELECT %s, Time, LogText FROM Log WHERE JobId=%s", + new_jobid, old_jobid); + db_sql_query(mig_jcr->db, query.c_str(), NULL, NULL); + Mmsg(query, "UPDATE Job SET Type='%c' WHERE JobId=%s", + (char)JT_JOB_COPY, new_jobid); + db_sql_query(mig_jcr->db, query.c_str(), NULL, NULL); + } if (!db_get_job_record(jcr, jcr->db, &jcr->jr)) { - Jmsg(jcr, M_WARNING, 0, _("Error getting job record for stats: %s"), + Jmsg(jcr, M_WARNING, 0, _("Error getting Job record for Job report: ERR=%s"), db_strerror(jcr->db)); - set_jcr_job_status(jcr, JS_ErrorTerminated); - } - - bstrncpy(mr.VolumeName, jcr->VolumeName, sizeof(mr.VolumeName)); - if (!db_get_media_record(jcr, jcr->db, &mr)) { - Jmsg(jcr, M_WARNING, 0, _("Error getting Media record for Volume \"%s\": ERR=%s"), - mr.VolumeName, db_strerror(jcr->db)); - set_jcr_job_status(jcr, JS_ErrorTerminated); + jcr->setJobStatus(JS_ErrorTerminated); } update_bootstrap_file(mig_jcr); if (!db_get_job_volume_names(mig_jcr, mig_jcr->db, mig_jcr->jr.JobId, &mig_jcr->VolumeName)) { /* - * Note, if the job has erred, most likely it did not write any + * Note, if the job has failed, most likely it did not write any * tape, so suppress this "error" message since in that case * it is normal. Or look at it the other way, only for a * normal exit should we complain about this error. @@ -936,42 +1231,65 @@ void migration_cleanup(JCR *jcr, int TermCode) } mig_jcr->VolumeName[0] = 0; /* none */ } - } - msg_type = M_INFO; /* by default INFO message */ - switch (jcr->JobStatus) { - case JS_Terminated: - if (jcr->Errors || jcr->SDErrors) { - term_msg = _("%s OK -- with warnings"); - } else { - term_msg = _("%s OK"); - } - break; - case JS_FatalError: - case JS_ErrorTerminated: - term_msg = _("*** %s Error ***"); - msg_type = M_ERROR; /* Generate error message */ - if (jcr->store_bsock) { - bnet_sig(jcr->store_bsock, BNET_TERMINATE); - if (jcr->SD_msg_chan) { - pthread_cancel(jcr->SD_msg_chan); + if (mig_jcr->VolumeName[0]) { + /* Find last volume name. Multiple vols are separated by | */ + char *p = strrchr(mig_jcr->VolumeName, '|'); + if (p) { + p++; /* skip | */ + } else { + p = mig_jcr->VolumeName; /* no |, take full name */ + } + bstrncpy(mr.VolumeName, p, sizeof(mr.VolumeName)); + if (!db_get_media_record(jcr, jcr->db, &mr)) { + Jmsg(jcr, M_WARNING, 0, _("Error getting Media record for Volume \"%s\": ERR=%s"), + mr.VolumeName, db_strerror(jcr->db)); } } - break; - case JS_Canceled: - term_msg = _("%s Canceled"); - if (jcr->store_bsock) { - bnet_sig(jcr->store_bsock, BNET_TERMINATE); - if (jcr->SD_msg_chan) { - pthread_cancel(jcr->SD_msg_chan); + + switch (jcr->JobStatus) { + case JS_Terminated: + if (jcr->JobErrors || jcr->SDErrors) { + term_msg = _("%s OK -- with warnings"); + } else { + term_msg = _("%s OK"); + } + break; + case JS_FatalError: + case JS_ErrorTerminated: + term_msg = _("*** %s Error ***"); + msg_type = M_ERROR; /* Generate error message */ + if (jcr->store_bsock) { + bnet_sig(jcr->store_bsock, BNET_TERMINATE); + if (jcr->SD_msg_chan) { + pthread_cancel(jcr->SD_msg_chan); + } } + break; + case JS_Canceled: + term_msg = _("%s Canceled"); + if (jcr->store_bsock) { + bnet_sig(jcr->store_bsock, BNET_TERMINATE); + if (jcr->SD_msg_chan) { + pthread_cancel(jcr->SD_msg_chan); + } + } + break; + default: + term_msg = _("Inappropriate %s term code"); + break; } - break; - default: - term_msg = _("Inappropriate %s term code"); - break; + } else { + if (jcr->getJobType() == JT_MIGRATE && jcr->previous_jr.JobId != 0) { + /* Mark previous job as migrated */ + Mmsg(query, "UPDATE Job SET Type='%c' WHERE JobId=%s", + (char)JT_MIGRATED_JOB, edit_uint64(jcr->previous_jr.JobId, ec1)); + db_sql_query(jcr->db, query.c_str(), NULL, NULL); + } + term_msg = _("%s -- no files to %s"); } - bsnprintf(term_code, sizeof(term_code), term_msg, "Migration"); + + bsnprintf(term_code, sizeof(term_code), term_msg, jcr->get_OperationName(), jcr->get_ActionName(0)); bstrftimes(sdt, sizeof(sdt), jcr->jr.StartTime); bstrftimes(edt, sizeof(edt), jcr->jr.EndTime); RunTime = jcr->jr.EndTime - jcr->jr.StartTime; @@ -981,19 +1299,23 @@ void migration_cleanup(JCR *jcr, int TermCode) kbps = (double)jcr->SDJobBytes / (1000 * RunTime); } - jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg)); - Jmsg(jcr, msg_type, 0, _("Bacula %s (%s): %s\n" + Jmsg(jcr, msg_type, 0, _("%s %s %s (%s):\n" +" Build OS: %s %s %s\n" " Prev Backup JobId: %s\n" +" Prev Backup Job: %s\n" " New Backup JobId: %s\n" -" Migration JobId: %s\n" -" Migration Job: %s\n" +" Current JobId: %s\n" +" Current Job: %s\n" " Backup Level: %s%s\n" " Client: %s\n" " FileSet: \"%s\" %s\n" -" Pool: \"%s\" (From %s)\n" -" Storage: \"%s\" (From %s)\n" +" Read Pool: \"%s\" (From %s)\n" +" Read Storage: \"%s\" (From %s)\n" +" Write Pool: \"%s\" (From %s)\n" +" Write Storage: \"%s\" (From %s)\n" +" Catalog: \"%s\" (From %s)\n" " Start time: %s\n" " End time: %s\n" " Elapsed time: %s\n" @@ -1008,18 +1330,23 @@ void migration_cleanup(JCR *jcr, int TermCode) " SD Errors: %d\n" " SD termination status: %s\n" " Termination: %s\n\n"), - VERSION, - LSMDATE, - edt, - mig_jcr ? edit_uint64(jcr->previous_jr.JobId, ec6) : "0", + BACULA, my_name, VERSION, LSMDATE, + HOST_OS, DISTNAME, DISTVER, + edit_uint64(jcr->previous_jr.JobId, ec6), + jcr->previous_jr.Job, mig_jcr ? edit_uint64(mig_jcr->jr.JobId, ec7) : "0", edit_uint64(jcr->jr.JobId, ec8), jcr->jr.Job, - level_to_str(jcr->JobLevel), jcr->since, + level_to_str(jcr->getJobLevel()), jcr->since, jcr->client->name(), jcr->fileset->name(), jcr->FSCreateTime, + jcr->rpool->name(), jcr->rpool_source, + jcr->rstore?jcr->rstore->name():"*None*", + NPRT(jcr->rstore_source), jcr->pool->name(), jcr->pool_source, - jcr->wstore->name(), jcr->storage_source, + jcr->wstore?jcr->wstore->name():"*None*", + NPRT(jcr->wstore_source), + jcr->catalog->name(), jcr->catalog_source, sdt, edt, edit_utime(RunTime, elapsed, sizeof(elapsed)), @@ -1055,11 +1382,12 @@ void migration_cleanup(JCR *jcr, int TermCode) */ static int get_next_dbid_from_list(char **p, DBId_t *DBId) { - char id[30]; + const int maxlen = 30; + char id[maxlen+1]; char *q = *p; id[0] = 0; - for (int i=0; i<(int)sizeof(id); i++) { + for (int i=0; iNextPool; + + if (!wpool) { + Jmsg(jcr, M_FATAL, 0, _("No Next Pool specification found in Pool \"%s\".\n"), + pool->hdr.name); + return false; + } + + if (!wpool->storage || wpool->storage->size() == 0) { + Jmsg(jcr, M_FATAL, 0, _("No Storage specification found in Next Pool \"%s\".\n"), + wpool->name()); + return false; + } + + /* If pool storage specified, use it instead of job storage for backup */ + copy_wstorage(jcr, wpool->storage, _("Storage from Pool's NextPool resource")); + return true; +}