2 Bacula® - The Network Backup Solution
4 Copyright (C) 2004-2008 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of Kern Sibbald.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
30 * Bacula Director -- migrate.c -- responsible for doing
31 * migration and copy jobs.
33 * Also handles Copy jobs (March MMVIII)
35 * Kern Sibbald, September MMIV
37 * Basic tasks done here:
38 * Open DB and create records for this job.
39 * Open Message Channel with Storage daemon to tell him a job will be starting.
40 * Open connection with Storage daemon and pass him commands
42 * When the Storage daemon finishes the job, update the DB.
51 #include "lib/bregex.h"
56 static const int dbglevel = 10;
58 static char OKbootstrap[] = "3000 OK bootstrap\n";
59 static int get_job_to_migrate(JCR *jcr);
61 static bool regex_find_jobids(JCR *jcr, idpkt *ids, const char *query1,
62 const char *query2, const char *type);
63 static bool find_mediaid_then_jobids(JCR *jcr, idpkt *ids, const char *query1,
65 static bool find_jobids_from_mediaid_list(JCR *jcr, idpkt *ids, const char *type);
66 static void start_migration_job(JCR *jcr);
67 static int get_next_dbid_from_list(char **p, DBId_t *DBId);
70 * Called here before the job is run to do the job
71 * specific setup. Note, one of the important things to
72 * complete in this init code is to make the definitive
73 * choice of input and output storage devices. This is
74 * because immediately after the init, the job is queued
75 * in the jobq.c code, and it checks that all the resources
76 * (storage resources in particular) are available, so these
77 * must all be properly defined.
79 * previous_jr refers to the job DB record of the Job that is
80 * going to be migrated.
81 * prev_job refers to the job resource of the Job that is
82 * going to be migrated.
83 * jcr is the jcr for the current "migration" job. It is a
84 * control job that is put in the DB as a migration job, which
85 * means that this job migrated a previous job to a new job.
86 * No Volume or File data is associated with this control
88 * mig_jcr refers to the newly migrated job that is run by
89 * the current jcr. It is a backup job that moves (migrates) the
90 * data written for the previous_jr into the new pool. This
91 * job (mig_jcr) becomes the new backup job that replaces
92 * the original backup job. Note, this jcr is not really run. It
93 * is simply attached to the current jcr. It will show up in
94 * the Director's status output, but not in the SD or FD, both of
95 * which deal only with the current migration job (i.e. jcr).
97 bool do_migration_init(JCR *jcr)
103 JCR *mig_jcr; /* newly migrated job */
107 apply_pool_overrides(jcr);
109 if (!allow_duplicate_job(jcr)) {
113 jcr->jr.PoolId = get_or_create_pool_record(jcr, jcr->pool->name());
114 if (jcr->jr.PoolId == 0) {
115 Dmsg1(dbglevel, "JobId=%d no PoolId\n", (int)jcr->JobId);
116 Jmsg(jcr, M_FATAL, 0, _("Could not get or create a Pool record.\n"));
120 * Note, at this point, pool is the pool for this job. We
121 * transfer it to rpool (read pool), and a bit later,
122 * pool will be changed to point to the write pool,
123 * which comes from pool->NextPool.
125 jcr->rpool = jcr->pool; /* save read pool */
126 pm_strcpy(jcr->rpool_source, jcr->pool_source);
129 Dmsg2(dbglevel, "Read pool=%s (From %s)\n", jcr->rpool->name(), jcr->rpool_source);
131 /* If we find a job or jobs to migrate it is previous_jr.JobId */
132 count = get_job_to_migrate(jcr);
140 Dmsg1(dbglevel, "Back from get_job_to_migrate JobId=%d\n", (int)jcr->JobId);
142 if (jcr->previous_jr.JobId == 0) {
143 Dmsg1(dbglevel, "JobId=%d no previous JobId\n", (int)jcr->JobId);
144 Jmsg(jcr, M_INFO, 0, _("No previous Job found to migrate.\n"));
145 return true; /* no work */
148 if (!get_or_create_fileset_record(jcr)) {
149 Dmsg1(dbglevel, "JobId=%d no FileSet\n", (int)jcr->JobId);
150 Jmsg(jcr, M_FATAL, 0, _("Could not get or create the FileSet record.\n"));
154 create_restore_bootstrap_file(jcr);
156 if (jcr->previous_jr.JobId == 0 || jcr->ExpectedFiles == 0) {
157 set_jcr_job_status(jcr, JS_Terminated);
158 Dmsg1(dbglevel, "JobId=%d expected files == 0\n", (int)jcr->JobId);
159 if (jcr->previous_jr.JobId == 0) {
160 Jmsg(jcr, M_INFO, 0, _("No previous Job found to migrate.\n"));
162 Jmsg(jcr, M_INFO, 0, _("Previous Job has no data to migrate.\n"));
164 return true; /* no work */
168 Dmsg5(dbglevel, "JobId=%d: Current: Name=%s JobId=%d Type=%c Level=%c\n",
170 jcr->jr.Name, (int)jcr->jr.JobId,
171 jcr->jr.JobType, jcr->jr.JobLevel);
174 job = (JOB *)GetResWithName(R_JOB, jcr->jr.Name);
175 prev_job = (JOB *)GetResWithName(R_JOB, jcr->previous_jr.Name);
178 Jmsg(jcr, M_FATAL, 0, _("Job resource not found for \"%s\".\n"), jcr->jr.Name);
182 Jmsg(jcr, M_FATAL, 0, _("Previous Job resource not found for \"%s\".\n"),
183 jcr->previous_jr.Name);
187 jcr->spool_data = job->spool_data; /* turn on spooling if requested in job */
189 /* Create a migation jcr */
190 mig_jcr = jcr->mig_jcr = new_jcr(sizeof(JCR), dird_free_jcr);
191 memcpy(&mig_jcr->previous_jr, &jcr->previous_jr, sizeof(mig_jcr->previous_jr));
194 * Turn the mig_jcr into a "real" job that takes on the aspects of
195 * the previous backup job "prev_job".
197 set_jcr_defaults(mig_jcr, prev_job);
198 if (!setup_job(mig_jcr)) {
199 Jmsg(jcr, M_FATAL, 0, _("setup job failed.\n"));
203 /* Now reset the job record from the previous job */
204 memcpy(&mig_jcr->jr, &jcr->previous_jr, sizeof(mig_jcr->jr));
205 /* Update the jr to reflect the new values of PoolId and JobId. */
206 mig_jcr->jr.PoolId = jcr->jr.PoolId;
207 mig_jcr->jr.JobId = mig_jcr->JobId;
209 Dmsg4(dbglevel, "mig_jcr: Name=%s JobId=%d Type=%c Level=%c\n",
210 mig_jcr->jr.Name, (int)mig_jcr->jr.JobId,
211 mig_jcr->jr.JobType, mig_jcr->jr.JobLevel);
214 * Get the PoolId used with the original job. Then
215 * find the pool name from the database record.
217 memset(&pr, 0, sizeof(pr));
218 pr.PoolId = mig_jcr->previous_jr.PoolId;
219 if (!db_get_pool_record(jcr, jcr->db, &pr)) {
220 Jmsg(jcr, M_FATAL, 0, _("Pool for JobId %s not in database. ERR=%s\n"),
221 edit_int64(pr.PoolId, ed1), db_strerror(jcr->db));
224 /* Get the pool resource corresponding to the original job */
225 pool = (POOL *)GetResWithName(R_POOL, pr.Name);
227 Jmsg(jcr, M_FATAL, 0, _("Pool resource \"%s\" not found.\n"), pr.Name);
231 /* If pool storage specified, use it for restore */
232 copy_rstorage(mig_jcr, pool->storage, _("Pool resource"));
233 copy_rstorage(jcr, pool->storage, _("Pool resource"));
236 * If the original backup pool has a NextPool, make sure a
237 * record exists in the database. Note, in this case, we
238 * will be migrating from pool to pool->NextPool.
240 if (pool->NextPool) {
241 jcr->jr.PoolId = get_or_create_pool_record(jcr, pool->NextPool->name());
242 if (jcr->jr.PoolId == 0) {
246 if (!set_migration_wstorage(jcr, pool)) {
249 mig_jcr->pool = jcr->pool = pool->NextPool;
250 pm_strcpy(jcr->pool_source, _("Job Pool's NextPool resource"));
251 mig_jcr->jr.PoolId = jcr->jr.PoolId;
253 Dmsg2(dbglevel, "Write pool=%s read rpool=%s\n", jcr->pool->name(), jcr->rpool->name());
258 * Do a Migration of a previous job
260 * Returns: false on failure
263 bool do_migration(JCR *jcr)
267 JCR *mig_jcr = jcr->mig_jcr; /* newly migrated job */
270 * If mig_jcr is NULL, there is nothing to do for this job,
271 * so set a normal status, cleanup and return OK.
274 set_jcr_job_status(jcr, JS_Terminated);
275 migration_cleanup(jcr, jcr->JobStatus);
279 if (!db_get_job_record(jcr, jcr->db, &jcr->previous_jr)) {
280 Jmsg(jcr, M_FATAL, 0, _("Could not get job record for JobId %s to migrate. ERR=%s"),
281 edit_int64(jcr->previous_jr.JobId, ed1),
282 db_strerror(jcr->db));
283 set_jcr_job_status(jcr, JS_Terminated);
284 migration_cleanup(jcr, jcr->JobStatus);
287 /* Make sure this job was not already migrated */
288 if (jcr->previous_jr.JobType != JT_BACKUP) {
289 Jmsg(jcr, M_INFO, 0, _("JobId %s already migrated probably by another Job. Migration stopped.\n"),
290 edit_int64(jcr->previous_jr.JobId, ed1));
291 set_jcr_job_status(jcr, JS_Terminated);
292 migration_cleanup(jcr, jcr->JobStatus);
296 /* Print Job Start message */
297 Jmsg(jcr, M_INFO, 0, _("Start %s JobId %s, Job=%s\n"),
298 jcr->get_JobType() == JT_MIGRATE ? "Migration" : "Copy",
299 edit_uint64(jcr->JobId, ed1), jcr->Job);
302 * Open a message channel connection with the Storage
303 * daemon. This is to let him know that our client
304 * will be contacting him for a backup session.
307 Dmsg0(110, "Open connection with storage daemon\n");
308 set_jcr_job_status(jcr, JS_WaitSD);
309 set_jcr_job_status(mig_jcr, JS_WaitSD);
311 * Start conversation with Storage daemon
313 if (!connect_to_storage_daemon(jcr, 10, SDConnectTimeout, 1)) {
316 sd = jcr->store_bsock;
318 * Now start a job with the Storage daemon
320 Dmsg2(dbglevel, "Read store=%s, write store=%s\n",
321 ((STORE *)jcr->rstorage->first())->name(),
322 ((STORE *)jcr->wstorage->first())->name());
323 if (((STORE *)jcr->rstorage->first())->name() == ((STORE *)jcr->wstorage->first())->name()) {
324 Jmsg(jcr, M_FATAL, 0, _("Read storage \"%s\" same as write storage.\n"),
325 ((STORE *)jcr->rstorage->first())->name());
328 if (!start_storage_daemon_job(jcr, jcr->rstorage, jcr->wstorage)) {
331 Dmsg0(150, "Storage daemon connection OK\n");
333 if (!send_bootstrap_file(jcr, sd) ||
334 !response(jcr, sd, OKbootstrap, "Bootstrap", DISPLAY_ERROR)) {
339 * We re-update the job start record so that the start
340 * time is set after the run before job. This avoids
341 * that any files created by the run before job will
342 * be saved twice. They will be backed up in the current
343 * job, but not in the next one unless they are changed.
344 * Without this, they will be backed up in this job and
345 * in the next job run because in that case, their date
346 * is after the start of this run.
348 jcr->start_time = time(NULL);
349 jcr->jr.StartTime = jcr->start_time;
350 jcr->jr.JobTDate = jcr->start_time;
351 set_jcr_job_status(jcr, JS_Running);
353 /* Update job start record for this migration control job */
354 if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) {
355 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db));
360 mig_jcr->start_time = time(NULL);
361 mig_jcr->jr.StartTime = mig_jcr->start_time;
362 mig_jcr->jr.JobTDate = mig_jcr->start_time;
363 set_jcr_job_status(mig_jcr, JS_Running);
365 /* Update job start record for the real migration backup job */
366 if (!db_update_job_start_record(mig_jcr, mig_jcr->db, &mig_jcr->jr)) {
367 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(mig_jcr->db));
371 Dmsg4(dbglevel, "mig_jcr: Name=%s JobId=%d Type=%c Level=%c\n",
372 mig_jcr->jr.Name, (int)mig_jcr->jr.JobId,
373 mig_jcr->jr.JobType, mig_jcr->jr.JobLevel);
377 * Start the job prior to starting the message thread below
378 * to avoid two threads from using the BSOCK structure at
381 if (!sd->fsend("run")) {
386 * Now start a Storage daemon message thread
388 if (!start_storage_daemon_message_thread(jcr)) {
393 set_jcr_job_status(jcr, JS_Running);
394 set_jcr_job_status(mig_jcr, JS_Running);
396 /* Pickup Job termination data */
397 /* Note, the SD stores in jcr->JobFiles/ReadBytes/JobBytes/Errors */
398 wait_for_storage_daemon_termination(jcr);
399 set_jcr_job_status(jcr, jcr->SDJobStatus);
400 db_write_batch_file_records(jcr); /* used by bulk batch file insert */
401 if (jcr->JobStatus != JS_Terminated) {
405 migration_cleanup(jcr, jcr->JobStatus);
406 if (jcr->get_JobType() == JT_MIGRATE && mig_jcr) {
408 UAContext *ua = new_ua_context(jcr);
409 edit_uint64(jcr->previous_jr.JobId, jobid);
410 /* Purge all old file records, but leave Job record */
411 purge_files_from_jobs(ua, jobid);
422 /* Add an item to the list if it is unique */
423 static void add_unique_id(idpkt *ids, char *item)
425 const int maxlen = 30;
429 /* Walk through current list to see if each item is the same as item */
432 for (int i=0; i<maxlen; i++) {
435 } else if (*q == ',') {
442 if (strcmp(item, id) == 0) {
446 /* Did not find item, so add it to list */
447 if (ids->count == 0) {
450 pm_strcat(ids->list, ",");
452 pm_strcat(ids->list, item);
454 // Dmsg3(0, "add_uniq count=%d Ids=%p %s\n", ids->count, ids->list, ids->list);
459 * Callback handler make list of DB Ids
461 static int unique_dbid_handler(void *ctx, int num_fields, char **row)
463 idpkt *ids = (idpkt *)ctx;
465 add_unique_id(ids, row[0]);
466 Dmsg3(dbglevel, "dbid_hdlr count=%d Ids=%p %s\n", ids->count, ids->list, ids->list);
476 static int item_compare(void *item1, void *item2)
478 uitem *i1 = (uitem *)item1;
479 uitem *i2 = (uitem *)item2;
480 return strcmp(i1->item, i2->item);
483 static int unique_name_handler(void *ctx, int num_fields, char **row)
485 dlist *list = (dlist *)ctx;
487 uitem *new_item = (uitem *)malloc(sizeof(uitem));
490 memset(new_item, 0, sizeof(uitem));
491 new_item->item = bstrdup(row[0]);
492 Dmsg1(dbglevel, "Unique_name_hdlr Item=%s\n", row[0]);
493 item = (uitem *)list->binary_insert((void *)new_item, item_compare);
494 if (item != new_item) { /* already in list */
495 free(new_item->item);
496 free((char *)new_item);
502 /* Get Job names in Pool */
503 const char *sql_job =
504 "SELECT DISTINCT Job.Name from Job,Pool"
505 " WHERE Pool.Name='%s' AND Job.PoolId=Pool.PoolId";
507 /* Get JobIds from regex'ed Job names */
508 const char *sql_jobids_from_job =
509 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Job,Pool"
510 " WHERE Job.Name='%s' AND Pool.Name='%s' AND Job.PoolId=Pool.PoolId"
511 " ORDER by Job.StartTime";
513 /* Get Client names in Pool */
514 const char *sql_client =
515 "SELECT DISTINCT Client.Name from Client,Pool,Job"
516 " WHERE Pool.Name='%s' AND Job.ClientId=Client.ClientId AND"
517 " Job.PoolId=Pool.PoolId";
519 /* Get JobIds from regex'ed Client names */
520 const char *sql_jobids_from_client =
521 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Job,Pool,Client"
522 " WHERE Client.Name='%s' AND Pool.Name='%s' AND Job.PoolId=Pool.PoolId"
523 " AND Job.ClientId=Client.ClientId AND Job.Type='B'"
524 " ORDER by Job.StartTime";
526 /* Get Volume names in Pool */
527 const char *sql_vol =
528 "SELECT DISTINCT VolumeName FROM Media,Pool WHERE"
529 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
530 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'";
532 /* Get JobIds from regex'ed Volume names */
533 const char *sql_jobids_from_vol =
534 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Media,JobMedia,Job"
535 " WHERE Media.VolumeName='%s' AND Media.MediaId=JobMedia.MediaId"
536 " AND JobMedia.JobId=Job.JobId AND Job.Type='B'"
537 " ORDER by Job.StartTime";
540 const char *sql_smallest_vol =
541 "SELECT Media.MediaId FROM Media,Pool,JobMedia WHERE"
542 " Media.MediaId in (SELECT DISTINCT MediaId from JobMedia) AND"
543 " Media.VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
544 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'"
545 " ORDER BY VolBytes ASC LIMIT 1";
547 const char *sql_oldest_vol =
548 "SELECT Media.MediaId FROM Media,Pool,JobMedia WHERE"
549 " Media.MediaId in (SELECT DISTINCT MediaId from JobMedia) AND"
550 " Media.VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
551 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'"
552 " ORDER BY LastWritten ASC LIMIT 1";
554 /* Get JobIds when we have selected MediaId */
555 const char *sql_jobids_from_mediaid =
556 "SELECT DISTINCT Job.JobId,Job.StartTime FROM JobMedia,Job"
557 " WHERE JobMedia.JobId=Job.JobId AND JobMedia.MediaId IN (%s)"
559 " ORDER by Job.StartTime";
561 /* Get the number of bytes in the pool */
562 const char *sql_pool_bytes =
563 "SELECT SUM(JobBytes) FROM Job WHERE JobId IN"
564 " (SELECT DISTINCT Job.JobId from Pool,Job,Media,JobMedia WHERE"
565 " Pool.Name='%s' AND Media.PoolId=Pool.PoolId AND"
566 " VolStatus in ('Full','Used','Error','Append') AND Media.Enabled=1 AND"
568 " JobMedia.JobId=Job.JobId AND Job.PoolId=Media.PoolId)";
572 "SELECT SUM(VolBytes) FROM Media,Pool WHERE"
573 " VolStatus in ('Full','Used','Error','Append') AND Media.Enabled=1 AND"
574 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'";
577 /* Get the number of bytes in the Jobs */
578 const char *sql_job_bytes =
579 "SELECT SUM(JobBytes) FROM Job WHERE JobId IN (%s)";
582 /* Get Media Ids in Pool */
583 const char *sql_mediaids =
584 "SELECT MediaId FROM Media,Pool WHERE"
585 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
586 " Media.PoolId=Pool.PoolId AND Pool.Name='%s' ORDER BY LastWritten ASC";
588 /* Get JobIds in Pool longer than specified time */
589 const char *sql_pool_time =
590 "SELECT DISTINCT Job.JobId from Pool,Job,Media,JobMedia WHERE"
591 " Pool.Name='%s' AND Media.PoolId=Pool.PoolId AND"
592 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
594 " JobMedia.JobId=Job.JobId AND Job.PoolId=Media.PoolId"
595 " AND Job.RealEndTime<='%s'";
598 * const char *sql_ujobid =
599 * "SELECT DISTINCT Job.Job from Client,Pool,Media,Job,JobMedia "
600 * " WHERE Media.PoolId=Pool.PoolId AND Pool.Name='%s' AND"
601 * " JobMedia.JobId=Job.JobId AND Job.PoolId=Media.PoolId";
608 * This is the central piece of code that finds a job or jobs
609 * actually JobIds to migrate. It first looks to see if one
610 * has been "manually" specified in jcr->MigrateJobId, and if
611 * so, it returns that JobId to be run. Otherwise, it
612 * examines the Selection Type to see what kind of migration
613 * we are doing (Volume, Job, Client, ...) and applies any
614 * Selection Pattern if appropriate to obtain a list of JobIds.
615 * Finally, it will loop over all the JobIds found, except the last
616 * one starting a new job with MigrationJobId set to that JobId, and
617 * finally, it returns the last JobId to the caller.
619 * Returns: -1 on error
620 * 0 if no jobs to migrate
621 * 1 if OK and jcr->previous_jr filled in
623 static int get_job_to_migrate(JCR *jcr)
625 char ed1[30], ed2[30];
626 POOL_MEM query(PM_MESSAGE);
631 idpkt ids, mid, jids;
636 char dt[MAX_TIME_LENGTH];
639 ids.list = get_pool_memory(PM_MESSAGE);
642 mid.list = get_pool_memory(PM_MESSAGE);
645 jids.list = get_pool_memory(PM_MESSAGE);
651 * If MigrateJobId is set, then we migrate only that Job,
652 * otherwise, we go through the full selection of jobs to
655 if (jcr->MigrateJobId != 0) {
656 Dmsg1(dbglevel, "At Job start previous jobid=%u\n", jcr->MigrateJobId);
657 edit_uint64(jcr->MigrateJobId, ids.list);
660 switch (jcr->job->selection_type) {
662 if (!regex_find_jobids(jcr, &ids, sql_job, sql_jobids_from_job, "Job")) {
667 if (!regex_find_jobids(jcr, &ids, sql_client, sql_jobids_from_client, "Client")) {
672 if (!regex_find_jobids(jcr, &ids, sql_vol, sql_jobids_from_vol, "Volume")) {
677 if (!jcr->job->selection_pattern) {
678 Jmsg(jcr, M_FATAL, 0, _("No Migration SQL selection pattern specified.\n"));
681 Dmsg1(dbglevel, "SQL=%s\n", jcr->job->selection_pattern);
682 if (!db_sql_query(jcr->db, jcr->job->selection_pattern,
683 unique_dbid_handler, (void *)&ids)) {
684 Jmsg(jcr, M_FATAL, 0,
685 _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
689 case MT_SMALLEST_VOL:
690 if (!find_mediaid_then_jobids(jcr, &ids, sql_smallest_vol, "Smallest Volume")) {
695 if (!find_mediaid_then_jobids(jcr, &ids, sql_oldest_vol, "Oldest Volume")) {
700 case MT_POOL_OCCUPANCY:
702 /* Find count of bytes in pool */
703 Mmsg(query, sql_pool_bytes, jcr->rpool->name());
704 if (!db_sql_query(jcr->db, query.c_str(), db_int64_handler, (void *)&ctx)) {
705 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
708 if (ctx.count == 0) {
709 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
712 pool_bytes = ctx.value;
713 Dmsg2(dbglevel, "highbytes=%lld pool=%lld\n", jcr->rpool->MigrationHighBytes,
715 if (pool_bytes < (int64_t)jcr->rpool->MigrationHighBytes) {
716 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
719 Dmsg0(dbglevel, "We should do Occupation migration.\n");
722 /* Find a list of MediaIds that could be migrated */
723 Mmsg(query, sql_mediaids, jcr->rpool->name());
724 Dmsg1(dbglevel, "query=%s\n", query.c_str());
725 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)&ids)) {
726 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
729 if (ids.count == 0) {
730 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
733 Dmsg2(dbglevel, "Pool Occupancy ids=%d MediaIds=%s\n", ids.count, ids.list);
735 if (!find_jobids_from_mediaid_list(jcr, &ids, "Volumes")) {
738 /* ids == list of jobs */
740 for (int i=0; i < (int)ids.count; i++) {
741 stat = get_next_dbid_from_list(&p, &DBId);
742 Dmsg2(dbglevel, "get_next_dbid stat=%d JobId=%u\n", stat, (uint32_t)DBId);
744 Jmsg(jcr, M_FATAL, 0, _("Invalid JobId found.\n"));
746 } else if (stat == 0) {
751 Mmsg(mid.list, "%s", edit_int64(DBId, ed1));
752 if (jids.count > 0) {
753 pm_strcat(jids.list, ",");
755 pm_strcat(jids.list, mid.list);
756 jids.count += mid.count;
758 /* Find count of bytes from Jobs */
759 Mmsg(query, sql_job_bytes, mid.list);
760 Dmsg1(dbglevel, "Jobbytes query: %s\n", query.c_str());
761 if (!db_sql_query(jcr->db, query.c_str(), db_int64_handler, (void *)&ctx)) {
762 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
765 pool_bytes -= ctx.value;
766 Dmsg1(dbglevel, "Total migrate Job bytes=%s\n", edit_int64_with_commas(ctx.value, ed1));
767 Dmsg2(dbglevel, "lowbytes=%s poolafter=%s\n",
768 edit_int64_with_commas(jcr->rpool->MigrationLowBytes, ed1),
769 edit_int64_with_commas(pool_bytes, ed2));
770 if (pool_bytes <= (int64_t)jcr->rpool->MigrationLowBytes) {
771 Dmsg0(dbglevel, "We should be done.\n");
775 /* Transfer jids to ids, where the jobs list is expected */
776 ids.count = jids.count;
777 pm_strcpy(ids.list, jids.list);
778 Dmsg2(dbglevel, "Pool Occupancy ids=%d JobIds=%s\n", ids.count, ids.list);
782 ttime = time(NULL) - (time_t)jcr->rpool->MigrationTime;
783 (void)localtime_r(&ttime, &tm);
784 strftime(dt, sizeof(dt), "%Y-%m-%d %H:%M:%S", &tm);
787 Mmsg(query, sql_pool_time, jcr->rpool->name(), dt);
788 Dmsg1(dbglevel, "query=%s\n", query.c_str());
789 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)&ids)) {
790 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
793 if (ids.count == 0) {
794 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
797 Dmsg2(dbglevel, "PoolTime ids=%d JobIds=%s\n", ids.count, ids.list);
801 Jmsg(jcr, M_FATAL, 0, _("Unknown Migration Selection Type.\n"));
807 * Loop over all jobids except the last one, sending
808 * them to start_migration_job(), which will start a job
809 * for each of them. For the last JobId, we handle it below.
812 if (ids.count == 0) {
813 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
817 Jmsg(jcr, M_INFO, 0, _("The following %u JobId%s were chosen to be migrated: %s\n"),
818 ids.count, ids.count==0?"":"s", ids.list);
820 Dmsg2(dbglevel, "Before loop count=%d ids=%s\n", ids.count, ids.list);
821 for (int i=1; i < (int)ids.count; i++) {
823 stat = get_next_jobid_from_list(&p, &JobId);
824 Dmsg3(dbglevel, "get_jobid_no=%d stat=%d JobId=%u\n", i, stat, JobId);
825 jcr->MigrateJobId = JobId;
826 start_migration_job(jcr);
827 Dmsg0(dbglevel, "Back from start_migration_job\n");
829 Jmsg(jcr, M_FATAL, 0, _("Invalid JobId found.\n"));
831 } else if (stat == 0) {
832 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
837 /* Now get the last JobId and handle it in the current job */
839 stat = get_next_jobid_from_list(&p, &JobId);
840 Dmsg2(dbglevel, "Last get_next_jobid stat=%d JobId=%u\n", stat, (int)JobId);
842 Jmsg(jcr, M_FATAL, 0, _("Invalid JobId found.\n"));
844 } else if (stat == 0) {
845 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
849 jcr->previous_jr.JobId = JobId;
850 Dmsg1(dbglevel, "Previous jobid=%d\n", (int)jcr->previous_jr.JobId);
852 if (!db_get_job_record(jcr, jcr->db, &jcr->previous_jr)) {
853 Jmsg(jcr, M_FATAL, 0, _("Could not get job record for JobId %s to migrate. ERR=%s"),
854 edit_int64(jcr->previous_jr.JobId, ed1),
855 db_strerror(jcr->db));
858 Jmsg(jcr, M_INFO, 0, _("Migration using JobId=%s Job=%s\n"),
859 edit_int64(jcr->previous_jr.JobId, ed1), jcr->previous_jr.Job);
860 Dmsg3(dbglevel, "Migration JobId=%d using JobId=%s Job=%s\n",
862 edit_int64(jcr->previous_jr.JobId, ed1), jcr->previous_jr.Job);
872 free_pool_memory(ids.list);
873 free_pool_memory(mid.list);
874 free_pool_memory(jids.list);
878 static void start_migration_job(JCR *jcr)
880 UAContext *ua = new_ua_context(jcr);
883 Mmsg(ua->cmd, "run %s jobid=%s", jcr->job->hdr.name,
884 edit_uint64(jcr->MigrateJobId, ed1));
885 Dmsg1(dbglevel, "=============== Migration cmd=%s\n", ua->cmd);
886 parse_ua_args(ua); /* parse command */
887 JobId_t jobid = run_cmd(ua, ua->cmd);
889 Jmsg(jcr, M_ERROR, 0, _("Could not start migration job.\n"));
891 Jmsg(jcr, M_INFO, 0, _("Migration JobId %d started.\n"), (int)jobid);
896 static bool find_mediaid_then_jobids(JCR *jcr, idpkt *ids, const char *query1,
900 POOL_MEM query(PM_MESSAGE);
903 /* Basic query for MediaId */
904 Mmsg(query, query1, jcr->rpool->name());
905 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
906 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
909 if (ids->count == 0) {
910 Jmsg(jcr, M_INFO, 0, _("No %s found to migrate.\n"), type);
911 ok = true; /* Not an error */
913 } else if (ids->count != 1) {
914 Jmsg(jcr, M_FATAL, 0, _("SQL error. Expected 1 MediaId got %d\n"), ids->count);
917 Dmsg2(dbglevel, "%s MediaIds=%s\n", type, ids->list);
919 ok = find_jobids_from_mediaid_list(jcr, ids, type);
926 * This routine returns:
927 * false if an error occurred
929 * ids.count number of jobids found (may be zero)
931 static bool find_jobids_from_mediaid_list(JCR *jcr, idpkt *ids, const char *type)
934 POOL_MEM query(PM_MESSAGE);
936 Mmsg(query, sql_jobids_from_mediaid, ids->list);
938 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
939 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
942 if (ids->count == 0) {
943 Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type);
951 static bool regex_find_jobids(JCR *jcr, idpkt *ids, const char *query1,
952 const char *query2, const char *type)
956 uitem *last_item = NULL;
961 POOL_MEM query(PM_MESSAGE);
963 item_chain = New(dlist(item, &item->link));
964 if (!jcr->job->selection_pattern) {
965 Jmsg(jcr, M_FATAL, 0, _("No Migration %s selection pattern specified.\n"),
969 Dmsg1(dbglevel, "regex-sel-pattern=%s\n", jcr->job->selection_pattern);
970 /* Basic query for names */
971 Mmsg(query, query1, jcr->rpool->name());
972 Dmsg1(dbglevel, "get name query1=%s\n", query.c_str());
973 if (!db_sql_query(jcr->db, query.c_str(), unique_name_handler,
974 (void *)item_chain)) {
975 Jmsg(jcr, M_FATAL, 0,
976 _("SQL to get %s failed. ERR=%s\n"), type, db_strerror(jcr->db));
979 Dmsg1(dbglevel, "query1 returned %d names\n", item_chain->size());
980 if (item_chain->size() == 0) {
981 Jmsg(jcr, M_INFO, 0, _("Query of Pool \"%s\" returned no Jobs to migrate.\n"),
984 goto bail_out; /* skip regex match */
986 /* Compile regex expression */
987 rc = regcomp(&preg, jcr->job->selection_pattern, REG_EXTENDED);
989 regerror(rc, &preg, prbuf, sizeof(prbuf));
990 Jmsg(jcr, M_FATAL, 0, _("Could not compile regex pattern \"%s\" ERR=%s\n"),
991 jcr->job->selection_pattern, prbuf);
994 /* Now apply the regex to the names and remove any item not matched */
995 foreach_dlist(item, item_chain) {
996 const int nmatch = 30;
997 regmatch_t pmatch[nmatch];
999 Dmsg1(dbglevel, "Remove item %s\n", last_item->item);
1000 free(last_item->item);
1001 item_chain->remove(last_item);
1003 Dmsg1(dbglevel, "get name Item=%s\n", item->item);
1004 rc = regexec(&preg, item->item, nmatch, pmatch, 0);
1006 last_item = NULL; /* keep this one */
1012 free(last_item->item);
1013 Dmsg1(dbglevel, "Remove item %s\n", last_item->item);
1014 item_chain->remove(last_item);
1018 if (item_chain->size() == 0) {
1019 Jmsg(jcr, M_INFO, 0, _("Regex pattern matched no Jobs to migrate.\n"));
1021 goto bail_out; /* skip regex match */
1025 * At this point, we have a list of items in item_chain
1026 * that have been matched by the regex, so now we need
1027 * to look up their jobids.
1030 foreach_dlist(item, item_chain) {
1031 Dmsg2(dbglevel, "Got %s: %s\n", type, item->item);
1032 Mmsg(query, query2, item->item, jcr->rpool->name());
1033 Dmsg1(dbglevel, "get id from name query2=%s\n", query.c_str());
1034 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
1035 Jmsg(jcr, M_FATAL, 0,
1036 _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
1040 if (ids->count == 0) {
1041 Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type);
1046 Dmsg2(dbglevel, "Count=%d Jobids=%s\n", ids->count, ids->list);
1047 foreach_dlist(item, item_chain) {
1056 * Release resources allocated during backup.
1058 void migration_cleanup(JCR *jcr, int TermCode)
1060 char sdt[MAX_TIME_LENGTH], edt[MAX_TIME_LENGTH];
1061 char ec1[30], ec2[30], ec3[30], ec4[30], ec5[30], elapsed[50];
1062 char ec6[50], ec7[50], ec8[50];
1063 char term_code[100], sd_term_msg[100];
1064 const char *term_msg;
1065 int msg_type = M_INFO;
1069 JCR *mig_jcr = jcr->mig_jcr;
1070 POOL_MEM query(PM_MESSAGE);
1072 Dmsg2(100, "Enter migrate_cleanup %d %c\n", TermCode, TermCode);
1073 update_job_end(jcr, TermCode);
1074 memset(&mr, 0, sizeof(mr));
1077 * Check if we actually did something.
1078 * mig_jcr is jcr of the newly migrated job.
1081 mig_jcr->JobFiles = jcr->JobFiles = jcr->SDJobFiles;
1082 mig_jcr->JobBytes = jcr->JobBytes = jcr->SDJobBytes;
1083 mig_jcr->VolSessionId = jcr->VolSessionId;
1084 mig_jcr->VolSessionTime = jcr->VolSessionTime;
1085 mig_jcr->jr.RealEndTime = 0;
1086 mig_jcr->jr.PriorJobId = jcr->previous_jr.JobId;
1088 update_job_end(mig_jcr, TermCode);
1090 /* Update final items to set them to the previous job's values */
1091 Mmsg(query, "UPDATE Job SET StartTime='%s',EndTime='%s',"
1092 "JobTDate=%s WHERE JobId=%s",
1093 jcr->previous_jr.cStartTime, jcr->previous_jr.cEndTime,
1094 edit_uint64(jcr->previous_jr.JobTDate, ec1),
1095 edit_uint64(mig_jcr->jr.JobId, ec2));
1096 db_sql_query(mig_jcr->db, query.c_str(), NULL, NULL);
1098 /* Now mark the previous job as migrated if it terminated normally */
1099 if (jcr->get_JobType() == JT_MIGRATE && jcr->JobStatus == JS_Terminated) {
1100 Mmsg(query, "UPDATE Job SET Type='%c' WHERE JobId=%s",
1101 (char)JT_MIGRATED_JOB, edit_uint64(jcr->previous_jr.JobId, ec1));
1102 db_sql_query(mig_jcr->db, query.c_str(), NULL, NULL);
1105 if (!db_get_job_record(jcr, jcr->db, &jcr->jr)) {
1106 Jmsg(jcr, M_WARNING, 0, _("Error getting Job record for Job report: ERR=%s"),
1107 db_strerror(jcr->db));
1108 set_jcr_job_status(jcr, JS_ErrorTerminated);
1111 update_bootstrap_file(mig_jcr);
1113 if (!db_get_job_volume_names(mig_jcr, mig_jcr->db, mig_jcr->jr.JobId, &mig_jcr->VolumeName)) {
1115 * Note, if the job has failed, most likely it did not write any
1116 * tape, so suppress this "error" message since in that case
1117 * it is normal. Or look at it the other way, only for a
1118 * normal exit should we complain about this error.
1120 if (jcr->JobStatus == JS_Terminated && jcr->jr.JobBytes) {
1121 Jmsg(jcr, M_ERROR, 0, "%s", db_strerror(mig_jcr->db));
1123 mig_jcr->VolumeName[0] = 0; /* none */
1126 if (mig_jcr->VolumeName[0]) {
1127 /* Find last volume name. Multiple vols are separated by | */
1128 char *p = strrchr(mig_jcr->VolumeName, '|');
1132 p = mig_jcr->VolumeName; /* no |, take full name */
1134 bstrncpy(mr.VolumeName, p, sizeof(mr.VolumeName));
1135 if (!db_get_media_record(jcr, jcr->db, &mr)) {
1136 Jmsg(jcr, M_WARNING, 0, _("Error getting Media record for Volume \"%s\": ERR=%s"),
1137 mr.VolumeName, db_strerror(jcr->db));
1141 switch (jcr->JobStatus) {
1143 if (jcr->Errors || jcr->SDErrors) {
1144 term_msg = _("%s OK -- with warnings");
1146 term_msg = _("%s OK");
1150 case JS_ErrorTerminated:
1151 term_msg = _("*** %s Error ***");
1152 msg_type = M_ERROR; /* Generate error message */
1153 if (jcr->store_bsock) {
1154 bnet_sig(jcr->store_bsock, BNET_TERMINATE);
1155 if (jcr->SD_msg_chan) {
1156 pthread_cancel(jcr->SD_msg_chan);
1161 term_msg = _("%s Canceled");
1162 if (jcr->store_bsock) {
1163 bnet_sig(jcr->store_bsock, BNET_TERMINATE);
1164 if (jcr->SD_msg_chan) {
1165 pthread_cancel(jcr->SD_msg_chan);
1170 term_msg = _("Inappropriate %s term code");
1174 if (jcr->get_JobType() == JT_MIGRATE && jcr->previous_jr.JobId != 0) {
1175 /* Mark previous job as migrated */
1176 Mmsg(query, "UPDATE Job SET Type='%c' WHERE JobId=%s",
1177 (char)JT_MIGRATED_JOB, edit_uint64(jcr->previous_jr.JobId, ec1));
1178 db_sql_query(jcr->db, query.c_str(), NULL, NULL);
1180 term_msg = _("%s -- no files to migrate");
1183 bsnprintf(term_code, sizeof(term_code), term_msg, "Migration");
1184 bstrftimes(sdt, sizeof(sdt), jcr->jr.StartTime);
1185 bstrftimes(edt, sizeof(edt), jcr->jr.EndTime);
1186 RunTime = jcr->jr.EndTime - jcr->jr.StartTime;
1190 kbps = (double)jcr->SDJobBytes / (1000 * RunTime);
1194 jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg));
1196 Jmsg(jcr, msg_type, 0, _("Bacula %s %s (%s): %s\n"
1197 " Build OS: %s %s %s\n"
1198 " Prev Backup JobId: %s\n"
1199 " New Backup JobId: %s\n"
1200 " Current JobId: %s\n"
1201 " Current Job: %s\n"
1202 " Backup Level: %s%s\n"
1204 " FileSet: \"%s\" %s\n"
1205 " Read Pool: \"%s\" (From %s)\n"
1206 " Read Storage: \"%s\" (From %s)\n"
1207 " Write Pool: \"%s\" (From %s)\n"
1208 " Write Storage: \"%s\" (From %s)\n"
1209 " Catalog: \"%s\" (From %s)\n"
1212 " Elapsed time: %s\n"
1214 " SD Files Written: %s\n"
1215 " SD Bytes Written: %s (%sB)\n"
1216 " Rate: %.1f KB/s\n"
1217 " Volume name(s): %s\n"
1218 " Volume Session Id: %d\n"
1219 " Volume Session Time: %d\n"
1220 " Last Volume Bytes: %s (%sB)\n"
1222 " SD termination status: %s\n"
1223 " Termination: %s\n\n"),
1224 my_name, VERSION, LSMDATE, edt,
1225 HOST_OS, DISTNAME, DISTVER,
1226 edit_uint64(jcr->previous_jr.JobId, ec6),
1227 mig_jcr ? edit_uint64(mig_jcr->jr.JobId, ec7) : "0",
1228 edit_uint64(jcr->jr.JobId, ec8),
1230 level_to_str(jcr->get_JobLevel()), jcr->since,
1231 jcr->client->name(),
1232 jcr->fileset->name(), jcr->FSCreateTime,
1233 jcr->rpool->name(), jcr->rpool_source,
1234 jcr->rstore?jcr->rstore->name():"*None*",
1235 NPRT(jcr->rstore_source),
1236 jcr->pool->name(), jcr->pool_source,
1237 jcr->wstore?jcr->wstore->name():"*None*",
1238 NPRT(jcr->wstore_source),
1239 jcr->catalog->name(), jcr->catalog_source,
1242 edit_utime(RunTime, elapsed, sizeof(elapsed)),
1244 edit_uint64_with_commas(jcr->SDJobFiles, ec1),
1245 edit_uint64_with_commas(jcr->SDJobBytes, ec2),
1246 edit_uint64_with_suffix(jcr->SDJobBytes, ec3),
1248 mig_jcr ? mig_jcr->VolumeName : "",
1250 jcr->VolSessionTime,
1251 edit_uint64_with_commas(mr.VolBytes, ec4),
1252 edit_uint64_with_suffix(mr.VolBytes, ec5),
1257 Dmsg1(100, "migrate_cleanup() mig_jcr=0x%x\n", jcr->mig_jcr);
1259 free_jcr(jcr->mig_jcr);
1260 jcr->mig_jcr = NULL;
1262 Dmsg0(100, "Leave migrate_cleanup()\n");
1266 * Return next DBId from comma separated list
1269 * 1 if next DBId returned
1270 * 0 if no more DBIds are in list
1271 * -1 there is an error
1273 static int get_next_dbid_from_list(char **p, DBId_t *DBId)
1275 const int maxlen = 30;
1280 for (int i=0; i<maxlen; i++) {
1283 } else if (*q == ',') {
1292 } else if (!is_a_number(id)) {
1293 return -1; /* error */
1296 *DBId = str_to_int64(id);
1300 bool set_migration_wstorage(JCR *jcr, POOL *pool)
1302 POOL *wpool = pool->NextPool;
1305 Jmsg(jcr, M_FATAL, 0, _("No Next Pool specification found in Pool \"%s\".\n"),
1310 if (!wpool->storage || wpool->storage->size() == 0) {
1311 Jmsg(jcr, M_FATAL, 0, _("No Storage specification found in Next Pool \"%s\".\n"),
1316 /* If pool storage specified, use it instead of job storage for backup */
1317 copy_wstorage(jcr, wpool->storage, _("Storage from Pool's NextPool resource"));