2 Bacula® - The Network Backup Solution
4 Copyright (C) 2004-2008 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of John Walker.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
30 * Bacula Director -- migrate.c -- responsible for doing
31 * migration and copy jobs.
33 * Also handles Copy jobs (March MMVIII)
35 * Kern Sibbald, September MMIV
37 * Basic tasks done here:
38 * Open DB and create records for this job.
39 * Open Message Channel with Storage daemon to tell him a job will be starting.
40 * Open connection with Storage daemon and pass him commands
42 * When the Storage daemon finishes the job, update the DB.
51 #include "lib/bregex.h"
56 static const int dbglevel = 10;
58 static char OKbootstrap[] = "3000 OK bootstrap\n";
59 static int get_job_to_migrate(JCR *jcr);
61 static bool regex_find_jobids(JCR *jcr, idpkt *ids, const char *query1,
62 const char *query2, const char *type);
63 static bool find_mediaid_then_jobids(JCR *jcr, idpkt *ids, const char *query1,
65 static bool find_jobids_from_mediaid_list(JCR *jcr, idpkt *ids, const char *type);
66 static void start_migration_job(JCR *jcr);
67 static int get_next_dbid_from_list(char **p, DBId_t *DBId);
70 * Called here before the job is run to do the job
71 * specific setup. Note, one of the important things to
72 * complete in this init code is to make the definitive
73 * choice of input and output storage devices. This is
74 * because immediately after the init, the job is queued
75 * in the jobq.c code, and it checks that all the resources
76 * (storage resources in particular) are available, so these
77 * must all be properly defined.
79 * previous_jr refers to the job DB record of the Job that is
80 * going to be migrated.
81 * prev_job refers to the job resource of the Job that is
82 * going to be migrated.
83 * jcr is the jcr for the current "migration" job. It is a
84 * control job that is put in the DB as a migration job, which
85 * means that this job migrated a previous job to a new job.
86 * No Volume or File data is associated with this control
88 * mig_jcr refers to the newly migrated job that is run by
89 * the current jcr. It is a backup job that moves (migrates) the
90 * data written for the previous_jr into the new pool. This
91 * job (mig_jcr) becomes the new backup job that replaces
92 * the original backup job. Note, this jcr is not really run. It
93 * is simply attached to the current jcr. It will show up in
94 * the Director's status output, but not in the SD or FD, both of
95 * which deal only with the current migration job (i.e. jcr).
97 bool do_migration_init(JCR *jcr)
103 JCR *mig_jcr; /* newly migrated job */
107 apply_pool_overrides(jcr);
109 if (!allow_duplicate_job(jcr)) {
113 jcr->jr.PoolId = get_or_create_pool_record(jcr, jcr->pool->name());
114 if (jcr->jr.PoolId == 0) {
115 Dmsg1(dbglevel, "JobId=%d no PoolId\n", (int)jcr->JobId);
116 Jmsg(jcr, M_FATAL, 0, _("Could not get or create a Pool record.\n"));
120 * Note, at this point, pool is the pool for this job. We
121 * transfer it to rpool (read pool), and a bit later,
122 * pool will be changed to point to the write pool,
123 * which comes from pool->NextPool.
125 jcr->rpool = jcr->pool; /* save read pool */
126 pm_strcpy(jcr->rpool_source, jcr->pool_source);
129 Dmsg2(dbglevel, "Read pool=%s (From %s)\n", jcr->rpool->name(), jcr->rpool_source);
131 /* If we find a job or jobs to migrate it is previous_jr.JobId */
132 count = get_job_to_migrate(jcr);
140 Dmsg1(dbglevel, "Back from get_job_to_migrate JobId=%d\n", (int)jcr->JobId);
142 if (jcr->previous_jr.JobId == 0) {
143 Dmsg1(dbglevel, "JobId=%d no previous JobId\n", (int)jcr->JobId);
144 Jmsg(jcr, M_INFO, 0, _("No previous Job found to migrate.\n"));
145 return true; /* no work */
148 if (!get_or_create_fileset_record(jcr)) {
149 Dmsg1(dbglevel, "JobId=%d no FileSet\n", (int)jcr->JobId);
150 Jmsg(jcr, M_FATAL, 0, _("Could not get or create the FileSet record.\n"));
154 create_restore_bootstrap_file(jcr);
156 if (jcr->previous_jr.JobId == 0 || jcr->ExpectedFiles == 0) {
157 set_jcr_job_status(jcr, JS_Terminated);
158 Dmsg1(dbglevel, "JobId=%d expected files == 0\n", (int)jcr->JobId);
159 if (jcr->previous_jr.JobId == 0) {
160 Jmsg(jcr, M_INFO, 0, _("No previous Job found to migrate.\n"));
162 Jmsg(jcr, M_INFO, 0, _("Previous Job has no data to migrate.\n"));
164 return true; /* no work */
167 Dmsg5(dbglevel, "JobId=%d: Previous: Name=%s JobId=%d Type=%c Level=%c\n",
169 jcr->previous_jr.Name, (int)jcr->previous_jr.JobId,
170 jcr->previous_jr.JobType, jcr->previous_jr.JobLevel);
172 Dmsg5(dbglevel, "JobId=%d: Current: Name=%s JobId=%d Type=%c Level=%c\n",
174 jcr->jr.Name, (int)jcr->jr.JobId,
175 jcr->jr.JobType, jcr->jr.JobLevel);
178 job = (JOB *)GetResWithName(R_JOB, jcr->jr.Name);
179 prev_job = (JOB *)GetResWithName(R_JOB, jcr->previous_jr.Name);
182 Jmsg(jcr, M_FATAL, 0, _("Job resource not found for \"%s\".\n"), jcr->jr.Name);
186 Jmsg(jcr, M_FATAL, 0, _("Previous Job resource not found for \"%s\".\n"),
187 jcr->previous_jr.Name);
191 jcr->spool_data = job->spool_data; /* turn on spooling if requested in job */
193 /* Create a migation jcr */
194 mig_jcr = jcr->mig_jcr = new_jcr(sizeof(JCR), dird_free_jcr);
195 memcpy(&mig_jcr->previous_jr, &jcr->previous_jr, sizeof(mig_jcr->previous_jr));
198 * Turn the mig_jcr into a "real" job that takes on the aspects of
199 * the previous backup job "prev_job".
201 set_jcr_defaults(mig_jcr, prev_job);
202 if (!setup_job(mig_jcr)) {
203 Jmsg(jcr, M_FATAL, 0, _("setup job failed.\n"));
207 /* Now reset the job record from the previous job */
208 memcpy(&mig_jcr->jr, &jcr->previous_jr, sizeof(mig_jcr->jr));
209 /* Update the jr to reflect the new values of PoolId and JobId. */
210 mig_jcr->jr.PoolId = jcr->jr.PoolId;
211 mig_jcr->jr.JobId = mig_jcr->JobId;
213 Dmsg4(dbglevel, "mig_jcr: Name=%s JobId=%d Type=%c Level=%c\n",
214 mig_jcr->jr.Name, (int)mig_jcr->jr.JobId,
215 mig_jcr->jr.JobType, mig_jcr->jr.JobLevel);
218 * Get the PoolId used with the original job. Then
219 * find the pool name from the database record.
221 memset(&pr, 0, sizeof(pr));
222 pr.PoolId = mig_jcr->previous_jr.PoolId;
223 if (!db_get_pool_record(jcr, jcr->db, &pr)) {
224 Jmsg(jcr, M_FATAL, 0, _("Pool for JobId %s not in database. ERR=%s\n"),
225 edit_int64(pr.PoolId, ed1), db_strerror(jcr->db));
228 /* Get the pool resource corresponding to the original job */
229 pool = (POOL *)GetResWithName(R_POOL, pr.Name);
231 Jmsg(jcr, M_FATAL, 0, _("Pool resource \"%s\" not found.\n"), pr.Name);
235 /* If pool storage specified, use it for restore */
236 copy_rstorage(mig_jcr, pool->storage, _("Pool resource"));
237 copy_rstorage(jcr, pool->storage, _("Pool resource"));
240 * If the original backup pool has a NextPool, make sure a
241 * record exists in the database. Note, in this case, we
242 * will be migrating from pool to pool->NextPool.
244 if (pool->NextPool) {
245 jcr->jr.PoolId = get_or_create_pool_record(jcr, pool->NextPool->name());
246 if (jcr->jr.PoolId == 0) {
250 if (!set_migration_wstorage(jcr, pool)) {
253 mig_jcr->pool = jcr->pool = pool->NextPool;
254 pm_strcpy(jcr->pool_source, _("Job Pool's NextPool resource"));
255 mig_jcr->jr.PoolId = jcr->jr.PoolId;
257 Dmsg2(dbglevel, "Write pool=%s read rpool=%s\n", jcr->pool->name(), jcr->rpool->name());
262 * Do a Migration of a previous job
264 * Returns: false on failure
267 bool do_migration(JCR *jcr)
271 JCR *mig_jcr = jcr->mig_jcr; /* newly migrated job */
274 * If mig_jcr is NULL, there is nothing to do for this job,
275 * so set a normal status, cleanup and return OK.
278 set_jcr_job_status(jcr, JS_Terminated);
279 migration_cleanup(jcr, jcr->JobStatus);
283 /* Print Job Start message */
284 Jmsg(jcr, M_INFO, 0, _("Start %s JobId %s, Job=%s\n"),
285 jcr->JobType == JT_MIGRATE ? "Migration" : "Copy",
286 edit_uint64(jcr->JobId, ed1), jcr->Job);
291 * Open a message channel connection with the Storage
292 * daemon. This is to let him know that our client
293 * will be contacting him for a backup session.
296 Dmsg0(110, "Open connection with storage daemon\n");
297 set_jcr_job_status(jcr, JS_WaitSD);
298 set_jcr_job_status(mig_jcr, JS_WaitSD);
300 * Start conversation with Storage daemon
302 if (!connect_to_storage_daemon(jcr, 10, SDConnectTimeout, 1)) {
305 sd = jcr->store_bsock;
307 * Now start a job with the Storage daemon
309 Dmsg2(dbglevel, "Read store=%s, write store=%s\n",
310 ((STORE *)jcr->rstorage->first())->name(),
311 ((STORE *)jcr->wstorage->first())->name());
312 if (((STORE *)jcr->rstorage->first())->name() == ((STORE *)jcr->wstorage->first())->name()) {
313 Jmsg(jcr, M_FATAL, 0, _("Read storage \"%s\" same as write storage.\n"),
314 ((STORE *)jcr->rstorage->first())->name());
317 if (!start_storage_daemon_job(jcr, jcr->rstorage, jcr->wstorage)) {
320 Dmsg0(150, "Storage daemon connection OK\n");
322 if (!send_bootstrap_file(jcr, sd) ||
323 !response(jcr, sd, OKbootstrap, "Bootstrap", DISPLAY_ERROR)) {
328 * We re-update the job start record so that the start
329 * time is set after the run before job. This avoids
330 * that any files created by the run before job will
331 * be saved twice. They will be backed up in the current
332 * job, but not in the next one unless they are changed.
333 * Without this, they will be backed up in this job and
334 * in the next job run because in that case, their date
335 * is after the start of this run.
337 jcr->start_time = time(NULL);
338 jcr->jr.StartTime = jcr->start_time;
339 jcr->jr.JobTDate = jcr->start_time;
340 set_jcr_job_status(jcr, JS_Running);
342 /* Update job start record for this migration control job */
343 if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) {
344 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db));
349 mig_jcr->start_time = time(NULL);
350 mig_jcr->jr.StartTime = mig_jcr->start_time;
351 mig_jcr->jr.JobTDate = mig_jcr->start_time;
352 set_jcr_job_status(mig_jcr, JS_Running);
354 /* Update job start record for the real migration backup job */
355 if (!db_update_job_start_record(mig_jcr, mig_jcr->db, &mig_jcr->jr)) {
356 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(mig_jcr->db));
360 Dmsg4(dbglevel, "mig_jcr: Name=%s JobId=%d Type=%c Level=%c\n",
361 mig_jcr->jr.Name, (int)mig_jcr->jr.JobId,
362 mig_jcr->jr.JobType, mig_jcr->jr.JobLevel);
366 * Start the job prior to starting the message thread below
367 * to avoid two threads from using the BSOCK structure at
370 if (!sd->fsend("run")) {
375 * Now start a Storage daemon message thread
377 if (!start_storage_daemon_message_thread(jcr)) {
382 set_jcr_job_status(jcr, JS_Running);
383 set_jcr_job_status(mig_jcr, JS_Running);
385 /* Pickup Job termination data */
386 /* Note, the SD stores in jcr->JobFiles/ReadBytes/JobBytes/Errors */
387 wait_for_storage_daemon_termination(jcr);
388 set_jcr_job_status(jcr, jcr->SDJobStatus);
389 db_write_batch_file_records(jcr); /* used by bulk batch file insert */
390 if (jcr->JobStatus != JS_Terminated) {
394 migration_cleanup(jcr, jcr->JobStatus);
395 if (jcr->JobType == JT_MIGRATE && mig_jcr) {
397 UAContext *ua = new_ua_context(jcr);
398 edit_uint64(jcr->previous_jr.JobId, jobid);
399 /* Purge all old file records, but leave Job record */
400 purge_files_from_jobs(ua, jobid);
411 /* Add an item to the list if it is unique */
412 static void add_unique_id(idpkt *ids, char *item)
414 const int maxlen = 30;
418 /* Walk through current list to see if each item is the same as item */
421 for (int i=0; i<maxlen; i++) {
424 } else if (*q == ',') {
431 if (strcmp(item, id) == 0) {
435 /* Did not find item, so add it to list */
436 if (ids->count == 0) {
439 pm_strcat(ids->list, ",");
441 pm_strcat(ids->list, item);
443 // Dmsg3(0, "add_uniq count=%d Ids=%p %s\n", ids->count, ids->list, ids->list);
448 * Callback handler make list of DB Ids
450 static int unique_dbid_handler(void *ctx, int num_fields, char **row)
452 idpkt *ids = (idpkt *)ctx;
454 add_unique_id(ids, row[0]);
455 Dmsg3(dbglevel, "dbid_hdlr count=%d Ids=%p %s\n", ids->count, ids->list, ids->list);
465 static int item_compare(void *item1, void *item2)
467 uitem *i1 = (uitem *)item1;
468 uitem *i2 = (uitem *)item2;
469 return strcmp(i1->item, i2->item);
472 static int unique_name_handler(void *ctx, int num_fields, char **row)
474 dlist *list = (dlist *)ctx;
476 uitem *new_item = (uitem *)malloc(sizeof(uitem));
479 memset(new_item, 0, sizeof(uitem));
480 new_item->item = bstrdup(row[0]);
481 Dmsg1(dbglevel, "Unique_name_hdlr Item=%s\n", row[0]);
482 item = (uitem *)list->binary_insert((void *)new_item, item_compare);
483 if (item != new_item) { /* already in list */
484 free(new_item->item);
485 free((char *)new_item);
491 /* Get Job names in Pool */
492 const char *sql_job =
493 "SELECT DISTINCT Job.Name from Job,Pool"
494 " WHERE Pool.Name='%s' AND Job.PoolId=Pool.PoolId";
496 /* Get JobIds from regex'ed Job names */
497 const char *sql_jobids_from_job =
498 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Job,Pool"
499 " WHERE Job.Name='%s' AND Pool.Name='%s' AND Job.PoolId=Pool.PoolId"
500 " ORDER by Job.StartTime";
502 /* Get Client names in Pool */
503 const char *sql_client =
504 "SELECT DISTINCT Client.Name from Client,Pool,Job"
505 " WHERE Pool.Name='%s' AND Job.ClientId=Client.ClientId AND"
506 " Job.PoolId=Pool.PoolId";
508 /* Get JobIds from regex'ed Client names */
509 const char *sql_jobids_from_client =
510 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Job,Pool,Client"
511 " WHERE Client.Name='%s' AND Pool.Name='%s' AND Job.PoolId=Pool.PoolId"
512 " AND Job.ClientId=Client.ClientId AND Job.Type='B'"
513 " ORDER by Job.StartTime";
515 /* Get Volume names in Pool */
516 const char *sql_vol =
517 "SELECT DISTINCT VolumeName FROM Media,Pool WHERE"
518 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
519 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'";
521 /* Get JobIds from regex'ed Volume names */
522 const char *sql_jobids_from_vol =
523 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Media,JobMedia,Job"
524 " WHERE Media.VolumeName='%s' AND Media.MediaId=JobMedia.MediaId"
525 " AND JobMedia.JobId=Job.JobId AND Job.Type='B'"
526 " ORDER by Job.StartTime";
529 const char *sql_smallest_vol =
530 "SELECT Media.MediaId FROM Media,Pool,JobMedia WHERE"
531 " Media.MediaId in (SELECT DISTINCT MediaId from JobMedia) AND"
532 " Media.VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
533 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'"
534 " ORDER BY VolBytes ASC LIMIT 1";
536 const char *sql_oldest_vol =
537 "SELECT Media.MediaId FROM Media,Pool,JobMedia WHERE"
538 " Media.MediaId in (SELECT DISTINCT MediaId from JobMedia) AND"
539 " Media.VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
540 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'"
541 " ORDER BY LastWritten ASC LIMIT 1";
543 /* Get JobIds when we have selected MediaId */
544 const char *sql_jobids_from_mediaid =
545 "SELECT DISTINCT Job.JobId,Job.StartTime FROM JobMedia,Job"
546 " WHERE JobMedia.JobId=Job.JobId AND JobMedia.MediaId IN (%s)"
548 " ORDER by Job.StartTime";
550 /* Get tne number of bytes in the pool */
551 const char *sql_pool_bytes =
552 "SELECT SUM(VolBytes) FROM Media,Pool WHERE"
553 " VolStatus in ('Full','Used','Error','Append') AND Media.Enabled=1 AND"
554 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'";
556 /* Get tne number of bytes in the Jobs */
557 const char *sql_job_bytes =
558 "SELECT SUM(JobBytes) FROM Job WHERE JobId IN (%s)";
561 /* Get Media Ids in Pool */
562 const char *sql_mediaids =
563 "SELECT MediaId FROM Media,Pool WHERE"
564 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
565 " Media.PoolId=Pool.PoolId AND Pool.Name='%s' ORDER BY LastWritten ASC";
567 /* Get JobIds in Pool longer than specified time */
568 const char *sql_pool_time =
569 "SELECT DISTINCT Job.JobId from Pool,Job,Media,JobMedia WHERE"
570 " Pool.Name='%s' AND Media.PoolId=Pool.PoolId AND"
571 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
573 " JobMedia.JobId=Job.JobId AND Job.PoolId=Media.PoolId"
574 " AND Job.RealEndTime<='%s'";
577 * const char *sql_ujobid =
578 * "SELECT DISTINCT Job.Job from Client,Pool,Media,Job,JobMedia "
579 * " WHERE Media.PoolId=Pool.PoolId AND Pool.Name='%s' AND"
580 * " JobMedia.JobId=Job.JobId AND Job.PoolId=Media.PoolId";
587 * This is the central piece of code that finds a job or jobs
588 * actually JobIds to migrate. It first looks to see if one
589 * has been "manually" specified in jcr->MigrateJobId, and if
590 * so, it returns that JobId to be run. Otherwise, it
591 * examines the Selection Type to see what kind of migration
592 * we are doing (Volume, Job, Client, ...) and applies any
593 * Selection Pattern if appropriate to obtain a list of JobIds.
594 * Finally, it will loop over all the JobIds found, except the last
595 * one starting a new job with MigrationJobId set to that JobId, and
596 * finally, it returns the last JobId to the caller.
598 * Returns: -1 on error
599 * 0 if no jobs to migrate
600 * 1 if OK and jcr->previous_jr filled in
602 static int get_job_to_migrate(JCR *jcr)
604 char ed1[30], ed2[30];
605 POOL_MEM query(PM_MESSAGE);
610 idpkt ids, mid, jids;
615 char dt[MAX_TIME_LENGTH];
618 ids.list = get_pool_memory(PM_MESSAGE);
621 mid.list = get_pool_memory(PM_MESSAGE);
624 jids.list = get_pool_memory(PM_MESSAGE);
630 * If MigrateJobId is set, then we migrate only that Job,
631 * otherwise, we go through the full selection of jobs to
634 if (jcr->MigrateJobId != 0) {
635 Dmsg1(dbglevel, "At Job start previous jobid=%u\n", jcr->MigrateJobId);
636 edit_uint64(jcr->MigrateJobId, ids.list);
639 switch (jcr->job->selection_type) {
641 if (!regex_find_jobids(jcr, &ids, sql_job, sql_jobids_from_job, "Job")) {
646 if (!regex_find_jobids(jcr, &ids, sql_client, sql_jobids_from_client, "Client")) {
651 if (!regex_find_jobids(jcr, &ids, sql_vol, sql_jobids_from_vol, "Volume")) {
656 if (!jcr->job->selection_pattern) {
657 Jmsg(jcr, M_FATAL, 0, _("No Migration SQL selection pattern specified.\n"));
660 Dmsg1(dbglevel, "SQL=%s\n", jcr->job->selection_pattern);
661 if (!db_sql_query(jcr->db, jcr->job->selection_pattern,
662 unique_dbid_handler, (void *)&ids)) {
663 Jmsg(jcr, M_FATAL, 0,
664 _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
668 case MT_SMALLEST_VOL:
669 if (!find_mediaid_then_jobids(jcr, &ids, sql_smallest_vol, "Smallest Volume")) {
674 if (!find_mediaid_then_jobids(jcr, &ids, sql_oldest_vol, "Oldest Volume")) {
679 case MT_POOL_OCCUPANCY:
681 /* Find count of bytes in pool */
682 Mmsg(query, sql_pool_bytes, jcr->rpool->name());
683 if (!db_sql_query(jcr->db, query.c_str(), db_int64_handler, (void *)&ctx)) {
684 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
687 if (ctx.count == 0) {
688 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
691 pool_bytes = ctx.value;
692 Dmsg2(dbglevel, "highbytes=%lld pool=%lld\n", jcr->rpool->MigrationHighBytes,
694 if (pool_bytes < (int64_t)jcr->rpool->MigrationHighBytes) {
695 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
698 Dmsg0(dbglevel, "We should do Occupation migration.\n");
701 /* Find a list of MediaIds that could be migrated */
702 Mmsg(query, sql_mediaids, jcr->rpool->name());
703 Dmsg1(dbglevel, "query=%s\n", query.c_str());
704 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)&ids)) {
705 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
708 if (ids.count == 0) {
709 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
712 Dmsg2(dbglevel, "Pool Occupancy ids=%d MediaIds=%s\n", ids.count, ids.list);
714 if (!find_jobids_from_mediaid_list(jcr, &ids, "Volumes")) {
717 /* ids == list of jobs */
719 for (int i=0; i < (int)ids.count; i++) {
720 stat = get_next_dbid_from_list(&p, &DBId);
721 Dmsg2(dbglevel, "get_next_dbid stat=%d JobId=%u\n", stat, (uint32_t)DBId);
723 Jmsg(jcr, M_FATAL, 0, _("Invalid JobId found.\n"));
725 } else if (stat == 0) {
730 Mmsg(mid.list, "%s", edit_int64(DBId, ed1));
731 if (jids.count > 0) {
732 pm_strcat(jids.list, ",");
734 pm_strcat(jids.list, mid.list);
735 jids.count += mid.count;
737 /* Find count of bytes from Jobs */
738 Mmsg(query, sql_job_bytes, mid.list);
739 Dmsg1(dbglevel, "Jobbytes query: %s\n", query.c_str());
740 if (!db_sql_query(jcr->db, query.c_str(), db_int64_handler, (void *)&ctx)) {
741 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
744 pool_bytes -= ctx.value;
745 Dmsg1(dbglevel, "Total migrate Job bytes=%s\n", edit_int64(ctx.value, ed1));
746 Dmsg2(dbglevel, "lowbytes=%s poolafter=%s\n",
747 edit_int64(jcr->rpool->MigrationLowBytes, ed1),
748 edit_int64(pool_bytes, ed2));
749 if (pool_bytes <= (int64_t)jcr->rpool->MigrationLowBytes) {
750 Dmsg0(dbglevel, "We should be done.\n");
754 /* Transfer jids to ids, where the jobs list is expected */
755 ids.count = jids.count;
756 pm_strcpy(ids.list, jids.list);
757 Dmsg2(dbglevel, "Pool Occupancy ids=%d JobIds=%s\n", ids.count, ids.list);
761 ttime = time(NULL) - (time_t)jcr->rpool->MigrationTime;
762 (void)localtime_r(&ttime, &tm);
763 strftime(dt, sizeof(dt), "%Y-%m-%d %H:%M:%S", &tm);
766 Mmsg(query, sql_pool_time, jcr->rpool->name(), dt);
767 Dmsg1(dbglevel, "query=%s\n", query.c_str());
768 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)&ids)) {
769 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
772 if (ids.count == 0) {
773 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
776 Dmsg2(dbglevel, "PoolTime ids=%d JobIds=%s\n", ids.count, ids.list);
780 Jmsg(jcr, M_FATAL, 0, _("Unknown Migration Selection Type.\n"));
786 * Loop over all jobids except the last one, sending
787 * them to start_migration_job(), which will start a job
788 * for each of them. For the last JobId, we handle it below.
791 if (ids.count == 0) {
792 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
796 Jmsg(jcr, M_INFO, 0, _("The following %u JobId%s were chosen to be migrated: %s\n"),
797 ids.count, ids.count==0?"":"s", ids.list);
799 Dmsg2(dbglevel, "Before loop count=%d ids=%s\n", ids.count, ids.list);
800 for (int i=1; i < (int)ids.count; i++) {
802 stat = get_next_jobid_from_list(&p, &JobId);
803 Dmsg3(dbglevel, "get_jobid_no=%d stat=%d JobId=%u\n", i, stat, JobId);
804 jcr->MigrateJobId = JobId;
805 start_migration_job(jcr);
806 Dmsg0(dbglevel, "Back from start_migration_job\n");
808 Jmsg(jcr, M_FATAL, 0, _("Invalid JobId found.\n"));
810 } else if (stat == 0) {
811 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
816 /* Now get the last JobId and handle it in the current job */
818 stat = get_next_jobid_from_list(&p, &JobId);
819 Dmsg2(dbglevel, "Last get_next_jobid stat=%d JobId=%u\n", stat, (int)JobId);
821 Jmsg(jcr, M_FATAL, 0, _("Invalid JobId found.\n"));
823 } else if (stat == 0) {
824 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
828 jcr->previous_jr.JobId = JobId;
829 Dmsg1(dbglevel, "Previous jobid=%d\n", (int)jcr->previous_jr.JobId);
831 if (!db_get_job_record(jcr, jcr->db, &jcr->previous_jr)) {
832 Jmsg(jcr, M_FATAL, 0, _("Could not get job record for JobId %s to migrate. ERR=%s"),
833 edit_int64(jcr->previous_jr.JobId, ed1),
834 db_strerror(jcr->db));
837 Jmsg(jcr, M_INFO, 0, _("Migration using JobId=%s Job=%s\n"),
838 edit_int64(jcr->previous_jr.JobId, ed1), jcr->previous_jr.Job);
839 Dmsg3(dbglevel, "Migration JobId=%d using JobId=%s Job=%s\n",
841 edit_int64(jcr->previous_jr.JobId, ed1), jcr->previous_jr.Job);
851 free_pool_memory(ids.list);
852 free_pool_memory(mid.list);
853 free_pool_memory(jids.list);
857 static void start_migration_job(JCR *jcr)
859 UAContext *ua = new_ua_context(jcr);
862 Mmsg(ua->cmd, "run %s jobid=%s", jcr->job->hdr.name,
863 edit_uint64(jcr->MigrateJobId, ed1));
864 Dmsg1(dbglevel, "=============== Migration cmd=%s\n", ua->cmd);
865 parse_ua_args(ua); /* parse command */
866 JobId_t jobid = run_cmd(ua, ua->cmd);
868 Jmsg(jcr, M_ERROR, 0, _("Could not start migration job.\n"));
870 Jmsg(jcr, M_INFO, 0, _("Migration JobId %d started.\n"), (int)jobid);
875 static bool find_mediaid_then_jobids(JCR *jcr, idpkt *ids, const char *query1,
879 POOL_MEM query(PM_MESSAGE);
882 /* Basic query for MediaId */
883 Mmsg(query, query1, jcr->rpool->name());
884 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
885 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
888 if (ids->count == 0) {
889 Jmsg(jcr, M_INFO, 0, _("No %s found to migrate.\n"), type);
890 ok = true; /* Not an error */
892 } else if (ids->count != 1) {
893 Jmsg(jcr, M_FATAL, 0, _("SQL error. Expected 1 MediaId got %d\n"), ids->count);
896 Dmsg2(dbglevel, "%s MediaIds=%s\n", type, ids->list);
898 ok = find_jobids_from_mediaid_list(jcr, ids, type);
905 * This routine returns:
906 * false if an error occurred
908 * ids.count number of jobids found (may be zero)
910 static bool find_jobids_from_mediaid_list(JCR *jcr, idpkt *ids, const char *type)
913 POOL_MEM query(PM_MESSAGE);
915 Mmsg(query, sql_jobids_from_mediaid, ids->list);
917 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
918 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
921 if (ids->count == 0) {
922 Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type);
930 static bool regex_find_jobids(JCR *jcr, idpkt *ids, const char *query1,
931 const char *query2, const char *type)
935 uitem *last_item = NULL;
940 POOL_MEM query(PM_MESSAGE);
942 item_chain = New(dlist(item, &item->link));
943 if (!jcr->job->selection_pattern) {
944 Jmsg(jcr, M_FATAL, 0, _("No Migration %s selection pattern specified.\n"),
948 Dmsg1(dbglevel, "regex-sel-pattern=%s\n", jcr->job->selection_pattern);
949 /* Basic query for names */
950 Mmsg(query, query1, jcr->rpool->name());
951 Dmsg1(dbglevel, "get name query1=%s\n", query.c_str());
952 if (!db_sql_query(jcr->db, query.c_str(), unique_name_handler,
953 (void *)item_chain)) {
954 Jmsg(jcr, M_FATAL, 0,
955 _("SQL to get %s failed. ERR=%s\n"), type, db_strerror(jcr->db));
958 Dmsg1(dbglevel, "query1 returned %d names\n", item_chain->size());
959 if (item_chain->size() == 0) {
960 Jmsg(jcr, M_INFO, 0, _("Query of Pool \"%s\" returned no Jobs to migrate.\n"),
963 goto bail_out; /* skip regex match */
965 /* Compile regex expression */
966 rc = regcomp(&preg, jcr->job->selection_pattern, REG_EXTENDED);
968 regerror(rc, &preg, prbuf, sizeof(prbuf));
969 Jmsg(jcr, M_FATAL, 0, _("Could not compile regex pattern \"%s\" ERR=%s\n"),
970 jcr->job->selection_pattern, prbuf);
973 /* Now apply the regex to the names and remove any item not matched */
974 foreach_dlist(item, item_chain) {
975 const int nmatch = 30;
976 regmatch_t pmatch[nmatch];
978 Dmsg1(dbglevel, "Remove item %s\n", last_item->item);
979 free(last_item->item);
980 item_chain->remove(last_item);
982 Dmsg1(dbglevel, "get name Item=%s\n", item->item);
983 rc = regexec(&preg, item->item, nmatch, pmatch, 0);
985 last_item = NULL; /* keep this one */
991 free(last_item->item);
992 Dmsg1(dbglevel, "Remove item %s\n", last_item->item);
993 item_chain->remove(last_item);
997 if (item_chain->size() == 0) {
998 Jmsg(jcr, M_INFO, 0, _("Regex pattern matched no Jobs to migrate.\n"));
1000 goto bail_out; /* skip regex match */
1004 * At this point, we have a list of items in item_chain
1005 * that have been matched by the regex, so now we need
1006 * to look up their jobids.
1009 foreach_dlist(item, item_chain) {
1010 Dmsg2(dbglevel, "Got %s: %s\n", type, item->item);
1011 Mmsg(query, query2, item->item, jcr->rpool->name());
1012 Dmsg1(dbglevel, "get id from name query2=%s\n", query.c_str());
1013 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
1014 Jmsg(jcr, M_FATAL, 0,
1015 _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
1019 if (ids->count == 0) {
1020 Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type);
1025 Dmsg2(dbglevel, "Count=%d Jobids=%s\n", ids->count, ids->list);
1026 foreach_dlist(item, item_chain) {
1035 * Release resources allocated during backup.
1037 void migration_cleanup(JCR *jcr, int TermCode)
1039 char sdt[MAX_TIME_LENGTH], edt[MAX_TIME_LENGTH];
1040 char ec1[30], ec2[30], ec3[30], ec4[30], ec5[30], elapsed[50];
1041 char ec6[50], ec7[50], ec8[50];
1042 char term_code[100], sd_term_msg[100];
1043 const char *term_msg;
1044 int msg_type = M_INFO;
1048 JCR *mig_jcr = jcr->mig_jcr;
1049 POOL_MEM query(PM_MESSAGE);
1051 Dmsg2(100, "Enter migrate_cleanup %d %c\n", TermCode, TermCode);
1052 update_job_end(jcr, TermCode);
1053 memset(&mr, 0, sizeof(mr));
1056 * Check if we actually did something.
1057 * mig_jcr is jcr of the newly migrated job.
1060 mig_jcr->JobFiles = jcr->JobFiles = jcr->SDJobFiles;
1061 mig_jcr->JobBytes = jcr->JobBytes = jcr->SDJobBytes;
1062 mig_jcr->VolSessionId = jcr->VolSessionId;
1063 mig_jcr->VolSessionTime = jcr->VolSessionTime;
1064 mig_jcr->jr.RealEndTime = 0;
1065 mig_jcr->jr.PriorJobId = jcr->previous_jr.JobId;
1067 update_job_end(mig_jcr, TermCode);
1069 /* Update final items to set them to the previous job's values */
1070 Mmsg(query, "UPDATE Job SET StartTime='%s',EndTime='%s',"
1071 "JobTDate=%s WHERE JobId=%s",
1072 jcr->previous_jr.cStartTime, jcr->previous_jr.cEndTime,
1073 edit_uint64(jcr->previous_jr.JobTDate, ec1),
1074 edit_uint64(mig_jcr->jr.JobId, ec2));
1075 db_sql_query(mig_jcr->db, query.c_str(), NULL, NULL);
1077 /* Now mark the previous job as migrated if it terminated normally */
1078 if (jcr->JobType == JT_MIGRATE && jcr->JobStatus == JS_Terminated) {
1079 Mmsg(query, "UPDATE Job SET Type='%c' WHERE JobId=%s",
1080 (char)JT_MIGRATED_JOB, edit_uint64(jcr->previous_jr.JobId, ec1));
1081 db_sql_query(mig_jcr->db, query.c_str(), NULL, NULL);
1084 if (!db_get_job_record(jcr, jcr->db, &jcr->jr)) {
1085 Jmsg(jcr, M_WARNING, 0, _("Error getting Job record for Job report: ERR=%s"),
1086 db_strerror(jcr->db));
1087 set_jcr_job_status(jcr, JS_ErrorTerminated);
1090 update_bootstrap_file(mig_jcr);
1092 if (!db_get_job_volume_names(mig_jcr, mig_jcr->db, mig_jcr->jr.JobId, &mig_jcr->VolumeName)) {
1094 * Note, if the job has failed, most likely it did not write any
1095 * tape, so suppress this "error" message since in that case
1096 * it is normal. Or look at it the other way, only for a
1097 * normal exit should we complain about this error.
1099 if (jcr->JobStatus == JS_Terminated && jcr->jr.JobBytes) {
1100 Jmsg(jcr, M_ERROR, 0, "%s", db_strerror(mig_jcr->db));
1102 mig_jcr->VolumeName[0] = 0; /* none */
1105 if (mig_jcr->VolumeName[0]) {
1106 /* Find last volume name. Multiple vols are separated by | */
1107 char *p = strrchr(mig_jcr->VolumeName, '|');
1111 p = mig_jcr->VolumeName; /* no |, take full name */
1113 bstrncpy(mr.VolumeName, p, sizeof(mr.VolumeName));
1114 if (!db_get_media_record(jcr, jcr->db, &mr)) {
1115 Jmsg(jcr, M_WARNING, 0, _("Error getting Media record for Volume \"%s\": ERR=%s"),
1116 mr.VolumeName, db_strerror(jcr->db));
1120 switch (jcr->JobStatus) {
1122 if (jcr->Errors || jcr->SDErrors) {
1123 term_msg = _("%s OK -- with warnings");
1125 term_msg = _("%s OK");
1129 case JS_ErrorTerminated:
1130 term_msg = _("*** %s Error ***");
1131 msg_type = M_ERROR; /* Generate error message */
1132 if (jcr->store_bsock) {
1133 bnet_sig(jcr->store_bsock, BNET_TERMINATE);
1134 if (jcr->SD_msg_chan) {
1135 pthread_cancel(jcr->SD_msg_chan);
1140 term_msg = _("%s Canceled");
1141 if (jcr->store_bsock) {
1142 bnet_sig(jcr->store_bsock, BNET_TERMINATE);
1143 if (jcr->SD_msg_chan) {
1144 pthread_cancel(jcr->SD_msg_chan);
1149 term_msg = _("Inappropriate %s term code");
1153 if (jcr->JobType == JT_MIGRATE && jcr->previous_jr.JobId != 0) {
1154 /* Mark previous job as migrated */
1155 Mmsg(query, "UPDATE Job SET Type='%c' WHERE JobId=%s",
1156 (char)JT_MIGRATED_JOB, edit_uint64(jcr->previous_jr.JobId, ec1));
1157 db_sql_query(jcr->db, query.c_str(), NULL, NULL);
1159 term_msg = _("%s -- no files to migrate");
1162 bsnprintf(term_code, sizeof(term_code), term_msg, "Migration");
1163 bstrftimes(sdt, sizeof(sdt), jcr->jr.StartTime);
1164 bstrftimes(edt, sizeof(edt), jcr->jr.EndTime);
1165 RunTime = jcr->jr.EndTime - jcr->jr.StartTime;
1169 kbps = (double)jcr->SDJobBytes / (1000 * RunTime);
1173 jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg));
1175 Jmsg(jcr, msg_type, 0, _("Bacula %s %s (%s): %s\n"
1176 " Build OS: %s %s %s\n"
1177 " Prev Backup JobId: %s\n"
1178 " New Backup JobId: %s\n"
1179 " Current JobId: %s\n"
1180 " Current Job: %s\n"
1181 " Backup Level: %s%s\n"
1183 " FileSet: \"%s\" %s\n"
1184 " Read Pool: \"%s\" (From %s)\n"
1185 " Read Storage: \"%s\" (From %s)\n"
1186 " Write Pool: \"%s\" (From %s)\n"
1187 " Write Storage: \"%s\" (From %s)\n"
1188 " Catalog: \"%s\" (From %s)\n"
1191 " Elapsed time: %s\n"
1193 " SD Files Written: %s\n"
1194 " SD Bytes Written: %s (%sB)\n"
1195 " Rate: %.1f KB/s\n"
1196 " Volume name(s): %s\n"
1197 " Volume Session Id: %d\n"
1198 " Volume Session Time: %d\n"
1199 " Last Volume Bytes: %s (%sB)\n"
1201 " SD termination status: %s\n"
1202 " Termination: %s\n\n"),
1203 my_name, VERSION, LSMDATE, edt,
1204 HOST_OS, DISTNAME, DISTVER,
1205 edit_uint64(jcr->previous_jr.JobId, ec6),
1206 mig_jcr ? edit_uint64(mig_jcr->jr.JobId, ec7) : "0",
1207 edit_uint64(jcr->jr.JobId, ec8),
1209 level_to_str(jcr->JobLevel), jcr->since,
1210 jcr->client->name(),
1211 jcr->fileset->name(), jcr->FSCreateTime,
1212 jcr->rpool->name(), jcr->rpool_source,
1213 jcr->rstore?jcr->rstore->name():"*None*",
1214 NPRT(jcr->rstore_source),
1215 jcr->pool->name(), jcr->pool_source,
1216 jcr->wstore?jcr->wstore->name():"*None*",
1217 NPRT(jcr->wstore_source),
1218 jcr->catalog->name(), jcr->catalog_source,
1221 edit_utime(RunTime, elapsed, sizeof(elapsed)),
1223 edit_uint64_with_commas(jcr->SDJobFiles, ec1),
1224 edit_uint64_with_commas(jcr->SDJobBytes, ec2),
1225 edit_uint64_with_suffix(jcr->SDJobBytes, ec3),
1227 mig_jcr ? mig_jcr->VolumeName : "",
1229 jcr->VolSessionTime,
1230 edit_uint64_with_commas(mr.VolBytes, ec4),
1231 edit_uint64_with_suffix(mr.VolBytes, ec5),
1236 Dmsg1(100, "migrate_cleanup() mig_jcr=0x%x\n", jcr->mig_jcr);
1238 free_jcr(jcr->mig_jcr);
1239 jcr->mig_jcr = NULL;
1241 Dmsg0(100, "Leave migrate_cleanup()\n");
1245 * Return next DBId from comma separated list
1248 * 1 if next DBId returned
1249 * 0 if no more DBIds are in list
1250 * -1 there is an error
1252 static int get_next_dbid_from_list(char **p, DBId_t *DBId)
1254 const int maxlen = 30;
1259 for (int i=0; i<maxlen; i++) {
1262 } else if (*q == ',') {
1271 } else if (!is_a_number(id)) {
1272 return -1; /* error */
1275 *DBId = str_to_int64(id);
1279 bool set_migration_wstorage(JCR *jcr, POOL *pool)
1281 POOL *wpool = pool->NextPool;
1284 Jmsg(jcr, M_FATAL, 0, _("No Next Pool specification found in Pool \"%s\".\n"),
1289 if (!wpool->storage || wpool->storage->size() == 0) {
1290 Jmsg(jcr, M_FATAL, 0, _("No Storage specification found in Next Pool \"%s\".\n"),
1295 /* If pool storage specified, use it instead of job storage for backup */
1296 copy_wstorage(jcr, wpool->storage, _("Storage from Pool's NextPool resource"));