3 * Bacula Director -- migrate.c -- responsible for doing
6 * Kern Sibbald, September MMIV
8 * Basic tasks done here:
9 * Open DB and create records for this job.
10 * Open Message Channel with Storage daemon to tell him a job will be starting.
11 * Open connection with Storage daemon and pass him commands
13 * When the Storage daemon finishes the job, update the DB.
18 Bacula® - The Network Backup Solution
20 Copyright (C) 2004-2006 Free Software Foundation Europe e.V.
22 The main author of Bacula is Kern Sibbald, with contributions from
23 many others, a complete list can be found in the file AUTHORS.
24 This program is Free Software; you can redistribute it and/or
25 modify it under the terms of version two of the GNU General Public
26 License as published by the Free Software Foundation plus additions
27 that are listed in the file LICENSE.
29 This program is distributed in the hope that it will be useful, but
30 WITHOUT ANY WARRANTY; without even the implied warranty of
31 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
32 General Public License for more details.
34 You should have received a copy of the GNU General Public License
35 along with this program; if not, write to the Free Software
36 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
39 Bacula® is a registered trademark of John Walker.
40 The licensor of Bacula is the Free Software Foundation Europe
41 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
42 Switzerland, email:ftf@fsfeurope.org.
49 #include "lib/bregex.h"
54 static const int dbglevel = 10;
56 static char OKbootstrap[] = "3000 OK bootstrap\n";
57 static bool get_job_to_migrate(JCR *jcr);
59 static bool regex_find_jobids(JCR *jcr, idpkt *ids, const char *query1,
60 const char *query2, const char *type);
61 static bool find_mediaid_then_jobids(JCR *jcr, idpkt *ids, const char *query1,
63 static bool find_jobids_from_mediaid_list(JCR *jcr, idpkt *ids, const char *type);
64 static void start_migration_job(JCR *jcr);
65 static int get_next_dbid_from_list(char **p, DBId_t *DBId);
68 * Called here before the job is run to do the job
69 * specific setup. Note, one of the important things to
70 * complete in this init code is to make the definitive
71 * choice of input and output storage devices. This is
72 * because immediately after the init, the job is queued
73 * in the jobq.c code, and it checks that all the resources
74 * (storage resources in particular) are available, so these
75 * must all be properly defined.
77 * previous_jr refers to the job DB record of the Job that is
78 * going to be migrated.
79 * prev_job refers to the job resource of the Job that is
80 * going to be migrated.
81 * jcr is the jcr for the current "migration" job. It is a
82 * control job that is put in the DB as a migration job, which
83 * means that this job migrated a previous job to a new job.
84 * No Volume or File data is associated with this control
86 * mig_jcr refers to the newly migrated job that is run by
87 * the current jcr. It is a backup job that moves (migrates) the
88 * data written for the previous_jr into the new pool. This
89 * job (mig_jcr) becomes the new backup job that replaces
90 * the original backup job.
92 bool do_migration_init(JCR *jcr)
98 JCR *mig_jcr; /* newly migrated job */
100 /* If we find a job or jobs to migrate it is previous_jr.JobId */
101 if (!get_job_to_migrate(jcr)) {
104 Dmsg1(dbglevel, "Back from get_job_to_migrate JobId=%d\n", (int)jcr->JobId);
106 if (jcr->previous_jr.JobId == 0) {
107 Dmsg1(dbglevel, "JobId=%d no previous JobId\n", (int)jcr->JobId);
108 Jmsg(jcr, M_INFO, 0, _("No previous Job found to migrate.\n"));
109 return true; /* no work */
112 if (!get_or_create_fileset_record(jcr)) {
113 Dmsg1(dbglevel, "JobId=%d no FileSet\n", (int)jcr->JobId);
114 Jmsg(jcr, M_FATAL, 0, _("Could not get or create the FileSet record.\n"));
118 apply_pool_overrides(jcr);
120 jcr->jr.PoolId = get_or_create_pool_record(jcr, jcr->pool->hdr.name);
121 if (jcr->jr.PoolId == 0) {
122 Dmsg1(dbglevel, "JobId=%d no PoolId\n", (int)jcr->JobId);
123 Jmsg(jcr, M_FATAL, 0, _("Could not get or create a Pool record.\n"));
127 create_restore_bootstrap_file(jcr);
129 if (jcr->previous_jr.JobId == 0 || jcr->ExpectedFiles == 0) {
130 set_jcr_job_status(jcr, JS_Terminated);
131 Dmsg1(dbglevel, "JobId=%d expected files == 0\n", (int)jcr->JobId);
132 if (jcr->previous_jr.JobId == 0) {
133 Jmsg(jcr, M_INFO, 0, _("No previous Job found to migrate.\n"));
135 Jmsg(jcr, M_INFO, 0, _("Previous Job has no data to migrate.\n"));
137 return true; /* no work */
140 Dmsg5(dbglevel, "JobId=%d: Previous: Name=%s JobId=%d Type=%c Level=%c\n",
142 jcr->previous_jr.Name, (int)jcr->previous_jr.JobId,
143 jcr->previous_jr.JobType, jcr->previous_jr.JobLevel);
145 Dmsg5(dbglevel, "JobId=%d: Current: Name=%s JobId=%d Type=%c Level=%c\n",
147 jcr->jr.Name, (int)jcr->jr.JobId,
148 jcr->jr.JobType, jcr->jr.JobLevel);
151 job = (JOB *)GetResWithName(R_JOB, jcr->jr.Name);
152 prev_job = (JOB *)GetResWithName(R_JOB, jcr->previous_jr.Name);
155 Jmsg(jcr, M_FATAL, 0, _("Job resource not found for \"%s\".\n"), jcr->jr.Name);
159 Jmsg(jcr, M_FATAL, 0, _("Previous Job resource not found for \"%s\".\n"),
160 jcr->previous_jr.Name);
164 /* Create a migation jcr */
165 mig_jcr = jcr->mig_jcr = new_jcr(sizeof(JCR), dird_free_jcr);
166 memcpy(&mig_jcr->previous_jr, &jcr->previous_jr, sizeof(mig_jcr->previous_jr));
169 * Turn the mig_jcr into a "real" job that takes on the aspects of
170 * the previous backup job "prev_job".
172 set_jcr_defaults(mig_jcr, prev_job);
173 if (!setup_job(mig_jcr)) {
174 Jmsg(jcr, M_FATAL, 0, _("setup job failed.\n"));
178 /* Now reset the job record from the previous job */
179 memcpy(&mig_jcr->jr, &jcr->previous_jr, sizeof(mig_jcr->jr));
180 /* Update the jr to reflect the new values of PoolId, FileSetId, and JobId. */
181 mig_jcr->jr.PoolId = jcr->jr.PoolId;
182 mig_jcr->jr.FileSetId = jcr->jr.FileSetId;
183 mig_jcr->jr.JobId = mig_jcr->JobId;
185 Dmsg4(dbglevel, "mig_jcr: Name=%s JobId=%d Type=%c Level=%c\n",
186 mig_jcr->jr.Name, (int)mig_jcr->jr.JobId,
187 mig_jcr->jr.JobType, mig_jcr->jr.JobLevel);
190 * Get the PoolId used with the original job. Then
191 * find the pool name from the database record.
193 memset(&pr, 0, sizeof(pr));
194 pr.PoolId = mig_jcr->previous_jr.PoolId;
195 if (!db_get_pool_record(jcr, jcr->db, &pr)) {
196 Jmsg(jcr, M_FATAL, 0, _("Pool for JobId %s not in database. ERR=%s\n"),
197 edit_int64(pr.PoolId, ed1), db_strerror(jcr->db));
200 /* Get the pool resource corresponding to the original job */
201 pool = (POOL *)GetResWithName(R_POOL, pr.Name);
203 Jmsg(jcr, M_FATAL, 0, _("Pool resource \"%s\" not found.\n"), pr.Name);
207 /* If pool storage specified, use it for restore */
208 copy_rstorage(mig_jcr, pool->storage, _("Pool resource"));
209 copy_rstorage(jcr, pool->storage, _("Pool resource"));
212 * If the original backup pool has a NextPool, make sure a
213 * record exists in the database. Note, in this case, we
214 * will be migrating from pool to pool->NextPool.
216 if (pool->NextPool) {
217 jcr->jr.PoolId = get_or_create_pool_record(jcr, pool->NextPool->hdr.name);
218 if (jcr->jr.PoolId == 0) {
222 * put the "NextPool" resource pointer in our jcr so that we
223 * can pull the Storage reference from it.
225 mig_jcr->pool = jcr->pool = pool->NextPool;
226 mig_jcr->jr.PoolId = jcr->jr.PoolId;
227 pm_strcpy(jcr->pool_source, _("NextPool in Pool resource"));
229 Jmsg(jcr, M_FATAL, 0, _("No Next Pool specification found in Pool \"%s\".\n"),
234 if (!jcr->pool->storage || jcr->pool->storage->size() == 0) {
235 Jmsg(jcr, M_FATAL, 0, _("No Storage specification found in Next Pool \"%s\".\n"),
236 jcr->pool->hdr.name);
240 /* If pool storage specified, use it instead of job storage for backup */
241 copy_wstorage(jcr, jcr->pool->storage, _("NextPool in Pool resource"));
247 * Do a Migration of a previous job
249 * Returns: false on failure
252 bool do_migration(JCR *jcr)
256 JCR *mig_jcr = jcr->mig_jcr; /* newly migrated job */
262 /* Print Job Start message */
263 Jmsg(jcr, M_INFO, 0, _("Start Migration JobId %s, Job=%s\n"),
264 edit_uint64(jcr->JobId, ed1), jcr->Job);
266 set_jcr_job_status(jcr, JS_Running);
267 set_jcr_job_status(mig_jcr, JS_Running);
268 Dmsg2(dbglevel, "JobId=%d JobLevel=%c\n", (int)jcr->jr.JobId, jcr->jr.JobLevel);
270 /* Update job start record for this migration control job */
271 if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) {
272 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db));
276 Dmsg4(dbglevel, "mig_jcr: Name=%s JobId=%d Type=%c Level=%c\n",
277 mig_jcr->jr.Name, (int)mig_jcr->jr.JobId,
278 mig_jcr->jr.JobType, mig_jcr->jr.JobLevel);
280 /* Update job start record for the real migration backup job */
281 if (!db_update_job_start_record(mig_jcr, mig_jcr->db, &mig_jcr->jr)) {
282 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(mig_jcr->db));
288 * Open a message channel connection with the Storage
289 * daemon. This is to let him know that our client
290 * will be contacting him for a backup session.
293 Dmsg0(110, "Open connection with storage daemon\n");
294 set_jcr_job_status(jcr, JS_WaitSD);
295 set_jcr_job_status(mig_jcr, JS_WaitSD);
297 * Start conversation with Storage daemon
299 if (!connect_to_storage_daemon(jcr, 10, SDConnectTimeout, 1)) {
302 sd = jcr->store_bsock;
304 * Now start a job with the Storage daemon
306 Dmsg2(dbglevel, "Read store=%s, write store=%s\n",
307 ((STORE *)jcr->rstorage->first())->name(),
308 ((STORE *)jcr->wstorage->first())->name());
309 if (((STORE *)jcr->rstorage->first())->name() == ((STORE *)jcr->wstorage->first())->name()) {
310 Jmsg(jcr, M_FATAL, 0, _("Read storage \"%s\" same as write storage.\n"),
311 ((STORE *)jcr->rstorage->first())->name());
314 if (!start_storage_daemon_job(jcr, jcr->rstorage, jcr->wstorage)) {
317 Dmsg0(150, "Storage daemon connection OK\n");
319 if (!send_bootstrap_file(jcr, sd) ||
320 !response(jcr, sd, OKbootstrap, "Bootstrap", DISPLAY_ERROR)) {
324 if (!bnet_fsend(sd, "run")) {
329 * Now start a Storage daemon message thread
331 if (!start_storage_daemon_message_thread(jcr)) {
336 set_jcr_job_status(jcr, JS_Running);
337 set_jcr_job_status(mig_jcr, JS_Running);
339 /* Pickup Job termination data */
340 /* Note, the SD stores in jcr->JobFiles/ReadBytes/JobBytes/Errors */
341 wait_for_storage_daemon_termination(jcr);
343 set_jcr_job_status(jcr, jcr->SDJobStatus);
344 if (jcr->JobStatus != JS_Terminated) {
348 migration_cleanup(jcr, jcr->JobStatus);
350 UAContext *ua = new_ua_context(jcr);
351 purge_job_records_from_catalog(ua, jcr->previous_jr.JobId);
362 /* Add an item to the list if it is unique */
363 static void add_unique_id(idpkt *ids, char *item)
368 /* Walk through current list to see if each item is the same as item */
371 for (int i=0; i<(int)sizeof(id); i++) {
374 } else if (*q == ',') {
381 if (strcmp(item, id) == 0) {
385 /* Did not find item, so add it to list */
386 if (ids->count == 0) {
389 pm_strcat(ids->list, ",");
391 pm_strcat(ids->list, item);
393 // Dmsg3(0, "add_uniq count=%d Ids=%p %s\n", ids->count, ids->list, ids->list);
398 * Callback handler make list of DB Ids
400 static int unique_dbid_handler(void *ctx, int num_fields, char **row)
402 idpkt *ids = (idpkt *)ctx;
404 add_unique_id(ids, row[0]);
405 Dmsg3(dbglevel, "dbid_hdlr count=%d Ids=%p %s\n", ids->count, ids->list, ids->list);
415 static int item_compare(void *item1, void *item2)
417 uitem *i1 = (uitem *)item1;
418 uitem *i2 = (uitem *)item2;
419 return strcmp(i1->item, i2->item);
422 static int unique_name_handler(void *ctx, int num_fields, char **row)
424 dlist *list = (dlist *)ctx;
426 uitem *new_item = (uitem *)malloc(sizeof(uitem));
429 memset(new_item, 0, sizeof(uitem));
430 new_item->item = bstrdup(row[0]);
431 Dmsg1(dbglevel, "Unique_name_hdlr Item=%s\n", row[0]);
432 item = (uitem *)list->binary_insert((void *)new_item, item_compare);
433 if (item != new_item) { /* already in list */
434 free(new_item->item);
435 free((char *)new_item);
441 /* Get Job names in Pool */
442 const char *sql_job =
443 "SELECT DISTINCT Job.Name from Job,Pool"
444 " WHERE Pool.Name='%s' AND Job.PoolId=Pool.PoolId";
446 /* Get JobIds from regex'ed Job names */
447 const char *sql_jobids_from_job =
448 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Job,Pool"
449 " WHERE Job.Name='%s' AND Pool.Name='%s' AND Job.PoolId=Pool.PoolId"
450 " ORDER by Job.StartTime";
452 /* Get Client names in Pool */
453 const char *sql_client =
454 "SELECT DISTINCT Client.Name from Client,Pool,Job"
455 " WHERE Pool.Name='%s' AND Job.ClientId=Client.ClientId AND"
456 " Job.PoolId=Pool.PoolId";
458 /* Get JobIds from regex'ed Client names */
459 const char *sql_jobids_from_client =
460 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Job,Pool,Client"
461 " WHERE Client.Name='%s' AND Pool.Name='%s' AND Job.PoolId=Pool.PoolId"
462 " AND Job.ClientId=Client.ClientId "
463 " ORDER by Job.StartTime";
465 /* Get Volume names in Pool */
466 const char *sql_vol =
467 "SELECT DISTINCT VolumeName FROM Media,Pool WHERE"
468 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
469 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'";
471 /* Get JobIds from regex'ed Volume names */
472 const char *sql_jobids_from_vol =
473 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Media,JobMedia,Job"
474 " WHERE Media.VolumeName='%s' AND Media.MediaId=JobMedia.MediaId"
475 " AND JobMedia.JobId=Job.JobId"
476 " ORDER by Job.StartTime";
479 const char *sql_smallest_vol =
480 "SELECT MediaId FROM Media,Pool WHERE"
481 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
482 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'"
483 " ORDER BY VolBytes ASC LIMIT 1";
485 const char *sql_oldest_vol =
486 "SELECT MediaId FROM Media,Pool WHERE"
487 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
488 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'"
489 " ORDER BY LastWritten ASC LIMIT 1";
491 /* Get JobIds when we have selected MediaId */
492 const char *sql_jobids_from_mediaid =
493 "SELECT DISTINCT Job.JobId,Job.StartTime FROM JobMedia,Job"
494 " WHERE JobMedia.JobId=Job.JobId AND JobMedia.MediaId=%s"
495 " ORDER by Job.StartTime";
497 /* Get tne number of bytes in the pool */
498 const char *sql_pool_bytes =
499 "SELECT SUM(VolBytes) FROM Media,Pool WHERE"
500 " VolStatus in ('Full','Used','Error','Append') AND Media.Enabled=1 AND"
501 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'";
503 /* Get tne number of bytes in the Jobs */
504 const char *sql_job_bytes =
505 "SELECT SUM(JobBytes) FROM Job WHERE JobId IN (%s)";
508 /* Get Media Ids in Pool */
509 const char *sql_mediaids =
510 "SELECT MediaId FROM Media,Pool WHERE"
511 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
512 " Media.PoolId=Pool.PoolId AND Pool.Name='%s' ORDER BY LastWritten ASC";
514 /* Get JobIds in Pool longer than specified time */
515 const char *sql_pool_time =
516 "SELECT DISTINCT Job.JobId from Pool,Job,Media,JobMedia WHERE"
517 " Pool.Name='%s' AND Media.PoolId=Pool.PoolId AND"
518 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
519 " JobMedia.JobId=Job.JobId AND Job.PoolId=Media.PoolId"
520 " AND Job.RealEndTime<='%s'";
523 * const char *sql_ujobid =
524 * "SELECT DISTINCT Job.Job from Client,Pool,Media,Job,JobMedia "
525 * " WHERE Media.PoolId=Pool.PoolId AND Pool.Name='%s' AND"
526 * " JobMedia.JobId=Job.JobId AND Job.PoolId=Media.PoolId";
533 * This is the central piece of code that finds a job or jobs
534 * actually JobIds to migrate. It first looks to see if one
535 * has been "manually" specified in jcr->MigrateJobId, and if
536 * so, it returns that JobId to be run. Otherwise, it
537 * examines the Selection Type to see what kind of migration
538 * we are doing (Volume, Job, Client, ...) and applies any
539 * Selection Pattern if appropriate to obtain a list of JobIds.
540 * Finally, it will loop over all the JobIds found, except the last
541 * one starting a new job with MigrationJobId set to that JobId, and
542 * finally, it returns the last JobId to the caller.
544 * Returns: false on error
545 * true if OK and jcr->previous_jr filled in
547 static bool get_job_to_migrate(JCR *jcr)
550 POOL_MEM query(PM_MESSAGE);
555 idpkt ids, mid, jids;
561 char dt[MAX_TIME_LENGTH];
563 ids.list = get_pool_memory(PM_MESSAGE);
566 mid.list = get_pool_memory(PM_MESSAGE);
569 jids.list = get_pool_memory(PM_MESSAGE);
575 * If MigrateJobId is set, then we migrate only that Job,
576 * otherwise, we go through the full selection of jobs to
579 if (jcr->MigrateJobId != 0) {
580 Dmsg1(dbglevel, "At Job start previous jobid=%u\n", jcr->MigrateJobId);
581 edit_uint64(jcr->MigrateJobId, ids.list);
584 switch (jcr->job->selection_type) {
586 if (!regex_find_jobids(jcr, &ids, sql_job, sql_jobids_from_job, "Job")) {
591 if (!regex_find_jobids(jcr, &ids, sql_client, sql_jobids_from_client, "Client")) {
596 if (!regex_find_jobids(jcr, &ids, sql_vol, sql_jobids_from_vol, "Volume")) {
601 if (!jcr->job->selection_pattern) {
602 Jmsg(jcr, M_FATAL, 0, _("No Migration SQL selection pattern specified.\n"));
605 Dmsg1(dbglevel, "SQL=%s\n", jcr->job->selection_pattern);
606 if (!db_sql_query(jcr->db, jcr->job->selection_pattern,
607 unique_dbid_handler, (void *)&ids)) {
608 Jmsg(jcr, M_FATAL, 0,
609 _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
613 case MT_SMALLEST_VOL:
614 if (!find_mediaid_then_jobids(jcr, &ids, sql_smallest_vol, "Smallest Volume")) {
619 if (!find_mediaid_then_jobids(jcr, &ids, sql_oldest_vol, "Oldest Volume")) {
624 case MT_POOL_OCCUPANCY:
626 /* Find count of bytes in pool */
627 Mmsg(query, sql_pool_bytes, jcr->pool->hdr.name);
628 if (!db_sql_query(jcr->db, query.c_str(), db_int64_handler, (void *)&ctx)) {
629 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
632 if (ctx.count == 0) {
633 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
636 pool_bytes = ctx.value;
637 Dmsg2(dbglevel, "highbytes=%d pool=%d\n", (int)jcr->pool->MigrationHighBytes,
639 if (pool_bytes < (int64_t)jcr->pool->MigrationHighBytes) {
640 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
643 Dmsg0(dbglevel, "We should do Occupation migration.\n");
646 /* Find a list of MediaIds that could be migrated */
647 Mmsg(query, sql_mediaids, jcr->pool->hdr.name);
648 Dmsg1(dbglevel, "query=%s\n", query.c_str());
649 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)&ids)) {
650 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
653 if (ids.count == 0) {
654 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
657 Dmsg2(dbglevel, "Pool Occupancy ids=%d MediaIds=%s\n", ids.count, ids.list);
660 * Now loop over MediaIds getting more JobIds to migrate until
661 * we reduce the pool occupancy below the low water mark.
664 for (int i=0; i < (int)ids.count; i++) {
665 stat = get_next_dbid_from_list(&p, &MediaId);
666 Dmsg2(dbglevel, "get_next_dbid stat=%d MediaId=%u\n", stat, MediaId);
668 Jmsg(jcr, M_FATAL, 0, _("Invalid MediaId found.\n"));
670 } else if (stat == 0) {
674 Mmsg(mid.list, "%s", edit_int64(MediaId, ed1));
675 ok = find_jobids_from_mediaid_list(jcr, &mid, "Volumes");
680 pm_strcat(jids.list, ",");
682 pm_strcat(jids.list, mid.list);
683 jids.count += mid.count;
685 /* Now get the count of bytes added */
687 /* Find count of bytes from Jobs */
688 Mmsg(query, sql_job_bytes, mid.list);
689 if (!db_sql_query(jcr->db, query.c_str(), db_int64_handler, (void *)&ctx)) {
690 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
693 pool_bytes -= ctx.value;
694 Dmsg1(dbglevel, "Job bytes=%d\n", (int)ctx.value);
695 Dmsg2(dbglevel, "lowbytes=%d pool=%d\n", (int)jcr->pool->MigrationLowBytes,
697 if (pool_bytes <= (int64_t)jcr->pool->MigrationLowBytes) {
698 Dmsg0(dbglevel, "We should be done.\n");
703 Dmsg2(dbglevel, "Pool Occupancy ids=%d JobIds=%s\n", jids.count, jids.list);
708 ttime = time(NULL) - (time_t)jcr->pool->MigrationTime;
709 (void)localtime_r(&ttime, &tm);
710 strftime(dt, sizeof(dt), "%Y-%m-%d %H:%M:%S", &tm);
713 Mmsg(query, sql_pool_time, jcr->pool->hdr.name, dt);
714 Dmsg1(dbglevel, "query=%s\n", query.c_str());
715 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)&ids)) {
716 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
719 if (ids.count == 0) {
720 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
723 Dmsg2(dbglevel, "PoolTime ids=%d JobIds=%s\n", ids.count, ids.list);
727 Jmsg(jcr, M_FATAL, 0, _("Unknown Migration Selection Type.\n"));
733 * Loop over all jobids except the last one, sending
734 * them to start_migration_job(), which will start a job
735 * for each of them. For the last JobId, we handle it below.
738 if (ids.count == 0) {
739 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
742 Jmsg(jcr, M_INFO, 0, _("The following %u JobId%s will be migrated: %s\n"),
743 ids.count, ids.count==0?"":"s", ids.list);
744 Dmsg2(dbglevel, "Before loop count=%d ids=%s\n", ids.count, ids.list);
745 for (int i=1; i < (int)ids.count; i++) {
747 stat = get_next_jobid_from_list(&p, &JobId);
748 Dmsg3(dbglevel, "get_jobid_no=%d stat=%d JobId=%u\n", i, stat, JobId);
749 jcr->MigrateJobId = JobId;
750 start_migration_job(jcr);
751 Dmsg0(dbglevel, "Back from start_migration_job\n");
753 Jmsg(jcr, M_FATAL, 0, _("Invalid JobId found.\n"));
755 } else if (stat == 0) {
756 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
761 /* Now get the last JobId and handle it in the current job */
763 stat = get_next_jobid_from_list(&p, &JobId);
764 Dmsg2(dbglevel, "Last get_next_jobid stat=%d JobId=%u\n", stat, (int)JobId);
766 Jmsg(jcr, M_FATAL, 0, _("Invalid JobId found.\n"));
768 } else if (stat == 0) {
769 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
773 jcr->previous_jr.JobId = JobId;
774 Dmsg1(dbglevel, "Previous jobid=%d\n", (int)jcr->previous_jr.JobId);
776 if (!db_get_job_record(jcr, jcr->db, &jcr->previous_jr)) {
777 Jmsg(jcr, M_FATAL, 0, _("Could not get job record for JobId %s to migrate. ERR=%s"),
778 edit_int64(jcr->previous_jr.JobId, ed1),
779 db_strerror(jcr->db));
782 Jmsg(jcr, M_INFO, 0, _("Migration using JobId=%s Job=%s\n"),
783 edit_int64(jcr->previous_jr.JobId, ed1), jcr->previous_jr.Job);
784 Dmsg3(dbglevel, "Migration JobId=%d using JobId=%s Job=%s\n",
786 edit_int64(jcr->previous_jr.JobId, ed1), jcr->previous_jr.Job);
796 free_pool_memory(ids.list);
797 free_pool_memory(mid.list);
798 free_pool_memory(jids.list);
802 static void start_migration_job(JCR *jcr)
804 UAContext *ua = new_ua_context(jcr);
807 Mmsg(ua->cmd, "run %s jobid=%s", jcr->job->hdr.name,
808 edit_uint64(jcr->MigrateJobId, ed1));
809 Dmsg1(dbglevel, "=============== Migration cmd=%s\n", ua->cmd);
810 parse_ua_args(ua); /* parse command */
811 int stat = run_cmd(ua, ua->cmd);
813 Jmsg(jcr, M_ERROR, 0, _("Could not start migration job.\n"));
815 Jmsg(jcr, M_INFO, 0, _("Migration JobId %d started.\n"), stat);
820 static bool find_mediaid_then_jobids(JCR *jcr, idpkt *ids, const char *query1,
824 POOL_MEM query(PM_MESSAGE);
827 /* Basic query for MediaId */
828 Mmsg(query, query1, jcr->pool->hdr.name);
829 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
830 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
833 if (ids->count == 0) {
834 Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type);
836 if (ids->count != 1) {
837 Jmsg(jcr, M_FATAL, 0, _("SQL logic error. Count should be 1 but is %d\n"),
841 Dmsg1(dbglevel, "Smallest Vol Jobids=%s\n", ids->list);
843 ok = find_jobids_from_mediaid_list(jcr, ids, type);
849 static bool find_jobids_from_mediaid_list(JCR *jcr, idpkt *ids, const char *type)
852 POOL_MEM query(PM_MESSAGE);
854 Mmsg(query, sql_jobids_from_mediaid, ids->list);
856 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
857 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
860 if (ids->count == 0) {
861 Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type);
868 static bool regex_find_jobids(JCR *jcr, idpkt *ids, const char *query1,
869 const char *query2, const char *type)
873 uitem *last_item = NULL;
878 POOL_MEM query(PM_MESSAGE);
880 item_chain = New(dlist(item, &item->link));
881 if (!jcr->job->selection_pattern) {
882 Jmsg(jcr, M_FATAL, 0, _("No Migration %s selection pattern specified.\n"),
886 Dmsg1(dbglevel, "regex=%s\n", jcr->job->selection_pattern);
887 /* Compile regex expression */
888 rc = regcomp(&preg, jcr->job->selection_pattern, REG_EXTENDED);
890 regerror(rc, &preg, prbuf, sizeof(prbuf));
891 Jmsg(jcr, M_FATAL, 0, _("Could not compile regex pattern \"%s\" ERR=%s\n"),
892 jcr->job->selection_pattern, prbuf);
895 /* Basic query for names */
896 Mmsg(query, query1, jcr->pool->hdr.name);
897 Dmsg1(dbglevel, "get name query1=%s\n", query.c_str());
898 if (!db_sql_query(jcr->db, query.c_str(), unique_name_handler,
899 (void *)item_chain)) {
900 Jmsg(jcr, M_FATAL, 0,
901 _("SQL to get %s failed. ERR=%s\n"), type, db_strerror(jcr->db));
904 /* Now apply the regex to the names and remove any item not matched */
905 foreach_dlist(item, item_chain) {
906 const int nmatch = 30;
907 regmatch_t pmatch[nmatch];
909 Dmsg1(dbglevel, "Remove item %s\n", last_item->item);
910 free(last_item->item);
911 item_chain->remove(last_item);
913 Dmsg1(dbglevel, "get name Item=%s\n", item->item);
914 rc = regexec(&preg, item->item, nmatch, pmatch, 0);
916 last_item = NULL; /* keep this one */
922 free(last_item->item);
923 Dmsg1(dbglevel, "Remove item %s\n", last_item->item);
924 item_chain->remove(last_item);
928 * At this point, we have a list of items in item_chain
929 * that have been matched by the regex, so now we need
930 * to look up their jobids.
933 foreach_dlist(item, item_chain) {
934 Dmsg2(dbglevel, "Got %s: %s\n", type, item->item);
935 Mmsg(query, query2, item->item, jcr->pool->hdr.name);
936 Dmsg1(dbglevel, "get id from name query2=%s\n", query.c_str());
937 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
938 Jmsg(jcr, M_FATAL, 0,
939 _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
943 if (ids->count == 0) {
944 Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type);
948 Dmsg2(dbglevel, "Count=%d Jobids=%s\n", ids->count, ids->list);
950 Dmsg0(dbglevel, "After delete item_chain\n");
956 * Release resources allocated during backup.
958 void migration_cleanup(JCR *jcr, int TermCode)
960 char sdt[MAX_TIME_LENGTH], edt[MAX_TIME_LENGTH];
961 char ec1[30], ec2[30], ec3[30], ec4[30], ec5[30], elapsed[50];
962 char ec6[50], ec7[50], ec8[50];
963 char term_code[100], sd_term_msg[100];
964 const char *term_msg;
965 int msg_type = M_INFO;
969 JCR *mig_jcr = jcr->mig_jcr;
970 POOL_MEM query(PM_MESSAGE);
972 Dmsg2(100, "Enter migrate_cleanup %d %c\n", TermCode, TermCode);
973 dequeue_messages(jcr); /* display any queued messages */
974 memset(&mr, 0, sizeof(mr));
975 set_jcr_job_status(jcr, TermCode);
976 update_job_end_record(jcr); /* update database */
979 * Check if we actually did something.
980 * mig_jcr is jcr of the newly migrated job.
983 mig_jcr->JobFiles = jcr->JobFiles = jcr->SDJobFiles;
984 mig_jcr->JobBytes = jcr->JobBytes = jcr->SDJobBytes;
985 mig_jcr->VolSessionId = jcr->VolSessionId;
986 mig_jcr->VolSessionTime = jcr->VolSessionTime;
987 mig_jcr->jr.RealEndTime = 0;
988 mig_jcr->jr.PriorJobId = jcr->previous_jr.JobId;
990 set_jcr_job_status(mig_jcr, TermCode);
991 update_job_end_record(mig_jcr);
993 /* Update final items to set them to the previous job's values */
994 Mmsg(query, "UPDATE Job SET StartTime='%s',EndTime='%s',"
995 "JobTDate=%s WHERE JobId=%s",
996 jcr->previous_jr.cStartTime, jcr->previous_jr.cEndTime,
997 edit_uint64(jcr->previous_jr.JobTDate, ec1),
998 edit_uint64(mig_jcr->jr.JobId, ec2));
999 db_sql_query(mig_jcr->db, query.c_str(), NULL, NULL);
1001 /* Now mark the previous job as migrated if it terminated normally */
1002 if (jcr->JobStatus == JS_Terminated) {
1003 Mmsg(query, "UPDATE Job SET Type='%c' WHERE JobId=%s",
1004 (char)JT_MIGRATED_JOB, edit_uint64(jcr->previous_jr.JobId, ec1));
1005 db_sql_query(mig_jcr->db, query.c_str(), NULL, NULL);
1008 if (!db_get_job_record(jcr, jcr->db, &jcr->jr)) {
1009 Jmsg(jcr, M_WARNING, 0, _("Error getting job record for stats: %s"),
1010 db_strerror(jcr->db));
1011 set_jcr_job_status(jcr, JS_ErrorTerminated);
1014 bstrncpy(mr.VolumeName, jcr->VolumeName, sizeof(mr.VolumeName));
1015 if (!db_get_media_record(jcr, jcr->db, &mr)) {
1016 Jmsg(jcr, M_WARNING, 0, _("Error getting Media record for Volume \"%s\": ERR=%s"),
1017 mr.VolumeName, db_strerror(jcr->db));
1018 set_jcr_job_status(jcr, JS_ErrorTerminated);
1021 update_bootstrap_file(mig_jcr);
1023 if (!db_get_job_volume_names(mig_jcr, mig_jcr->db, mig_jcr->jr.JobId, &mig_jcr->VolumeName)) {
1025 * Note, if the job has failed, most likely it did not write any
1026 * tape, so suppress this "error" message since in that case
1027 * it is normal. Or look at it the other way, only for a
1028 * normal exit should we complain about this error.
1030 if (jcr->JobStatus == JS_Terminated && jcr->jr.JobBytes) {
1031 Jmsg(jcr, M_ERROR, 0, "%s", db_strerror(mig_jcr->db));
1033 mig_jcr->VolumeName[0] = 0; /* none */
1035 switch (jcr->JobStatus) {
1037 if (jcr->Errors || jcr->SDErrors) {
1038 term_msg = _("%s OK -- with warnings");
1040 term_msg = _("%s OK");
1044 case JS_ErrorTerminated:
1045 term_msg = _("*** %s Error ***");
1046 msg_type = M_ERROR; /* Generate error message */
1047 if (jcr->store_bsock) {
1048 bnet_sig(jcr->store_bsock, BNET_TERMINATE);
1049 if (jcr->SD_msg_chan) {
1050 pthread_cancel(jcr->SD_msg_chan);
1055 term_msg = _("%s Canceled");
1056 if (jcr->store_bsock) {
1057 bnet_sig(jcr->store_bsock, BNET_TERMINATE);
1058 if (jcr->SD_msg_chan) {
1059 pthread_cancel(jcr->SD_msg_chan);
1064 term_msg = _("Inappropriate %s term code");
1068 msg_type = M_ERROR; /* Generate error message */
1069 term_msg = _("*** %s Error ***");
1072 bsnprintf(term_code, sizeof(term_code), term_msg, "Migration");
1073 bstrftimes(sdt, sizeof(sdt), jcr->jr.StartTime);
1074 bstrftimes(edt, sizeof(edt), jcr->jr.EndTime);
1075 RunTime = jcr->jr.EndTime - jcr->jr.StartTime;
1079 kbps = (double)jcr->SDJobBytes / (1000 * RunTime);
1083 jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg));
1085 Jmsg(jcr, msg_type, 0, _("Bacula %s (%s): %s\n"
1086 " Prev Backup JobId: %s\n"
1087 " New Backup JobId: %s\n"
1088 " Migration JobId: %s\n"
1089 " Migration Job: %s\n"
1090 " Backup Level: %s%s\n"
1092 " FileSet: \"%s\" %s\n"
1093 " Pool: \"%s\" (From %s)\n"
1094 " Read Storage: \"%s\" (From %s)\n"
1095 " Write Storage: \"%s\" (From %s)\n"
1098 " Elapsed time: %s\n"
1100 " SD Files Written: %s\n"
1101 " SD Bytes Written: %s (%sB)\n"
1102 " Rate: %.1f KB/s\n"
1103 " Volume name(s): %s\n"
1104 " Volume Session Id: %d\n"
1105 " Volume Session Time: %d\n"
1106 " Last Volume Bytes: %s (%sB)\n"
1108 " SD termination status: %s\n"
1109 " Termination: %s\n\n"),
1113 edit_uint64(jcr->previous_jr.JobId, ec6),
1114 mig_jcr ? edit_uint64(mig_jcr->jr.JobId, ec7) : "0",
1115 edit_uint64(jcr->jr.JobId, ec8),
1117 level_to_str(jcr->JobLevel), jcr->since,
1118 jcr->client->name(),
1119 jcr->fileset->name(), jcr->FSCreateTime,
1120 jcr->pool->name(), jcr->pool_source,
1121 jcr->rstore?jcr->rstore->name():"*None*",
1122 NPRT(jcr->rstore_source),
1123 jcr->wstore?jcr->wstore->name():"*None*",
1124 NPRT(jcr->wstore_source),
1127 edit_utime(RunTime, elapsed, sizeof(elapsed)),
1129 edit_uint64_with_commas(jcr->SDJobFiles, ec1),
1130 edit_uint64_with_commas(jcr->SDJobBytes, ec2),
1131 edit_uint64_with_suffix(jcr->SDJobBytes, ec3),
1133 mig_jcr ? mig_jcr->VolumeName : "",
1135 jcr->VolSessionTime,
1136 edit_uint64_with_commas(mr.VolBytes, ec4),
1137 edit_uint64_with_suffix(mr.VolBytes, ec5),
1142 Dmsg1(100, "migrate_cleanup() mig_jcr=0x%x\n", jcr->mig_jcr);
1144 free_jcr(jcr->mig_jcr);
1145 jcr->mig_jcr = NULL;
1147 Dmsg0(100, "Leave migrate_cleanup()\n");
1151 * Return next DBId from comma separated list
1154 * 1 if next DBId returned
1155 * 0 if no more DBIds are in list
1156 * -1 there is an error
1158 static int get_next_dbid_from_list(char **p, DBId_t *DBId)
1164 for (int i=0; i<(int)sizeof(id); i++) {
1167 } else if (*q == ',') {
1176 } else if (!is_a_number(id)) {
1177 return -1; /* error */
1180 *DBId = str_to_int64(id);