3 * Bacula Director -- migrate.c -- responsible for doing
6 * Kern Sibbald, September MMIV
8 * Basic tasks done here:
9 * Open DB and create records for this job.
10 * Open Message Channel with Storage daemon to tell him a job will be starting.
11 * Open connection with Storage daemon and pass him commands
13 * When the Storage daemon finishes the job, update the DB.
18 Bacula® - The Network Backup Solution
20 Copyright (C) 2004-2006 Free Software Foundation Europe e.V.
22 The main author of Bacula is Kern Sibbald, with contributions from
23 many others, a complete list can be found in the file AUTHORS.
24 This program is Free Software; you can redistribute it and/or
25 modify it under the terms of version two of the GNU General Public
26 License as published by the Free Software Foundation plus additions
27 that are listed in the file LICENSE.
29 This program is distributed in the hope that it will be useful, but
30 WITHOUT ANY WARRANTY; without even the implied warranty of
31 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
32 General Public License for more details.
34 You should have received a copy of the GNU General Public License
35 along with this program; if not, write to the Free Software
36 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
39 Bacula® is a registered trademark of John Walker.
40 The licensor of Bacula is the Free Software Foundation Europe
41 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
42 Switzerland, email:ftf@fsfeurope.org.
49 #include "lib/bregex.h"
54 static const int dbglevel = 10;
56 static char OKbootstrap[] = "3000 OK bootstrap\n";
57 static bool get_job_to_migrate(JCR *jcr);
59 static bool regex_find_jobids(JCR *jcr, idpkt *ids, const char *query1,
60 const char *query2, const char *type);
61 static bool find_mediaid_then_jobids(JCR *jcr, idpkt *ids, const char *query1,
63 static bool find_jobids_from_mediaid_list(JCR *jcr, idpkt *ids, const char *type);
64 static void start_migration_job(JCR *jcr);
65 static int get_next_dbid_from_list(char **p, DBId_t *DBId);
68 * Called here before the job is run to do the job
69 * specific setup. Note, one of the important things to
70 * complete in this init code is to make the definitive
71 * choice of input and output storage devices. This is
72 * because immediately after the init, the job is queued
73 * in the jobq.c code, and it checks that all the resources
74 * (storage resources in particular) are available, so these
75 * must all be properly defined.
77 * previous_jr refers to the job DB record of the Job that is
78 * going to be migrated.
79 * prev_job refers to the job resource of the Job that is
80 * going to be migrated.
81 * jcr is the jcr for the current "migration" job. It is a
82 * control job that is put in the DB as a migration job, which
83 * means that this job migrated a previous job to a new job.
84 * No Volume or File data is associated with this control
86 * mig_jcr refers to the newly migrated job that is run by
87 * the current jcr. It is a backup job that moves (migrates) the
88 * data written for the previous_jr into the new pool. This
89 * job (mig_jcr) becomes the new backup job that replaces
90 * the original backup job.
92 bool do_migration_init(JCR *jcr)
98 JCR *mig_jcr; /* newly migrated job */
100 /* If we find a job or jobs to migrate it is previous_jr.JobId */
101 if (!get_job_to_migrate(jcr)) {
104 Dmsg1(dbglevel, "Back from get_job_to_migrate JobId=%d\n", (int)jcr->JobId);
106 if (jcr->previous_jr.JobId == 0) {
107 Dmsg1(dbglevel, "JobId=%d no previous JobId\n", (int)jcr->JobId);
108 Jmsg(jcr, M_INFO, 0, _("No previous Job found to migrate.\n"));
109 return true; /* no work */
112 if (!get_or_create_fileset_record(jcr)) {
113 Dmsg1(dbglevel, "JobId=%d no FileSet\n", (int)jcr->JobId);
114 Jmsg(jcr, M_FATAL, 0, _("Could not get or create the FileSet record.\n"));
118 apply_pool_overrides(jcr);
120 jcr->jr.PoolId = get_or_create_pool_record(jcr, jcr->pool->hdr.name);
121 if (jcr->jr.PoolId == 0) {
122 Dmsg1(dbglevel, "JobId=%d no PoolId\n", (int)jcr->JobId);
123 Jmsg(jcr, M_FATAL, 0, _("Could not get or create a Pool record.\n"));
127 /* If pool storage specified, use it instead of job storage */
128 copy_wstorage(jcr, jcr->pool->storage, _("Pool resource"));
130 if (jcr->wstorage->size() == 0) {
131 Jmsg(jcr, M_FATAL, 0, _("No Storage specification found in Job or Pool.\n"));
135 create_restore_bootstrap_file(jcr);
137 if (jcr->previous_jr.JobId == 0 || jcr->ExpectedFiles == 0) {
138 set_jcr_job_status(jcr, JS_Terminated);
139 Dmsg1(dbglevel, "JobId=%d expected files == 0\n", (int)jcr->JobId);
140 if (jcr->previous_jr.JobId == 0) {
141 Jmsg(jcr, M_INFO, 0, _("No previous Job found to migrate.\n"));
143 Jmsg(jcr, M_INFO, 0, _("Previous Job has no data to migrate.\n"));
145 return true; /* no work */
148 Dmsg5(dbglevel, "JobId=%d: Previous: Name=%s JobId=%d Type=%c Level=%c\n",
150 jcr->previous_jr.Name, (int)jcr->previous_jr.JobId,
151 jcr->previous_jr.JobType, jcr->previous_jr.JobLevel);
153 Dmsg5(dbglevel, "JobId=%d: Current: Name=%s JobId=%d Type=%c Level=%c\n",
155 jcr->jr.Name, (int)jcr->jr.JobId,
156 jcr->jr.JobType, jcr->jr.JobLevel);
159 job = (JOB *)GetResWithName(R_JOB, jcr->jr.Name);
160 prev_job = (JOB *)GetResWithName(R_JOB, jcr->previous_jr.Name);
162 if (!job || !prev_job) {
166 /* Create a migation jcr */
167 mig_jcr = jcr->mig_jcr = new_jcr(sizeof(JCR), dird_free_jcr);
168 memcpy(&mig_jcr->previous_jr, &jcr->previous_jr, sizeof(mig_jcr->previous_jr));
171 * Turn the mig_jcr into a "real" job that takes on the aspects of
172 * the previous backup job "prev_job".
174 set_jcr_defaults(mig_jcr, prev_job);
175 if (!setup_job(mig_jcr)) {
179 /* Now reset the job record from the previous job */
180 memcpy(&mig_jcr->jr, &jcr->previous_jr, sizeof(mig_jcr->jr));
181 /* Update the jr to reflect the new values of PoolId, FileSetId, and JobId. */
182 mig_jcr->jr.PoolId = jcr->jr.PoolId;
183 mig_jcr->jr.FileSetId = jcr->jr.FileSetId;
184 mig_jcr->jr.JobId = mig_jcr->JobId;
186 Dmsg4(dbglevel, "mig_jcr: Name=%s JobId=%d Type=%c Level=%c\n",
187 mig_jcr->jr.Name, (int)mig_jcr->jr.JobId,
188 mig_jcr->jr.JobType, mig_jcr->jr.JobLevel);
191 * Get the PoolId used with the original job. Then
192 * find the pool name from the database record.
194 memset(&pr, 0, sizeof(pr));
195 pr.PoolId = mig_jcr->previous_jr.PoolId;
196 if (!db_get_pool_record(jcr, jcr->db, &pr)) {
197 Jmsg(jcr, M_FATAL, 0, _("Pool for JobId %s not in database. ERR=%s\n"),
198 edit_int64(pr.PoolId, ed1), db_strerror(jcr->db));
201 /* Get the pool resource corresponding to the original job */
202 pool = (POOL *)GetResWithName(R_POOL, pr.Name);
204 Jmsg(jcr, M_FATAL, 0, _("Pool resource \"%s\" not found.\n"), pr.Name);
208 /* If pool storage specified, use it for restore */
209 copy_rstorage(mig_jcr, pool->storage, _("Pool resource"));
210 copy_rstorage(jcr, pool->storage, _("Pool resource"));
213 * If the original backup pool has a NextPool, make sure a
214 * record exists in the database. Note, in this case, we
215 * will be migrating from pool to pool->NextPool.
217 if (pool->NextPool) {
218 jcr->jr.PoolId = get_or_create_pool_record(jcr, pool->NextPool->hdr.name);
219 if (jcr->jr.PoolId == 0) {
223 * put the "NextPool" resource pointer in our jcr so that we
224 * can pull the Storage reference from it.
226 mig_jcr->pool = jcr->pool = pool->NextPool;
227 mig_jcr->jr.PoolId = jcr->jr.PoolId;
228 pm_strcpy(jcr->pool_source, _("NextPool in Pool resource"));
230 Jmsg(jcr, M_FATAL, 0, _("No Next Pool specification found in Pool \"%s\".\n"),
235 if (!jcr->pool->storage) {
236 Jmsg(jcr, M_FATAL, 0, _("No Storage specification found in Next Pool \"%s\".\n"),
237 jcr->pool->hdr.name);
241 /* If pool storage specified, use it instead of job storage for backup */
242 copy_wstorage(jcr, jcr->pool->storage, _("Next pool resource"));
249 * Do a Migration of a previous job
251 * Returns: false on failure
254 bool do_migration(JCR *jcr)
258 JCR *mig_jcr = jcr->mig_jcr; /* newly migrated job */
261 /* Print Job Start message */
262 Jmsg(jcr, M_INFO, 0, _("Start Migration JobId %s, Job=%s\n"),
263 edit_uint64(jcr->JobId, ed1), jcr->Job);
265 set_jcr_job_status(jcr, JS_Running);
266 set_jcr_job_status(mig_jcr, JS_Running);
267 Dmsg2(dbglevel, "JobId=%d JobLevel=%c\n", (int)jcr->jr.JobId, jcr->jr.JobLevel);
269 /* Update job start record for this migration control job */
270 if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) {
271 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db));
275 Dmsg4(dbglevel, "mig_jcr: Name=%s JobId=%d Type=%c Level=%c\n",
276 mig_jcr->jr.Name, (int)mig_jcr->jr.JobId,
277 mig_jcr->jr.JobType, mig_jcr->jr.JobLevel);
279 /* Update job start record for the real migration backup job */
280 if (!db_update_job_start_record(mig_jcr, mig_jcr->db, &mig_jcr->jr)) {
281 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(mig_jcr->db));
287 * Open a message channel connection with the Storage
288 * daemon. This is to let him know that our client
289 * will be contacting him for a backup session.
292 Dmsg0(110, "Open connection with storage daemon\n");
293 set_jcr_job_status(jcr, JS_WaitSD);
294 set_jcr_job_status(mig_jcr, JS_WaitSD);
296 * Start conversation with Storage daemon
298 if (!connect_to_storage_daemon(jcr, 10, SDConnectTimeout, 1)) {
301 sd = jcr->store_bsock;
303 * Now start a job with the Storage daemon
305 Dmsg2(dbglevel, "Read store=%s, write store=%s\n",
306 ((STORE *)jcr->rstorage->first())->name(),
307 ((STORE *)jcr->wstorage->first())->name());
308 if (((STORE *)jcr->rstorage->first())->name() == ((STORE *)jcr->wstorage->first())->name()) {
309 Jmsg(jcr, M_FATAL, 0, _("Read storage \"%s\" same as write storage.\n"),
310 ((STORE *)jcr->rstorage->first())->name());
313 if (!start_storage_daemon_job(jcr, jcr->rstorage, jcr->wstorage)) {
316 Dmsg0(150, "Storage daemon connection OK\n");
318 if (!send_bootstrap_file(jcr, sd) ||
319 !response(jcr, sd, OKbootstrap, "Bootstrap", DISPLAY_ERROR)) {
323 if (!bnet_fsend(sd, "run")) {
328 * Now start a Storage daemon message thread
330 if (!start_storage_daemon_message_thread(jcr)) {
335 set_jcr_job_status(jcr, JS_Running);
336 set_jcr_job_status(mig_jcr, JS_Running);
338 /* Pickup Job termination data */
339 /* Note, the SD stores in jcr->JobFiles/ReadBytes/JobBytes/Errors */
340 wait_for_storage_daemon_termination(jcr);
342 set_jcr_job_status(jcr, jcr->SDJobStatus);
343 if (jcr->JobStatus != JS_Terminated) {
346 migration_cleanup(jcr, jcr->JobStatus);
348 UAContext *ua = new_ua_context(jcr);
349 purge_files_from_job(ua, jcr->previous_jr.JobId);
360 /* Add an item to the list if it is unique */
361 static void add_unique_id(idpkt *ids, char *item)
366 /* Walk through current list to see if each item is the same as item */
369 for (int i=0; i<(int)sizeof(id); i++) {
372 } else if (*q == ',') {
379 if (strcmp(item, id) == 0) {
383 /* Did not find item, so add it to list */
384 if (ids->count == 0) {
387 pm_strcat(ids->list, ",");
389 pm_strcat(ids->list, item);
391 // Dmsg3(0, "add_uniq count=%d Ids=%p %s\n", ids->count, ids->list, ids->list);
396 * Callback handler make list of DB Ids
398 static int unique_dbid_handler(void *ctx, int num_fields, char **row)
400 idpkt *ids = (idpkt *)ctx;
402 add_unique_id(ids, row[0]);
403 Dmsg3(dbglevel, "dbid_hdlr count=%d Ids=%p %s\n", ids->count, ids->list, ids->list);
413 static int item_compare(void *item1, void *item2)
415 uitem *i1 = (uitem *)item1;
416 uitem *i2 = (uitem *)item2;
417 return strcmp(i1->item, i2->item);
420 static int unique_name_handler(void *ctx, int num_fields, char **row)
422 dlist *list = (dlist *)ctx;
424 uitem *new_item = (uitem *)malloc(sizeof(uitem));
427 memset(new_item, 0, sizeof(uitem));
428 new_item->item = bstrdup(row[0]);
429 Dmsg1(dbglevel, "Unique_name_hdlr Item=%s\n", row[0]);
430 item = (uitem *)list->binary_insert((void *)new_item, item_compare);
431 if (item != new_item) { /* already in list */
432 free(new_item->item);
433 free((char *)new_item);
439 /* Get Job names in Pool */
440 const char *sql_job =
441 "SELECT DISTINCT Job.Name from Job,Pool"
442 " WHERE Pool.Name='%s' AND Job.PoolId=Pool.PoolId";
444 /* Get JobIds from regex'ed Job names */
445 const char *sql_jobids_from_job =
446 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Job,Pool"
447 " WHERE Job.Name='%s' AND Pool.Name='%s' AND Job.PoolId=Pool.PoolId"
448 " ORDER by Job.StartTime";
450 /* Get Client names in Pool */
451 const char *sql_client =
452 "SELECT DISTINCT Client.Name from Client,Pool,Job"
453 " WHERE Pool.Name='%s' AND Job.ClientId=Client.ClientId AND"
454 " Job.PoolId=Pool.PoolId";
456 /* Get JobIds from regex'ed Client names */
457 const char *sql_jobids_from_client =
458 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Job,Pool,Client"
459 " WHERE Client.Name='%s' AND Pool.Name='%s' AND Job.PoolId=Pool.PoolId"
460 " AND Job.ClientId=Client.ClientId "
461 " ORDER by Job.StartTime";
463 /* Get Volume names in Pool */
464 const char *sql_vol =
465 "SELECT DISTINCT VolumeName FROM Media,Pool WHERE"
466 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
467 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'";
469 /* Get JobIds from regex'ed Volume names */
470 const char *sql_jobids_from_vol =
471 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Media,JobMedia,Job"
472 " WHERE Media.VolumeName='%s' AND Media.MediaId=JobMedia.MediaId"
473 " AND JobMedia.JobId=Job.JobId"
474 " ORDER by Job.StartTime";
477 const char *sql_smallest_vol =
478 "SELECT MediaId FROM Media,Pool WHERE"
479 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
480 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'"
481 " ORDER BY VolBytes ASC LIMIT 1";
483 const char *sql_oldest_vol =
484 "SELECT MediaId FROM Media,Pool WHERE"
485 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
486 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'"
487 " ORDER BY LastWritten ASC LIMIT 1";
489 /* Get JobIds when we have selected MediaId */
490 const char *sql_jobids_from_mediaid =
491 "SELECT DISTINCT Job.JobId,Job.StartTime FROM JobMedia,Job"
492 " WHERE JobMedia.JobId=Job.JobId AND JobMedia.MediaId=%s"
493 " ORDER by Job.StartTime";
495 /* Get tne number of bytes in the pool */
496 const char *sql_pool_bytes =
497 "SELECT SUM(VolBytes) FROM Media,Pool WHERE"
498 " VolStatus in ('Full','Used','Error','Append') AND Media.Enabled=1 AND"
499 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'";
501 /* Get tne number of bytes in the Jobs */
502 const char *sql_job_bytes =
503 "SELECT SUM(JobBytes) FROM Job WHERE JobId IN (%s)";
506 /* Get Media Ids in Pool */
507 const char *sql_mediaids =
508 "SELECT MediaId FROM Media,Pool WHERE"
509 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
510 " Media.PoolId=Pool.PoolId AND Pool.Name='%s' ORDER BY LastWritten ASC";
512 /* Get JobIds in Pool longer than specified time */
513 const char *sql_pool_time =
514 "SELECT DISTINCT Job.JobId from Pool,Job,Media,JobMedia WHERE"
515 " Pool.Name='%s' AND Media.PoolId=Pool.PoolId AND"
516 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
517 " JobMedia.JobId=Job.JobId AND Job.PoolId=Media.PoolId"
518 " AND Job.RealEndTime<='%s'";
521 * const char *sql_ujobid =
522 * "SELECT DISTINCT Job.Job from Client,Pool,Media,Job,JobMedia "
523 * " WHERE Media.PoolId=Pool.PoolId AND Pool.Name='%s' AND"
524 * " JobMedia.JobId=Job.JobId AND Job.PoolId=Media.PoolId";
531 * This is the central piece of code that finds a job or jobs
532 * actually JobIds to migrate. It first looks to see if one
533 * has been "manually" specified in jcr->MigrateJobId, and if
534 * so, it returns that JobId to be run. Otherwise, it
535 * examines the Selection Type to see what kind of migration
536 * we are doing (Volume, Job, Client, ...) and applies any
537 * Selection Pattern if appropriate to obtain a list of JobIds.
538 * Finally, it will loop over all the JobIds found, except the last
539 * one starting a new job with MigrationJobId set to that JobId, and
540 * finally, it returns the last JobId to the caller.
542 * Returns: false on error
543 * true if OK and jcr->previous_jr filled in
545 static bool get_job_to_migrate(JCR *jcr)
548 POOL_MEM query(PM_MESSAGE);
553 idpkt ids, mid, jids;
559 char dt[MAX_TIME_LENGTH];
561 ids.list = get_pool_memory(PM_MESSAGE);
564 mid.list = get_pool_memory(PM_MESSAGE);
567 jids.list = get_pool_memory(PM_MESSAGE);
573 * If MigrateJobId is set, then we migrate only that Job,
574 * otherwise, we go through the full selection of jobs to
577 if (jcr->MigrateJobId != 0) {
578 Dmsg1(dbglevel, "At Job start previous jobid=%u\n", jcr->MigrateJobId);
579 edit_uint64(jcr->MigrateJobId, ids.list);
582 switch (jcr->job->selection_type) {
584 if (!regex_find_jobids(jcr, &ids, sql_job, sql_jobids_from_job, "Job")) {
589 if (!regex_find_jobids(jcr, &ids, sql_client, sql_jobids_from_client, "Client")) {
594 if (!regex_find_jobids(jcr, &ids, sql_vol, sql_jobids_from_vol, "Volume")) {
599 if (!jcr->job->selection_pattern) {
600 Jmsg(jcr, M_FATAL, 0, _("No Migration SQL selection pattern specified.\n"));
603 Dmsg1(dbglevel, "SQL=%s\n", jcr->job->selection_pattern);
604 if (!db_sql_query(jcr->db, jcr->job->selection_pattern,
605 unique_dbid_handler, (void *)&ids)) {
606 Jmsg(jcr, M_FATAL, 0,
607 _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
611 case MT_SMALLEST_VOL:
612 if (!find_mediaid_then_jobids(jcr, &ids, sql_smallest_vol, "Smallest Volume")) {
617 if (!find_mediaid_then_jobids(jcr, &ids, sql_oldest_vol, "Oldest Volume")) {
622 case MT_POOL_OCCUPANCY:
624 /* Find count of bytes in pool */
625 Mmsg(query, sql_pool_bytes, jcr->pool->hdr.name);
626 if (!db_sql_query(jcr->db, query.c_str(), db_int64_handler, (void *)&ctx)) {
627 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
630 if (ctx.count == 0) {
631 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
634 pool_bytes = ctx.value;
635 Dmsg2(dbglevel, "highbytes=%d pool=%d\n", (int)jcr->pool->MigrationHighBytes,
637 if (pool_bytes < (int64_t)jcr->pool->MigrationHighBytes) {
638 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
641 Dmsg0(dbglevel, "We should do Occupation migration.\n");
644 /* Find a list of MediaIds that could be migrated */
645 Mmsg(query, sql_mediaids, jcr->pool->hdr.name);
646 Dmsg1(dbglevel, "query=%s\n", query.c_str());
647 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)&ids)) {
648 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
651 if (ids.count == 0) {
652 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
655 Dmsg2(dbglevel, "Pool Occupancy ids=%d MediaIds=%s\n", ids.count, ids.list);
658 * Now loop over MediaIds getting more JobIds to migrate until
659 * we reduce the pool occupancy below the low water mark.
662 for (int i=0; i < (int)ids.count; i++) {
663 stat = get_next_dbid_from_list(&p, &MediaId);
664 Dmsg2(dbglevel, "get_next_dbid stat=%d MediaId=%u\n", stat, MediaId);
666 Jmsg(jcr, M_FATAL, 0, _("Invalid MediaId found.\n"));
668 } else if (stat == 0) {
672 Mmsg(mid.list, "%s", edit_int64(MediaId, ed1));
673 ok = find_jobids_from_mediaid_list(jcr, &mid, "Volumes");
678 pm_strcat(jids.list, ",");
680 pm_strcat(jids.list, mid.list);
681 jids.count += mid.count;
683 /* Now get the count of bytes added */
685 /* Find count of bytes from Jobs */
686 Mmsg(query, sql_job_bytes, mid.list);
687 if (!db_sql_query(jcr->db, query.c_str(), db_int64_handler, (void *)&ctx)) {
688 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
691 pool_bytes -= ctx.value;
692 Dmsg1(dbglevel, "Job bytes=%d\n", (int)ctx.value);
693 Dmsg2(dbglevel, "lowbytes=%d pool=%d\n", (int)jcr->pool->MigrationLowBytes,
695 if (pool_bytes <= (int64_t)jcr->pool->MigrationLowBytes) {
696 Dmsg0(dbglevel, "We should be done.\n");
701 Dmsg2(dbglevel, "Pool Occupancy ids=%d JobIds=%s\n", jids.count, jids.list);
706 ttime = time(NULL) - (time_t)jcr->pool->MigrationTime;
707 (void)localtime_r(&ttime, &tm);
708 strftime(dt, sizeof(dt), "%Y-%m-%d %H:%M:%S", &tm);
711 Mmsg(query, sql_pool_time, jcr->pool->hdr.name, dt);
712 Dmsg1(dbglevel, "query=%s\n", query.c_str());
713 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)&ids)) {
714 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
717 if (ids.count == 0) {
718 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
721 Dmsg2(dbglevel, "PoolTime ids=%d JobIds=%s\n", ids.count, ids.list);
725 Jmsg(jcr, M_FATAL, 0, _("Unknown Migration Selection Type.\n"));
731 * Loop over all jobids except the last one, sending
732 * them to start_migration_job(), which will start a job
733 * for each of them. For the last JobId, we handle it below.
736 if (ids.count == 0) {
737 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
740 Jmsg(jcr, M_INFO, 0, _("The following %u JobId%s will be migrated: %s\n"),
741 ids.count, ids.count==0?"":"s", ids.list);
742 Dmsg2(dbglevel, "Before loop count=%d ids=%s\n", ids.count, ids.list);
743 for (int i=1; i < (int)ids.count; i++) {
745 stat = get_next_jobid_from_list(&p, &JobId);
746 Dmsg3(dbglevel, "get_jobid_no=%d stat=%d JobId=%u\n", i, stat, JobId);
747 jcr->MigrateJobId = JobId;
748 start_migration_job(jcr);
749 Dmsg0(dbglevel, "Back from start_migration_job\n");
751 Jmsg(jcr, M_FATAL, 0, _("Invalid JobId found.\n"));
753 } else if (stat == 0) {
754 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
759 /* Now get the last JobId and handle it in the current job */
761 stat = get_next_jobid_from_list(&p, &JobId);
762 Dmsg2(dbglevel, "Last get_next_jobid stat=%d JobId=%u\n", stat, (int)JobId);
764 Jmsg(jcr, M_FATAL, 0, _("Invalid JobId found.\n"));
766 } else if (stat == 0) {
767 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
771 jcr->previous_jr.JobId = JobId;
772 Dmsg1(dbglevel, "Previous jobid=%d\n", (int)jcr->previous_jr.JobId);
774 if (!db_get_job_record(jcr, jcr->db, &jcr->previous_jr)) {
775 Jmsg(jcr, M_FATAL, 0, _("Could not get job record for JobId %s to migrate. ERR=%s"),
776 edit_int64(jcr->previous_jr.JobId, ed1),
777 db_strerror(jcr->db));
780 Jmsg(jcr, M_INFO, 0, _("Migration using JobId=%s Job=%s\n"),
781 edit_int64(jcr->previous_jr.JobId, ed1), jcr->previous_jr.Job);
782 Dmsg3(dbglevel, "Migration JobId=%d using JobId=%s Job=%s\n",
784 edit_int64(jcr->previous_jr.JobId, ed1), jcr->previous_jr.Job);
791 jcr->MigrateJobId = 0;
795 free_pool_memory(ids.list);
796 free_pool_memory(mid.list);
797 free_pool_memory(jids.list);
801 static void start_migration_job(JCR *jcr)
803 UAContext *ua = new_ua_context(jcr);
806 Mmsg(ua->cmd, "run %s jobid=%s", jcr->job->hdr.name,
807 edit_uint64(jcr->MigrateJobId, ed1));
808 Dmsg1(dbglevel, "=============== Migration cmd=%s\n", ua->cmd);
809 parse_ua_args(ua); /* parse command */
810 int stat = run_cmd(ua, ua->cmd);
812 Jmsg(jcr, M_ERROR, 0, _("Could not start migration job.\n"));
814 Jmsg(jcr, M_INFO, 0, _("Migration JobId %d started.\n"), stat);
819 static bool find_mediaid_then_jobids(JCR *jcr, idpkt *ids, const char *query1,
823 POOL_MEM query(PM_MESSAGE);
826 /* Basic query for MediaId */
827 Mmsg(query, query1, jcr->pool->hdr.name);
828 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
829 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
832 if (ids->count == 0) {
833 Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type);
835 if (ids->count != 1) {
836 Jmsg(jcr, M_FATAL, 0, _("SQL logic error. Count should be 1 but is %d\n"),
840 Dmsg1(dbglevel, "Smallest Vol Jobids=%s\n", ids->list);
842 ok = find_jobids_from_mediaid_list(jcr, ids, type);
848 static bool find_jobids_from_mediaid_list(JCR *jcr, idpkt *ids, const char *type)
851 POOL_MEM query(PM_MESSAGE);
853 Mmsg(query, sql_jobids_from_mediaid, ids->list);
855 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
856 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
859 if (ids->count == 0) {
860 Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type);
867 static bool regex_find_jobids(JCR *jcr, idpkt *ids, const char *query1,
868 const char *query2, const char *type)
872 uitem *last_item = NULL;
877 POOL_MEM query(PM_MESSAGE);
879 item_chain = New(dlist(item, &item->link));
880 if (!jcr->job->selection_pattern) {
881 Jmsg(jcr, M_FATAL, 0, _("No Migration %s selection pattern specified.\n"),
885 Dmsg1(dbglevel, "regex=%s\n", jcr->job->selection_pattern);
886 /* Compile regex expression */
887 rc = regcomp(&preg, jcr->job->selection_pattern, REG_EXTENDED);
889 regerror(rc, &preg, prbuf, sizeof(prbuf));
890 Jmsg(jcr, M_FATAL, 0, _("Could not compile regex pattern \"%s\" ERR=%s\n"),
891 jcr->job->selection_pattern, prbuf);
894 /* Basic query for names */
895 Mmsg(query, query1, jcr->pool->hdr.name);
896 Dmsg1(dbglevel, "get name query1=%s\n", query.c_str());
897 if (!db_sql_query(jcr->db, query.c_str(), unique_name_handler,
898 (void *)item_chain)) {
899 Jmsg(jcr, M_FATAL, 0,
900 _("SQL to get %s failed. ERR=%s\n"), type, db_strerror(jcr->db));
903 /* Now apply the regex to the names and remove any item not matched */
904 foreach_dlist(item, item_chain) {
905 const int nmatch = 30;
906 regmatch_t pmatch[nmatch];
908 Dmsg1(dbglevel, "Remove item %s\n", last_item->item);
909 free(last_item->item);
910 item_chain->remove(last_item);
912 Dmsg1(dbglevel, "get name Item=%s\n", item->item);
913 rc = regexec(&preg, item->item, nmatch, pmatch, 0);
915 last_item = NULL; /* keep this one */
921 free(last_item->item);
922 Dmsg1(dbglevel, "Remove item %s\n", last_item->item);
923 item_chain->remove(last_item);
927 * At this point, we have a list of items in item_chain
928 * that have been matched by the regex, so now we need
929 * to look up their jobids.
932 foreach_dlist(item, item_chain) {
933 Dmsg2(dbglevel, "Got %s: %s\n", type, item->item);
934 Mmsg(query, query2, item->item, jcr->pool->hdr.name);
935 Dmsg1(dbglevel, "get id from name query2=%s\n", query.c_str());
936 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
937 Jmsg(jcr, M_FATAL, 0,
938 _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
942 if (ids->count == 0) {
943 Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type);
947 Dmsg2(dbglevel, "Count=%d Jobids=%s\n", ids->count, ids->list);
949 Dmsg0(dbglevel, "After delete item_chain\n");
955 * Release resources allocated during backup.
957 void migration_cleanup(JCR *jcr, int TermCode)
959 char sdt[MAX_TIME_LENGTH], edt[MAX_TIME_LENGTH];
960 char ec1[30], ec2[30], ec3[30], ec4[30], ec5[30], elapsed[50];
961 char ec6[50], ec7[50], ec8[50];
962 char term_code[100], sd_term_msg[100];
963 const char *term_msg;
968 JCR *mig_jcr = jcr->mig_jcr;
969 POOL_MEM query(PM_MESSAGE);
971 Dmsg2(100, "Enter migrate_cleanup %d %c\n", TermCode, TermCode);
972 dequeue_messages(jcr); /* display any queued messages */
973 memset(&mr, 0, sizeof(mr));
974 set_jcr_job_status(jcr, TermCode);
975 update_job_end_record(jcr); /* update database */
978 * Check if we actually did something.
979 * mig_jcr is jcr of the newly migrated job.
982 mig_jcr->JobFiles = jcr->JobFiles = jcr->SDJobFiles;
983 mig_jcr->JobBytes = jcr->JobBytes = jcr->SDJobBytes;
984 mig_jcr->VolSessionId = jcr->VolSessionId;
985 mig_jcr->VolSessionTime = jcr->VolSessionTime;
986 mig_jcr->jr.RealEndTime = 0;
987 mig_jcr->jr.PriorJobId = jcr->previous_jr.JobId;
989 set_jcr_job_status(mig_jcr, TermCode);
992 update_job_end_record(mig_jcr);
994 /* Update final items to set them to the previous job's values */
995 Mmsg(query, "UPDATE Job SET StartTime='%s',EndTime='%s',"
996 "JobTDate=%s WHERE JobId=%s",
997 jcr->previous_jr.cStartTime, jcr->previous_jr.cEndTime,
998 edit_uint64(jcr->previous_jr.JobTDate, ec1),
999 edit_uint64(mig_jcr->jr.JobId, ec2));
1000 db_sql_query(mig_jcr->db, query.c_str(), NULL, NULL);
1002 /* Now marke the previous job as migrated */
1003 Mmsg(query, "UPDATE Job SET Type='%c' WHERE JobId=%s",
1004 (char)JT_MIGRATED_JOB, edit_uint64(jcr->previous_jr.JobId, ec1));
1005 db_sql_query(mig_jcr->db, query.c_str(), NULL, NULL);
1007 if (!db_get_job_record(jcr, jcr->db, &jcr->jr)) {
1008 Jmsg(jcr, M_WARNING, 0, _("Error getting job record for stats: %s"),
1009 db_strerror(jcr->db));
1010 set_jcr_job_status(jcr, JS_ErrorTerminated);
1013 bstrncpy(mr.VolumeName, jcr->VolumeName, sizeof(mr.VolumeName));
1014 if (!db_get_media_record(jcr, jcr->db, &mr)) {
1015 Jmsg(jcr, M_WARNING, 0, _("Error getting Media record for Volume \"%s\": ERR=%s"),
1016 mr.VolumeName, db_strerror(jcr->db));
1017 set_jcr_job_status(jcr, JS_ErrorTerminated);
1020 update_bootstrap_file(mig_jcr);
1022 if (!db_get_job_volume_names(mig_jcr, mig_jcr->db, mig_jcr->jr.JobId, &mig_jcr->VolumeName)) {
1024 * Note, if the job has erred, most likely it did not write any
1025 * tape, so suppress this "error" message since in that case
1026 * it is normal. Or look at it the other way, only for a
1027 * normal exit should we complain about this error.
1029 if (jcr->JobStatus == JS_Terminated && jcr->jr.JobBytes) {
1030 Jmsg(jcr, M_ERROR, 0, "%s", db_strerror(mig_jcr->db));
1032 mig_jcr->VolumeName[0] = 0; /* none */
1036 msg_type = M_INFO; /* by default INFO message */
1037 switch (jcr->JobStatus) {
1039 if (jcr->Errors || jcr->SDErrors) {
1040 term_msg = _("%s OK -- with warnings");
1042 term_msg = _("%s OK");
1046 case JS_ErrorTerminated:
1047 term_msg = _("*** %s Error ***");
1048 msg_type = M_ERROR; /* Generate error message */
1049 if (jcr->store_bsock) {
1050 bnet_sig(jcr->store_bsock, BNET_TERMINATE);
1051 if (jcr->SD_msg_chan) {
1052 pthread_cancel(jcr->SD_msg_chan);
1057 term_msg = _("%s Canceled");
1058 if (jcr->store_bsock) {
1059 bnet_sig(jcr->store_bsock, BNET_TERMINATE);
1060 if (jcr->SD_msg_chan) {
1061 pthread_cancel(jcr->SD_msg_chan);
1066 term_msg = _("Inappropriate %s term code");
1069 bsnprintf(term_code, sizeof(term_code), term_msg, "Migration");
1070 bstrftimes(sdt, sizeof(sdt), jcr->jr.StartTime);
1071 bstrftimes(edt, sizeof(edt), jcr->jr.EndTime);
1072 RunTime = jcr->jr.EndTime - jcr->jr.StartTime;
1076 kbps = (double)jcr->SDJobBytes / (1000 * RunTime);
1080 jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg));
1082 Jmsg(jcr, msg_type, 0, _("Bacula %s (%s): %s\n"
1083 " Prev Backup JobId: %s\n"
1084 " New Backup JobId: %s\n"
1085 " Migration JobId: %s\n"
1086 " Migration Job: %s\n"
1087 " Backup Level: %s%s\n"
1089 " FileSet: \"%s\" %s\n"
1090 " Pool: \"%s\" (From %s)\n"
1091 " Read Storage: \"%s\" (From %s_\n"
1092 " Write Storage: \"%s\" (From %s)\n"
1095 " Elapsed time: %s\n"
1097 " SD Files Written: %s\n"
1098 " SD Bytes Written: %s (%sB)\n"
1099 " Rate: %.1f KB/s\n"
1100 " Volume name(s): %s\n"
1101 " Volume Session Id: %d\n"
1102 " Volume Session Time: %d\n"
1103 " Last Volume Bytes: %s (%sB)\n"
1105 " SD termination status: %s\n"
1106 " Termination: %s\n\n"),
1110 mig_jcr ? edit_uint64(jcr->previous_jr.JobId, ec6) : "0",
1111 mig_jcr ? edit_uint64(mig_jcr->jr.JobId, ec7) : "0",
1112 edit_uint64(jcr->jr.JobId, ec8),
1114 level_to_str(jcr->JobLevel), jcr->since,
1115 jcr->client->name(),
1116 jcr->fileset->name(), jcr->FSCreateTime,
1117 jcr->pool->name(), jcr->pool_source,
1118 jcr->rstore->name(), jcr->rstore_source,
1119 jcr->wstore->name(), jcr->wstore_source,
1122 edit_utime(RunTime, elapsed, sizeof(elapsed)),
1124 edit_uint64_with_commas(jcr->SDJobFiles, ec1),
1125 edit_uint64_with_commas(jcr->SDJobBytes, ec2),
1126 edit_uint64_with_suffix(jcr->SDJobBytes, ec3),
1128 mig_jcr ? mig_jcr->VolumeName : "",
1130 jcr->VolSessionTime,
1131 edit_uint64_with_commas(mr.VolBytes, ec4),
1132 edit_uint64_with_suffix(mr.VolBytes, ec5),
1137 Dmsg1(100, "migrate_cleanup() mig_jcr=0x%x\n", jcr->mig_jcr);
1139 free_jcr(jcr->mig_jcr);
1140 jcr->mig_jcr = NULL;
1142 Dmsg0(100, "Leave migrate_cleanup()\n");
1146 * Return next DBId from comma separated list
1149 * 1 if next DBId returned
1150 * 0 if no more DBIds are in list
1151 * -1 there is an error
1153 static int get_next_dbid_from_list(char **p, DBId_t *DBId)
1159 for (int i=0; i<(int)sizeof(id); i++) {
1162 } else if (*q == ',') {
1171 } else if (!is_a_number(id)) {
1172 return -1; /* error */
1175 *DBId = str_to_int64(id);