3 * Bacula Director -- migrate.c -- responsible for doing
6 * Kern Sibbald, September MMIV
8 * Basic tasks done here:
9 * Open DB and create records for this job.
10 * Open Message Channel with Storage daemon to tell him a job will be starting.
11 * Open connection with Storage daemon and pass him commands
13 * When the Storage daemon finishes the job, update the DB.
18 Copyright (C) 2004-2006 Kern Sibbald
20 This program is free software; you can redistribute it and/or
21 modify it under the terms of the GNU General Public License
22 version 2 as amended with additional clauses defined in the
23 file LICENSE in the main source directory.
25 This program is distributed in the hope that it will be useful,
26 but WITHOUT ANY WARRANTY; without even the implied warranty of
27 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
28 the file LICENSE for additional details.
36 #include "lib/bregex.h"
41 static const int dbglevel = 10;
43 static char OKbootstrap[] = "3000 OK bootstrap\n";
44 static bool get_job_to_migrate(JCR *jcr);
46 static bool regex_find_jobids(JCR *jcr, idpkt *ids, const char *query1,
47 const char *query2, const char *type);
48 static bool find_mediaid_then_jobids(JCR *jcr, idpkt *ids, const char *query1,
50 static bool find_jobids_from_mediaid_list(JCR *jcr, idpkt *ids, const char *type);
51 static void start_migration_job(JCR *jcr);
52 static int get_next_dbid_from_list(char **p, DBId_t *DBId);
55 * Called here before the job is run to do the job
58 bool do_migration_init(JCR *jcr)
60 /* If we find a job or jobs to migrate it is previous_jr.JobId */
61 if (!get_job_to_migrate(jcr)) {
64 Dmsg1(dbglevel, "Back from get_job_to_migrate JobId=%d\n", (int)jcr->JobId);
66 if (jcr->previous_jr.JobId == 0) {
67 Dmsg1(dbglevel, "JobId=%d no previous JobId\n", (int)jcr->JobId);
68 Jmsg(jcr, M_INFO, 0, _("No previous Job found to migrate.\n"));
69 return true; /* no work */
72 if (!get_or_create_fileset_record(jcr)) {
73 Dmsg1(dbglevel, "JobId=%d no FileSet\n", (int)jcr->JobId);
74 Jmsg(jcr, M_FATAL, 0, _("Could not get or create the FileSet record.\n"));
78 apply_pool_overrides(jcr);
80 jcr->jr.PoolId = get_or_create_pool_record(jcr, jcr->pool->hdr.name);
81 if (jcr->jr.PoolId == 0) {
82 Dmsg1(dbglevel, "JobId=%d no PoolId\n", (int)jcr->JobId);
83 Jmsg(jcr, M_FATAL, 0, _("Could not get or create a Pool record.\n"));
87 /* If pool storage specified, use it instead of job storage */
88 copy_wstorage(jcr, jcr->pool->storage, _("Pool resource"));
90 if (jcr->wstorage->size() == 0) {
91 Jmsg(jcr, M_FATAL, 0, _("No Storage specification found in Job or Pool.\n"));
95 create_restore_bootstrap_file(jcr);
100 * Do a Migration of a previous job
102 * Returns: false on failure
105 bool do_migration(JCR *jcr)
112 JCR *mig_jcr; /* newly migrated job */
115 * previous_jr refers to the job DB record of the Job that is
116 * going to be migrated.
117 * prev_job refers to the job resource of the Job that is
118 * going to be migrated.
119 * jcr is the jcr for the current "migration" job. It is a
120 * control job that is put in the DB as a migration job, which
121 * means that this job migrated a previous job to a new job.
122 * No Volume or File data is associated with this control
124 * mig_jcr refers to the newly migrated job that is run by
125 * the current jcr. It is a backup job that moves (migrates) the
126 * data written for the previous_jr into the new pool. This
127 * job (mig_jcr) becomes the new backup job that replaces
128 * the original backup job.
130 if (jcr->previous_jr.JobId == 0 || jcr->ExpectedFiles == 0) {
131 set_jcr_job_status(jcr, JS_Terminated);
132 Dmsg1(dbglevel, "JobId=%d expected files == 0\n", (int)jcr->JobId);
133 if (jcr->previous_jr.JobId == 0) {
134 Jmsg(jcr, M_INFO, 0, _("No previous Job found to migrate.\n"));
136 Jmsg(jcr, M_INFO, 0, _("Previous Job has no data to migrate.\n"));
138 migration_cleanup(jcr, jcr->JobStatus);
139 return true; /* no work */
142 Dmsg5(dbglevel, "JobId=%d: Previous: Name=%s JobId=%d Type=%c Level=%c\n",
144 jcr->previous_jr.Name, (int)jcr->previous_jr.JobId,
145 jcr->previous_jr.JobType, jcr->previous_jr.JobLevel);
147 Dmsg5(dbglevel, "JobId=%d: Current: Name=%s JobId=%d Type=%c Level=%c\n",
149 jcr->jr.Name, (int)jcr->jr.JobId,
150 jcr->jr.JobType, jcr->jr.JobLevel);
153 job = (JOB *)GetResWithName(R_JOB, jcr->jr.Name);
154 prev_job = (JOB *)GetResWithName(R_JOB, jcr->previous_jr.Name);
156 if (!job || !prev_job) {
160 /* Create a migation jcr */
161 mig_jcr = jcr->mig_jcr = new_jcr(sizeof(JCR), dird_free_jcr);
162 memcpy(&mig_jcr->previous_jr, &jcr->previous_jr, sizeof(mig_jcr->previous_jr));
165 * Turn the mig_jcr into a "real" job that takes on the aspects of
166 * the previous backup job "prev_job".
168 set_jcr_defaults(mig_jcr, prev_job);
169 if (!setup_job(mig_jcr)) {
173 /* Now reset the job record from the previous job */
174 memcpy(&mig_jcr->jr, &jcr->previous_jr, sizeof(mig_jcr->jr));
175 /* Update the jr to reflect the new values of PoolId, FileSetId, and JobId. */
176 mig_jcr->jr.PoolId = jcr->jr.PoolId;
177 mig_jcr->jr.FileSetId = jcr->jr.FileSetId;
178 mig_jcr->jr.JobId = mig_jcr->JobId;
180 Dmsg4(dbglevel, "mig_jcr: Name=%s JobId=%d Type=%c Level=%c\n",
181 mig_jcr->jr.Name, (int)mig_jcr->jr.JobId,
182 mig_jcr->jr.JobType, mig_jcr->jr.JobLevel);
185 * Get the PoolId used with the original job. Then
186 * find the pool name from the database record.
188 memset(&pr, 0, sizeof(pr));
189 pr.PoolId = mig_jcr->previous_jr.PoolId;
190 if (!db_get_pool_record(jcr, jcr->db, &pr)) {
191 Jmsg(jcr, M_FATAL, 0, _("Pool for JobId %s not in database. ERR=%s\n"),
192 edit_int64(pr.PoolId, ed1), db_strerror(jcr->db));
195 /* Get the pool resource corresponding to the original job */
196 pool = (POOL *)GetResWithName(R_POOL, pr.Name);
198 Jmsg(jcr, M_FATAL, 0, _("Pool resource \"%s\" not found.\n"), pr.Name);
202 /* If pool storage specified, use it for restore */
203 copy_rstorage(mig_jcr, pool->storage, _("Pool resource"));
204 copy_rstorage(jcr, pool->storage, _("Pool resource"));
207 * If the original backup pool has a NextPool, make sure a
208 * record exists in the database. Note, in this case, we
209 * will be migrating from pool to pool->NextPool.
211 if (pool->NextPool) {
212 jcr->jr.PoolId = get_or_create_pool_record(jcr, pool->NextPool->hdr.name);
213 if (jcr->jr.PoolId == 0) {
217 * put the "NextPool" resource pointer in our jcr so that we
218 * can pull the Storage reference from it.
220 mig_jcr->pool = jcr->pool = pool->NextPool;
221 mig_jcr->jr.PoolId = jcr->jr.PoolId;
222 pm_strcpy(jcr->pool_source, _("NextPool in Pool resource"));
224 Jmsg(jcr, M_FATAL, 0, _("No Next Pool specification found in Pool \"%s\".\n"),
229 if (!jcr->pool->storage) {
230 Jmsg(jcr, M_FATAL, 0, _("No Storage specification found in Next Pool \"%s\".\n"),
231 jcr->pool->hdr.name);
235 /* If pool storage specified, use it instead of job storage for backup */
236 copy_wstorage(jcr, jcr->pool->storage, _("Next pool resource"));
238 /* Print Job Start message */
239 Jmsg(jcr, M_INFO, 0, _("Start Migration JobId %s, Job=%s\n"),
240 edit_uint64(jcr->JobId, ed1), jcr->Job);
242 set_jcr_job_status(jcr, JS_Running);
243 set_jcr_job_status(mig_jcr, JS_Running);
244 Dmsg2(dbglevel, "JobId=%d JobLevel=%c\n", (int)jcr->jr.JobId, jcr->jr.JobLevel);
246 /* Update job start record for this migration control job */
247 if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) {
248 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db));
252 Dmsg4(dbglevel, "mig_jcr: Name=%s JobId=%d Type=%c Level=%c\n",
253 mig_jcr->jr.Name, (int)mig_jcr->jr.JobId,
254 mig_jcr->jr.JobType, mig_jcr->jr.JobLevel);
256 /* Update job start record for the real migration backup job */
257 if (!db_update_job_start_record(mig_jcr, mig_jcr->db, &mig_jcr->jr)) {
258 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(mig_jcr->db));
264 * Open a message channel connection with the Storage
265 * daemon. This is to let him know that our client
266 * will be contacting him for a backup session.
269 Dmsg0(110, "Open connection with storage daemon\n");
270 set_jcr_job_status(jcr, JS_WaitSD);
271 set_jcr_job_status(mig_jcr, JS_WaitSD);
273 * Start conversation with Storage daemon
275 if (!connect_to_storage_daemon(jcr, 10, SDConnectTimeout, 1)) {
278 sd = jcr->store_bsock;
280 * Now start a job with the Storage daemon
282 Dmsg2(dbglevel, "Read store=%s, write store=%s\n",
283 ((STORE *)jcr->rstorage->first())->name(),
284 ((STORE *)jcr->wstorage->first())->name());
285 if (((STORE *)jcr->rstorage->first())->name() == ((STORE *)jcr->wstorage->first())->name()) {
286 Jmsg(jcr, M_FATAL, 0, _("Read storage \"%s\" same as write storage.\n"),
287 ((STORE *)jcr->rstorage->first())->name());
290 if (!start_storage_daemon_job(jcr, jcr->rstorage, jcr->wstorage)) {
293 Dmsg0(150, "Storage daemon connection OK\n");
295 if (!send_bootstrap_file(jcr, sd) ||
296 !response(jcr, sd, OKbootstrap, "Bootstrap", DISPLAY_ERROR)) {
300 if (!bnet_fsend(sd, "run")) {
305 * Now start a Storage daemon message thread
307 if (!start_storage_daemon_message_thread(jcr)) {
312 set_jcr_job_status(jcr, JS_Running);
313 set_jcr_job_status(mig_jcr, JS_Running);
315 /* Pickup Job termination data */
316 /* Note, the SD stores in jcr->JobFiles/ReadBytes/JobBytes/Errors */
317 wait_for_storage_daemon_termination(jcr);
319 set_jcr_job_status(jcr, jcr->SDJobStatus);
320 if (jcr->JobStatus != JS_Terminated) {
323 migration_cleanup(jcr, jcr->JobStatus);
325 UAContext *ua = new_ua_context(jcr);
326 purge_files_from_job(ua, jcr->previous_jr.JobId);
337 /* Add an item to the list if it is unique */
338 static void add_unique_id(idpkt *ids, char *item)
343 /* Walk through current list to see if each item is the same as item */
346 for (int i=0; i<(int)sizeof(id); i++) {
349 } else if (*q == ',') {
356 if (strcmp(item, id) == 0) {
360 /* Did not find item, so add it to list */
361 if (ids->count == 0) {
364 pm_strcat(ids->list, ",");
366 pm_strcat(ids->list, item);
368 // Dmsg3(0, "add_uniq count=%d Ids=%p %s\n", ids->count, ids->list, ids->list);
373 * Callback handler make list of DB Ids
375 static int unique_dbid_handler(void *ctx, int num_fields, char **row)
377 idpkt *ids = (idpkt *)ctx;
379 add_unique_id(ids, row[0]);
380 Dmsg3(dbglevel, "dbid_hdlr count=%d Ids=%p %s\n", ids->count, ids->list, ids->list);
390 static int item_compare(void *item1, void *item2)
392 uitem *i1 = (uitem *)item1;
393 uitem *i2 = (uitem *)item2;
394 return strcmp(i1->item, i2->item);
397 static int unique_name_handler(void *ctx, int num_fields, char **row)
399 dlist *list = (dlist *)ctx;
401 uitem *new_item = (uitem *)malloc(sizeof(uitem));
404 memset(new_item, 0, sizeof(uitem));
405 new_item->item = bstrdup(row[0]);
406 Dmsg1(dbglevel, "Unique_name_hdlr Item=%s\n", row[0]);
407 item = (uitem *)list->binary_insert((void *)new_item, item_compare);
408 if (item != new_item) { /* already in list */
409 free(new_item->item);
410 free((char *)new_item);
416 /* Get Job names in Pool */
417 const char *sql_job =
418 "SELECT DISTINCT Job.Name from Job,Pool"
419 " WHERE Pool.Name='%s' AND Job.PoolId=Pool.PoolId";
421 /* Get JobIds from regex'ed Job names */
422 const char *sql_jobids_from_job =
423 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Job,Pool"
424 " WHERE Job.Name='%s' AND Pool.Name='%s' AND Job.PoolId=Pool.PoolId"
425 " ORDER by Job.StartTime";
427 /* Get Client names in Pool */
428 const char *sql_client =
429 "SELECT DISTINCT Client.Name from Client,Pool,Job"
430 " WHERE Pool.Name='%s' AND Job.ClientId=Client.ClientId AND"
431 " Job.PoolId=Pool.PoolId";
433 /* Get JobIds from regex'ed Client names */
434 const char *sql_jobids_from_client =
435 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Job,Pool,Client"
436 " WHERE Client.Name='%s' AND Pool.Name='%s' AND Job.PoolId=Pool.PoolId"
437 " AND Job.ClientId=Client.ClientId "
438 " ORDER by Job.StartTime";
440 /* Get Volume names in Pool */
441 const char *sql_vol =
442 "SELECT DISTINCT VolumeName FROM Media,Pool WHERE"
443 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
444 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'";
446 /* Get JobIds from regex'ed Volume names */
447 const char *sql_jobids_from_vol =
448 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Media,JobMedia,Job"
449 " WHERE Media.VolumeName='%s' AND Media.MediaId=JobMedia.MediaId"
450 " AND JobMedia.JobId=Job.JobId"
451 " ORDER by Job.StartTime";
454 const char *sql_smallest_vol =
455 "SELECT MediaId FROM Media,Pool WHERE"
456 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
457 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'"
458 " ORDER BY VolBytes ASC LIMIT 1";
460 const char *sql_oldest_vol =
461 "SELECT MediaId FROM Media,Pool WHERE"
462 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
463 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'"
464 " ORDER BY LastWritten ASC LIMIT 1";
466 /* Get JobIds when we have selected MediaId */
467 const char *sql_jobids_from_mediaid =
468 "SELECT DISTINCT Job.JobId,Job.StartTime FROM JobMedia,Job"
469 " WHERE JobMedia.JobId=Job.JobId AND JobMedia.MediaId=%s"
470 " ORDER by Job.StartTime";
472 /* Get tne number of bytes in the pool */
473 const char *sql_pool_bytes =
474 "SELECT SUM(VolBytes) FROM Media,Pool WHERE"
475 " VolStatus in ('Full','Used','Error','Append') AND Media.Enabled=1 AND"
476 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'";
478 /* Get tne number of bytes in the Jobs */
479 const char *sql_job_bytes =
480 "SELECT SUM(JobBytes) FROM Job WHERE JobId IN (%s)";
483 /* Get Media Ids in Pool */
484 const char *sql_mediaids =
485 "SELECT MediaId FROM Media,Pool WHERE"
486 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
487 " Media.PoolId=Pool.PoolId AND Pool.Name='%s' ORDER BY LastWritten ASC";
489 /* Get JobIds in Pool longer than specified time */
490 const char *sql_pool_time =
491 "SELECT DISTINCT Job.JobId from Pool,Job,Media,JobMedia WHERE"
492 " Pool.Name='%s' AND Media.PoolId=Pool.PoolId AND"
493 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
494 " JobMedia.JobId=Job.JobId AND Job.PoolId=Media.PoolId"
495 " AND Job.RealEndTime<='%s'";
498 * const char *sql_ujobid =
499 * "SELECT DISTINCT Job.Job from Client,Pool,Media,Job,JobMedia "
500 * " WHERE Media.PoolId=Pool.PoolId AND Pool.Name='%s' AND"
501 * " JobMedia.JobId=Job.JobId AND Job.PoolId=Media.PoolId";
508 * This is the central piece of code that finds a job or jobs
509 * actually JobIds to migrate. It first looks to see if one
510 * has been "manually" specified in jcr->MigrateJobId, and if
511 * so, it returns that JobId to be run. Otherwise, it
512 * examines the Selection Type to see what kind of migration
513 * we are doing (Volume, Job, Client, ...) and applies any
514 * Selection Pattern if appropriate to obtain a list of JobIds.
515 * Finally, it will loop over all the JobIds found, except the last
516 * one starting a new job with MigrationJobId set to that JobId, and
517 * finally, it returns the last JobId to the caller.
519 * Returns: false on error
520 * true if OK and jcr->previous_jr filled in
522 static bool get_job_to_migrate(JCR *jcr)
525 POOL_MEM query(PM_MESSAGE);
530 idpkt ids, mid, jids;
536 char dt[MAX_TIME_LENGTH];
538 ids.list = get_pool_memory(PM_MESSAGE);
541 mid.list = get_pool_memory(PM_MESSAGE);
544 jids.list = get_pool_memory(PM_MESSAGE);
550 * If MigrateJobId is set, then we migrate only that Job,
551 * otherwise, we go through the full selection of jobs to
554 if (jcr->MigrateJobId != 0) {
555 Dmsg1(dbglevel, "At Job start previous jobid=%u\n", jcr->MigrateJobId);
556 edit_uint64(jcr->MigrateJobId, ids.list);
559 switch (jcr->job->selection_type) {
561 if (!regex_find_jobids(jcr, &ids, sql_job, sql_jobids_from_job, "Job")) {
566 if (!regex_find_jobids(jcr, &ids, sql_client, sql_jobids_from_client, "Client")) {
571 if (!regex_find_jobids(jcr, &ids, sql_vol, sql_jobids_from_vol, "Volume")) {
576 if (!jcr->job->selection_pattern) {
577 Jmsg(jcr, M_FATAL, 0, _("No Migration SQL selection pattern specified.\n"));
580 Dmsg1(dbglevel, "SQL=%s\n", jcr->job->selection_pattern);
581 if (!db_sql_query(jcr->db, jcr->job->selection_pattern,
582 unique_dbid_handler, (void *)&ids)) {
583 Jmsg(jcr, M_FATAL, 0,
584 _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
588 case MT_SMALLEST_VOL:
589 if (!find_mediaid_then_jobids(jcr, &ids, sql_smallest_vol, "Smallest Volume")) {
594 if (!find_mediaid_then_jobids(jcr, &ids, sql_oldest_vol, "Oldest Volume")) {
599 case MT_POOL_OCCUPANCY:
601 /* Find count of bytes in pool */
602 Mmsg(query, sql_pool_bytes, jcr->pool->hdr.name);
603 if (!db_sql_query(jcr->db, query.c_str(), db_int64_handler, (void *)&ctx)) {
604 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
607 if (ctx.count == 0) {
608 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
611 pool_bytes = ctx.value;
612 Dmsg2(dbglevel, "highbytes=%d pool=%d\n", (int)jcr->pool->MigrationHighBytes,
614 if (pool_bytes < (int64_t)jcr->pool->MigrationHighBytes) {
615 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
618 Dmsg0(dbglevel, "We should do Occupation migration.\n");
621 /* Find a list of MediaIds that could be migrated */
622 Mmsg(query, sql_mediaids, jcr->pool->hdr.name);
623 Dmsg1(dbglevel, "query=%s\n", query.c_str());
624 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)&ids)) {
625 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
628 if (ids.count == 0) {
629 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
632 Dmsg2(dbglevel, "Pool Occupancy ids=%d MediaIds=%s\n", ids.count, ids.list);
635 * Now loop over MediaIds getting more JobIds to migrate until
636 * we reduce the pool occupancy below the low water mark.
639 for (int i=0; i < (int)ids.count; i++) {
640 stat = get_next_dbid_from_list(&p, &MediaId);
641 Dmsg2(dbglevel, "get_next_dbid stat=%d MediaId=%u\n", stat, MediaId);
643 Jmsg(jcr, M_FATAL, 0, _("Invalid MediaId found.\n"));
645 } else if (stat == 0) {
649 Mmsg(mid.list, "%s", edit_int64(MediaId, ed1));
650 ok = find_jobids_from_mediaid_list(jcr, &mid, "Volumes");
655 pm_strcat(jids.list, ",");
657 pm_strcat(jids.list, mid.list);
658 jids.count += mid.count;
660 /* Now get the count of bytes added */
662 /* Find count of bytes from Jobs */
663 Mmsg(query, sql_job_bytes, mid.list);
664 if (!db_sql_query(jcr->db, query.c_str(), db_int64_handler, (void *)&ctx)) {
665 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
668 pool_bytes -= ctx.value;
669 Dmsg1(dbglevel, "Job bytes=%d\n", (int)ctx.value);
670 Dmsg2(dbglevel, "lowbytes=%d pool=%d\n", (int)jcr->pool->MigrationLowBytes,
672 if (pool_bytes <= (int64_t)jcr->pool->MigrationLowBytes) {
673 Dmsg0(dbglevel, "We should be done.\n");
678 Dmsg2(dbglevel, "Pool Occupancy ids=%d JobIds=%s\n", jids.count, jids.list);
683 ttime = time(NULL) - (time_t)jcr->pool->MigrationTime;
684 (void)localtime_r(&ttime, &tm);
685 strftime(dt, sizeof(dt), "%Y-%m-%d %H:%M:%S", &tm);
688 Mmsg(query, sql_pool_time, jcr->pool->hdr.name, dt);
689 Dmsg1(dbglevel, "query=%s\n", query.c_str());
690 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)&ids)) {
691 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
694 if (ids.count == 0) {
695 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
698 Dmsg2(dbglevel, "PoolTime ids=%d JobIds=%s\n", ids.count, ids.list);
702 Jmsg(jcr, M_FATAL, 0, _("Unknown Migration Selection Type.\n"));
708 * Loop over all jobids except the last one, sending
709 * them to start_migration_job(), which will start a job
710 * for each of them. For the last JobId, we handle it below.
713 if (ids.count == 0) {
714 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
717 Jmsg(jcr, M_INFO, 0, _("The following %u JobId%s will be migrated: %s\n"),
718 ids.count, ids.count==0?"":"s", ids.list);
719 Dmsg2(dbglevel, "Before loop count=%d ids=%s\n", ids.count, ids.list);
720 for (int i=1; i < (int)ids.count; i++) {
722 stat = get_next_jobid_from_list(&p, &JobId);
723 Dmsg3(dbglevel, "get_jobid_no=%d stat=%d JobId=%u\n", i, stat, JobId);
724 jcr->MigrateJobId = JobId;
725 start_migration_job(jcr);
726 Dmsg0(dbglevel, "Back from start_migration_job\n");
728 Jmsg(jcr, M_FATAL, 0, _("Invalid JobId found.\n"));
730 } else if (stat == 0) {
731 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
736 /* Now get the last JobId and handle it in the current job */
738 stat = get_next_jobid_from_list(&p, &JobId);
739 Dmsg2(dbglevel, "Last get_next_jobid stat=%d JobId=%u\n", stat, (int)JobId);
741 Jmsg(jcr, M_FATAL, 0, _("Invalid JobId found.\n"));
743 } else if (stat == 0) {
744 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
748 jcr->previous_jr.JobId = JobId;
749 Dmsg1(dbglevel, "Previous jobid=%d\n", (int)jcr->previous_jr.JobId);
751 if (!db_get_job_record(jcr, jcr->db, &jcr->previous_jr)) {
752 Jmsg(jcr, M_FATAL, 0, _("Could not get job record for JobId %s to migrate. ERR=%s"),
753 edit_int64(jcr->previous_jr.JobId, ed1),
754 db_strerror(jcr->db));
757 Jmsg(jcr, M_INFO, 0, _("Migration using JobId=%s Job=%s\n"),
758 edit_int64(jcr->previous_jr.JobId, ed1), jcr->previous_jr.Job);
759 Dmsg3(dbglevel, "Migration JobId=%d using JobId=%s Job=%s\n",
761 edit_int64(jcr->previous_jr.JobId, ed1), jcr->previous_jr.Job);
768 jcr->MigrateJobId = 0;
772 free_pool_memory(ids.list);
773 free_pool_memory(mid.list);
774 free_pool_memory(jids.list);
778 static void start_migration_job(JCR *jcr)
780 UAContext *ua = new_ua_context(jcr);
783 Mmsg(ua->cmd, "run %s jobid=%s", jcr->job->hdr.name,
784 edit_uint64(jcr->MigrateJobId, ed1));
785 Dmsg1(dbglevel, "=============== Migration cmd=%s\n", ua->cmd);
786 parse_ua_args(ua); /* parse command */
787 int stat = run_cmd(ua, ua->cmd);
789 Jmsg(jcr, M_ERROR, 0, _("Could not start migration job.\n"));
791 Jmsg(jcr, M_INFO, 0, _("Migration JobId %d started.\n"), stat);
796 static bool find_mediaid_then_jobids(JCR *jcr, idpkt *ids, const char *query1,
800 POOL_MEM query(PM_MESSAGE);
803 /* Basic query for MediaId */
804 Mmsg(query, query1, jcr->pool->hdr.name);
805 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
806 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
809 if (ids->count == 0) {
810 Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type);
812 if (ids->count != 1) {
813 Jmsg(jcr, M_FATAL, 0, _("SQL logic error. Count should be 1 but is %d\n"),
817 Dmsg1(dbglevel, "Smallest Vol Jobids=%s\n", ids->list);
819 ok = find_jobids_from_mediaid_list(jcr, ids, type);
825 static bool find_jobids_from_mediaid_list(JCR *jcr, idpkt *ids, const char *type)
828 POOL_MEM query(PM_MESSAGE);
830 Mmsg(query, sql_jobids_from_mediaid, ids->list);
832 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
833 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
836 if (ids->count == 0) {
837 Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type);
844 static bool regex_find_jobids(JCR *jcr, idpkt *ids, const char *query1,
845 const char *query2, const char *type)
849 uitem *last_item = NULL;
854 POOL_MEM query(PM_MESSAGE);
856 item_chain = New(dlist(item, &item->link));
857 if (!jcr->job->selection_pattern) {
858 Jmsg(jcr, M_FATAL, 0, _("No Migration %s selection pattern specified.\n"),
862 Dmsg1(dbglevel, "regex=%s\n", jcr->job->selection_pattern);
863 /* Compile regex expression */
864 rc = regcomp(&preg, jcr->job->selection_pattern, REG_EXTENDED);
866 regerror(rc, &preg, prbuf, sizeof(prbuf));
867 Jmsg(jcr, M_FATAL, 0, _("Could not compile regex pattern \"%s\" ERR=%s\n"),
868 jcr->job->selection_pattern, prbuf);
871 /* Basic query for names */
872 Mmsg(query, query1, jcr->pool->hdr.name);
873 Dmsg1(dbglevel, "get name query1=%s\n", query.c_str());
874 if (!db_sql_query(jcr->db, query.c_str(), unique_name_handler,
875 (void *)item_chain)) {
876 Jmsg(jcr, M_FATAL, 0,
877 _("SQL to get %s failed. ERR=%s\n"), type, db_strerror(jcr->db));
880 /* Now apply the regex to the names and remove any item not matched */
881 foreach_dlist(item, item_chain) {
882 const int nmatch = 30;
883 regmatch_t pmatch[nmatch];
885 Dmsg1(dbglevel, "Remove item %s\n", last_item->item);
886 free(last_item->item);
887 item_chain->remove(last_item);
889 Dmsg1(dbglevel, "get name Item=%s\n", item->item);
890 rc = regexec(&preg, item->item, nmatch, pmatch, 0);
892 last_item = NULL; /* keep this one */
898 free(last_item->item);
899 Dmsg1(dbglevel, "Remove item %s\n", last_item->item);
900 item_chain->remove(last_item);
904 * At this point, we have a list of items in item_chain
905 * that have been matched by the regex, so now we need
906 * to look up their jobids.
909 foreach_dlist(item, item_chain) {
910 Dmsg2(dbglevel, "Got %s: %s\n", type, item->item);
911 Mmsg(query, query2, item->item, jcr->pool->hdr.name);
912 Dmsg1(dbglevel, "get id from name query2=%s\n", query.c_str());
913 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
914 Jmsg(jcr, M_FATAL, 0,
915 _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
919 if (ids->count == 0) {
920 Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type);
924 Dmsg2(dbglevel, "Count=%d Jobids=%s\n", ids->count, ids->list);
926 Dmsg0(dbglevel, "After delete item_chain\n");
932 * Release resources allocated during backup.
934 void migration_cleanup(JCR *jcr, int TermCode)
936 char sdt[MAX_TIME_LENGTH], edt[MAX_TIME_LENGTH];
937 char ec1[30], ec2[30], ec3[30], ec4[30], ec5[30], elapsed[50];
938 char ec6[50], ec7[50], ec8[50];
939 char term_code[100], sd_term_msg[100];
940 const char *term_msg;
945 JCR *mig_jcr = jcr->mig_jcr;
946 POOL_MEM query(PM_MESSAGE);
948 Dmsg2(100, "Enter migrate_cleanup %d %c\n", TermCode, TermCode);
949 dequeue_messages(jcr); /* display any queued messages */
950 memset(&mr, 0, sizeof(mr));
951 set_jcr_job_status(jcr, TermCode);
952 update_job_end_record(jcr); /* update database */
955 * Check if we actually did something.
956 * mig_jcr is jcr of the newly migrated job.
959 mig_jcr->JobFiles = jcr->JobFiles = jcr->SDJobFiles;
960 mig_jcr->JobBytes = jcr->JobBytes = jcr->SDJobBytes;
961 mig_jcr->VolSessionId = jcr->VolSessionId;
962 mig_jcr->VolSessionTime = jcr->VolSessionTime;
963 mig_jcr->jr.RealEndTime = 0;
964 mig_jcr->jr.PriorJobId = jcr->previous_jr.JobId;
966 set_jcr_job_status(mig_jcr, TermCode);
969 update_job_end_record(mig_jcr);
971 /* Update final items to set them to the previous job's values */
972 Mmsg(query, "UPDATE Job SET StartTime='%s',EndTime='%s',"
973 "JobTDate=%s WHERE JobId=%s",
974 jcr->previous_jr.cStartTime, jcr->previous_jr.cEndTime,
975 edit_uint64(jcr->previous_jr.JobTDate, ec1),
976 edit_uint64(mig_jcr->jr.JobId, ec2));
977 db_sql_query(mig_jcr->db, query.c_str(), NULL, NULL);
979 /* Now marke the previous job as migrated */
980 Mmsg(query, "UPDATE Job SET Type='%c' WHERE JobId=%s",
981 (char)JT_MIGRATED_JOB, edit_uint64(jcr->previous_jr.JobId, ec1));
982 db_sql_query(mig_jcr->db, query.c_str(), NULL, NULL);
984 if (!db_get_job_record(jcr, jcr->db, &jcr->jr)) {
985 Jmsg(jcr, M_WARNING, 0, _("Error getting job record for stats: %s"),
986 db_strerror(jcr->db));
987 set_jcr_job_status(jcr, JS_ErrorTerminated);
990 bstrncpy(mr.VolumeName, jcr->VolumeName, sizeof(mr.VolumeName));
991 if (!db_get_media_record(jcr, jcr->db, &mr)) {
992 Jmsg(jcr, M_WARNING, 0, _("Error getting Media record for Volume \"%s\": ERR=%s"),
993 mr.VolumeName, db_strerror(jcr->db));
994 set_jcr_job_status(jcr, JS_ErrorTerminated);
997 update_bootstrap_file(mig_jcr);
999 if (!db_get_job_volume_names(mig_jcr, mig_jcr->db, mig_jcr->jr.JobId, &mig_jcr->VolumeName)) {
1001 * Note, if the job has erred, most likely it did not write any
1002 * tape, so suppress this "error" message since in that case
1003 * it is normal. Or look at it the other way, only for a
1004 * normal exit should we complain about this error.
1006 if (jcr->JobStatus == JS_Terminated && jcr->jr.JobBytes) {
1007 Jmsg(jcr, M_ERROR, 0, "%s", db_strerror(mig_jcr->db));
1009 mig_jcr->VolumeName[0] = 0; /* none */
1013 msg_type = M_INFO; /* by default INFO message */
1014 switch (jcr->JobStatus) {
1016 if (jcr->Errors || jcr->SDErrors) {
1017 term_msg = _("%s OK -- with warnings");
1019 term_msg = _("%s OK");
1023 case JS_ErrorTerminated:
1024 term_msg = _("*** %s Error ***");
1025 msg_type = M_ERROR; /* Generate error message */
1026 if (jcr->store_bsock) {
1027 bnet_sig(jcr->store_bsock, BNET_TERMINATE);
1028 if (jcr->SD_msg_chan) {
1029 pthread_cancel(jcr->SD_msg_chan);
1034 term_msg = _("%s Canceled");
1035 if (jcr->store_bsock) {
1036 bnet_sig(jcr->store_bsock, BNET_TERMINATE);
1037 if (jcr->SD_msg_chan) {
1038 pthread_cancel(jcr->SD_msg_chan);
1043 term_msg = _("Inappropriate %s term code");
1046 bsnprintf(term_code, sizeof(term_code), term_msg, "Migration");
1047 bstrftimes(sdt, sizeof(sdt), jcr->jr.StartTime);
1048 bstrftimes(edt, sizeof(edt), jcr->jr.EndTime);
1049 RunTime = jcr->jr.EndTime - jcr->jr.StartTime;
1053 kbps = (double)jcr->SDJobBytes / (1000 * RunTime);
1057 jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg));
1059 Jmsg(jcr, msg_type, 0, _("Bacula %s (%s): %s\n"
1060 " Prev Backup JobId: %s\n"
1061 " New Backup JobId: %s\n"
1062 " Migration JobId: %s\n"
1063 " Migration Job: %s\n"
1064 " Backup Level: %s%s\n"
1066 " FileSet: \"%s\" %s\n"
1067 " Pool: \"%s\" (From %s)\n"
1068 " Storage: \"%s\" (From %s)\n"
1071 " Elapsed time: %s\n"
1073 " SD Files Written: %s\n"
1074 " SD Bytes Written: %s (%sB)\n"
1075 " Rate: %.1f KB/s\n"
1076 " Volume name(s): %s\n"
1077 " Volume Session Id: %d\n"
1078 " Volume Session Time: %d\n"
1079 " Last Volume Bytes: %s (%sB)\n"
1081 " SD termination status: %s\n"
1082 " Termination: %s\n\n"),
1086 mig_jcr ? edit_uint64(jcr->previous_jr.JobId, ec6) : "0",
1087 mig_jcr ? edit_uint64(mig_jcr->jr.JobId, ec7) : "0",
1088 edit_uint64(jcr->jr.JobId, ec8),
1090 level_to_str(jcr->JobLevel), jcr->since,
1091 jcr->client->name(),
1092 jcr->fileset->name(), jcr->FSCreateTime,
1093 jcr->pool->name(), jcr->pool_source,
1094 jcr->wstore->name(), jcr->storage_source,
1097 edit_utime(RunTime, elapsed, sizeof(elapsed)),
1099 edit_uint64_with_commas(jcr->SDJobFiles, ec1),
1100 edit_uint64_with_commas(jcr->SDJobBytes, ec2),
1101 edit_uint64_with_suffix(jcr->SDJobBytes, ec3),
1103 mig_jcr ? mig_jcr->VolumeName : "",
1105 jcr->VolSessionTime,
1106 edit_uint64_with_commas(mr.VolBytes, ec4),
1107 edit_uint64_with_suffix(mr.VolBytes, ec5),
1112 Dmsg1(100, "migrate_cleanup() mig_jcr=0x%x\n", jcr->mig_jcr);
1114 free_jcr(jcr->mig_jcr);
1115 jcr->mig_jcr = NULL;
1117 Dmsg0(100, "Leave migrate_cleanup()\n");
1121 * Return next DBId from comma separated list
1124 * 1 if next DBId returned
1125 * 0 if no more DBIds are in list
1126 * -1 there is an error
1128 static int get_next_dbid_from_list(char **p, DBId_t *DBId)
1134 for (int i=0; i<(int)sizeof(id); i++) {
1137 } else if (*q == ',') {
1146 } else if (!is_a_number(id)) {
1147 return -1; /* error */
1150 *DBId = str_to_int64(id);