3 * Bacula Director -- migrate.c -- responsible for doing
6 * Kern Sibbald, September MMIV
8 * Basic tasks done here:
9 * Open DB and create records for this job.
10 * Open Message Channel with Storage daemon to tell him a job will be starting.
11 * Open connection with Storage daemon and pass him commands
13 * When the Storage daemon finishes the job, update the DB.
18 Bacula® - The Network Backup Solution
20 Copyright (C) 2004-2006 Free Software Foundation Europe e.V.
22 The main author of Bacula is Kern Sibbald, with contributions from
23 many others, a complete list can be found in the file AUTHORS.
24 This program is Free Software; you can redistribute it and/or
25 modify it under the terms of version two of the GNU General Public
26 License as published by the Free Software Foundation plus additions
27 that are listed in the file LICENSE.
29 This program is distributed in the hope that it will be useful, but
30 WITHOUT ANY WARRANTY; without even the implied warranty of
31 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
32 General Public License for more details.
34 You should have received a copy of the GNU General Public License
35 along with this program; if not, write to the Free Software
36 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
39 Bacula® is a registered trademark of John Walker.
40 The licensor of Bacula is the Free Software Foundation Europe
41 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
42 Switzerland, email:ftf@fsfeurope.org.
49 #include "lib/bregex.h"
54 static const int dbglevel = 10;
56 static char OKbootstrap[] = "3000 OK bootstrap\n";
57 static int get_job_to_migrate(JCR *jcr);
59 static bool regex_find_jobids(JCR *jcr, idpkt *ids, const char *query1,
60 const char *query2, const char *type);
61 static bool find_mediaid_then_jobids(JCR *jcr, idpkt *ids, const char *query1,
63 static bool find_jobids_from_mediaid_list(JCR *jcr, idpkt *ids, const char *type);
64 static void start_migration_job(JCR *jcr);
65 static int get_next_dbid_from_list(char **p, DBId_t *DBId);
68 * Called here before the job is run to do the job
69 * specific setup. Note, one of the important things to
70 * complete in this init code is to make the definitive
71 * choice of input and output storage devices. This is
72 * because immediately after the init, the job is queued
73 * in the jobq.c code, and it checks that all the resources
74 * (storage resources in particular) are available, so these
75 * must all be properly defined.
77 * previous_jr refers to the job DB record of the Job that is
78 * going to be migrated.
79 * prev_job refers to the job resource of the Job that is
80 * going to be migrated.
81 * jcr is the jcr for the current "migration" job. It is a
82 * control job that is put in the DB as a migration job, which
83 * means that this job migrated a previous job to a new job.
84 * No Volume or File data is associated with this control
86 * mig_jcr refers to the newly migrated job that is run by
87 * the current jcr. It is a backup job that moves (migrates) the
88 * data written for the previous_jr into the new pool. This
89 * job (mig_jcr) becomes the new backup job that replaces
90 * the original backup job. Note, this jcr is not really run. It
91 * is simply attached to the current jcr. It will show up in
92 * the Director's status output, but not in the SD or FD, both of
93 * which deal only with the current migration job (i.e. jcr).
95 bool do_migration_init(JCR *jcr)
101 JCR *mig_jcr; /* newly migrated job */
105 apply_pool_overrides(jcr);
107 jcr->jr.PoolId = get_or_create_pool_record(jcr, jcr->pool->name());
108 if (jcr->jr.PoolId == 0) {
109 Dmsg1(dbglevel, "JobId=%d no PoolId\n", (int)jcr->JobId);
110 Jmsg(jcr, M_FATAL, 0, _("Could not get or create a Pool record.\n"));
114 * Note, at this point, pool is the pool for this job. We
115 * transfer it to rpool (read pool), and a bit later,
116 * pool will be changed to point to the write pool,
117 * which comes from pool->NextPool.
119 jcr->rpool = jcr->pool; /* save read pool */
120 pm_strcpy(jcr->rpool_source, jcr->pool_source);
123 Dmsg2(dbglevel, "Read pool=%s (From %s)\n", jcr->rpool->name(), jcr->rpool_source);
125 /* If we find a job or jobs to migrate it is previous_jr.JobId */
126 count = get_job_to_migrate(jcr);
134 Dmsg1(dbglevel, "Back from get_job_to_migrate JobId=%d\n", (int)jcr->JobId);
136 if (jcr->previous_jr.JobId == 0) {
137 Dmsg1(dbglevel, "JobId=%d no previous JobId\n", (int)jcr->JobId);
138 Jmsg(jcr, M_INFO, 0, _("No previous Job found to migrate.\n"));
139 return true; /* no work */
142 if (!get_or_create_fileset_record(jcr)) {
143 Dmsg1(dbglevel, "JobId=%d no FileSet\n", (int)jcr->JobId);
144 Jmsg(jcr, M_FATAL, 0, _("Could not get or create the FileSet record.\n"));
148 create_restore_bootstrap_file(jcr);
150 if (jcr->previous_jr.JobId == 0 || jcr->ExpectedFiles == 0) {
151 set_jcr_job_status(jcr, JS_Terminated);
152 Dmsg1(dbglevel, "JobId=%d expected files == 0\n", (int)jcr->JobId);
153 if (jcr->previous_jr.JobId == 0) {
154 Jmsg(jcr, M_INFO, 0, _("No previous Job found to migrate.\n"));
156 Jmsg(jcr, M_INFO, 0, _("Previous Job has no data to migrate.\n"));
158 return true; /* no work */
161 Dmsg5(dbglevel, "JobId=%d: Previous: Name=%s JobId=%d Type=%c Level=%c\n",
163 jcr->previous_jr.Name, (int)jcr->previous_jr.JobId,
164 jcr->previous_jr.JobType, jcr->previous_jr.JobLevel);
166 Dmsg5(dbglevel, "JobId=%d: Current: Name=%s JobId=%d Type=%c Level=%c\n",
168 jcr->jr.Name, (int)jcr->jr.JobId,
169 jcr->jr.JobType, jcr->jr.JobLevel);
172 job = (JOB *)GetResWithName(R_JOB, jcr->jr.Name);
173 prev_job = (JOB *)GetResWithName(R_JOB, jcr->previous_jr.Name);
176 Jmsg(jcr, M_FATAL, 0, _("Job resource not found for \"%s\".\n"), jcr->jr.Name);
180 Jmsg(jcr, M_FATAL, 0, _("Previous Job resource not found for \"%s\".\n"),
181 jcr->previous_jr.Name);
185 /* Create a migation jcr */
186 mig_jcr = jcr->mig_jcr = new_jcr(sizeof(JCR), dird_free_jcr);
187 memcpy(&mig_jcr->previous_jr, &jcr->previous_jr, sizeof(mig_jcr->previous_jr));
190 * Turn the mig_jcr into a "real" job that takes on the aspects of
191 * the previous backup job "prev_job".
193 set_jcr_defaults(mig_jcr, prev_job);
194 if (!setup_job(mig_jcr)) {
195 Jmsg(jcr, M_FATAL, 0, _("setup job failed.\n"));
199 /* Now reset the job record from the previous job */
200 memcpy(&mig_jcr->jr, &jcr->previous_jr, sizeof(mig_jcr->jr));
201 /* Update the jr to reflect the new values of PoolId, FileSetId, and JobId. */
202 mig_jcr->jr.PoolId = jcr->jr.PoolId;
203 mig_jcr->jr.FileSetId = jcr->jr.FileSetId;
204 mig_jcr->jr.JobId = mig_jcr->JobId;
206 Dmsg4(dbglevel, "mig_jcr: Name=%s JobId=%d Type=%c Level=%c\n",
207 mig_jcr->jr.Name, (int)mig_jcr->jr.JobId,
208 mig_jcr->jr.JobType, mig_jcr->jr.JobLevel);
211 * Get the PoolId used with the original job. Then
212 * find the pool name from the database record.
214 memset(&pr, 0, sizeof(pr));
215 pr.PoolId = mig_jcr->previous_jr.PoolId;
216 if (!db_get_pool_record(jcr, jcr->db, &pr)) {
217 Jmsg(jcr, M_FATAL, 0, _("Pool for JobId %s not in database. ERR=%s\n"),
218 edit_int64(pr.PoolId, ed1), db_strerror(jcr->db));
221 /* Get the pool resource corresponding to the original job */
222 pool = (POOL *)GetResWithName(R_POOL, pr.Name);
224 Jmsg(jcr, M_FATAL, 0, _("Pool resource \"%s\" not found.\n"), pr.Name);
228 /* If pool storage specified, use it for restore */
229 copy_rstorage(mig_jcr, pool->storage, _("Pool resource"));
230 copy_rstorage(jcr, pool->storage, _("Pool resource"));
233 * If the original backup pool has a NextPool, make sure a
234 * record exists in the database. Note, in this case, we
235 * will be migrating from pool to pool->NextPool.
237 if (pool->NextPool) {
238 jcr->jr.PoolId = get_or_create_pool_record(jcr, pool->NextPool->name());
239 if (jcr->jr.PoolId == 0) {
243 if (!set_migration_wstorage(jcr, pool)) {
246 mig_jcr->pool = jcr->pool = pool->NextPool;
247 pm_strcpy(jcr->pool_source, _("Job Pool's NextPool resource"));
248 mig_jcr->jr.PoolId = jcr->jr.PoolId;
250 Dmsg2(dbglevel, "Write pool=%s read rpool=%s\n", jcr->pool->name(), jcr->rpool->name());
255 * Do a Migration of a previous job
257 * Returns: false on failure
260 bool do_migration(JCR *jcr)
264 JCR *mig_jcr = jcr->mig_jcr; /* newly migrated job */
267 * If mig_jcr is NULL, there is nothing to do for this job,
268 * so set a normal status, cleanup and return OK.
271 set_jcr_job_status(jcr, JS_Terminated);
272 migration_cleanup(jcr, jcr->JobStatus);
276 /* Print Job Start message */
277 Jmsg(jcr, M_INFO, 0, _("Start Migration JobId %s, Job=%s\n"),
278 edit_uint64(jcr->JobId, ed1), jcr->Job);
280 set_jcr_job_status(jcr, JS_Running);
281 set_jcr_job_status(mig_jcr, JS_Running);
282 Dmsg2(dbglevel, "JobId=%d JobLevel=%c\n", (int)jcr->jr.JobId, jcr->jr.JobLevel);
284 /* Update job start record for this migration control job */
285 if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) {
286 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db));
290 Dmsg4(dbglevel, "mig_jcr: Name=%s JobId=%d Type=%c Level=%c\n",
291 mig_jcr->jr.Name, (int)mig_jcr->jr.JobId,
292 mig_jcr->jr.JobType, mig_jcr->jr.JobLevel);
294 /* Update job start record for the real migration backup job */
295 if (!db_update_job_start_record(mig_jcr, mig_jcr->db, &mig_jcr->jr)) {
296 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(mig_jcr->db));
302 * Open a message channel connection with the Storage
303 * daemon. This is to let him know that our client
304 * will be contacting him for a backup session.
307 Dmsg0(110, "Open connection with storage daemon\n");
308 set_jcr_job_status(jcr, JS_WaitSD);
309 set_jcr_job_status(mig_jcr, JS_WaitSD);
311 * Start conversation with Storage daemon
313 if (!connect_to_storage_daemon(jcr, 10, SDConnectTimeout, 1)) {
316 sd = jcr->store_bsock;
318 * Now start a job with the Storage daemon
320 Dmsg2(dbglevel, "Read store=%s, write store=%s\n",
321 ((STORE *)jcr->rstorage->first())->name(),
322 ((STORE *)jcr->wstorage->first())->name());
323 if (((STORE *)jcr->rstorage->first())->name() == ((STORE *)jcr->wstorage->first())->name()) {
324 Jmsg(jcr, M_FATAL, 0, _("Read storage \"%s\" same as write storage.\n"),
325 ((STORE *)jcr->rstorage->first())->name());
328 if (!start_storage_daemon_job(jcr, jcr->rstorage, jcr->wstorage)) {
331 Dmsg0(150, "Storage daemon connection OK\n");
333 if (!send_bootstrap_file(jcr, sd) ||
334 !response(jcr, sd, OKbootstrap, "Bootstrap", DISPLAY_ERROR)) {
338 if (!bnet_fsend(sd, "run")) {
343 * Now start a Storage daemon message thread
345 if (!start_storage_daemon_message_thread(jcr)) {
350 set_jcr_job_status(jcr, JS_Running);
351 set_jcr_job_status(mig_jcr, JS_Running);
353 /* Pickup Job termination data */
354 /* Note, the SD stores in jcr->JobFiles/ReadBytes/JobBytes/Errors */
355 wait_for_storage_daemon_termination(jcr);
357 set_jcr_job_status(jcr, jcr->SDJobStatus);
358 if (jcr->JobStatus != JS_Terminated) {
362 migration_cleanup(jcr, jcr->JobStatus);
364 UAContext *ua = new_ua_context(jcr);
365 purge_job_records_from_catalog(ua, jcr->previous_jr.JobId);
376 /* Add an item to the list if it is unique */
377 static void add_unique_id(idpkt *ids, char *item)
382 /* Walk through current list to see if each item is the same as item */
385 for (int i=0; i<(int)sizeof(id); i++) {
388 } else if (*q == ',') {
395 if (strcmp(item, id) == 0) {
399 /* Did not find item, so add it to list */
400 if (ids->count == 0) {
403 pm_strcat(ids->list, ",");
405 pm_strcat(ids->list, item);
407 // Dmsg3(0, "add_uniq count=%d Ids=%p %s\n", ids->count, ids->list, ids->list);
412 * Callback handler make list of DB Ids
414 static int unique_dbid_handler(void *ctx, int num_fields, char **row)
416 idpkt *ids = (idpkt *)ctx;
418 add_unique_id(ids, row[0]);
419 Dmsg3(dbglevel, "dbid_hdlr count=%d Ids=%p %s\n", ids->count, ids->list, ids->list);
429 static int item_compare(void *item1, void *item2)
431 uitem *i1 = (uitem *)item1;
432 uitem *i2 = (uitem *)item2;
433 return strcmp(i1->item, i2->item);
436 static int unique_name_handler(void *ctx, int num_fields, char **row)
438 dlist *list = (dlist *)ctx;
440 uitem *new_item = (uitem *)malloc(sizeof(uitem));
443 memset(new_item, 0, sizeof(uitem));
444 new_item->item = bstrdup(row[0]);
445 Dmsg1(dbglevel, "Unique_name_hdlr Item=%s\n", row[0]);
446 item = (uitem *)list->binary_insert((void *)new_item, item_compare);
447 if (item != new_item) { /* already in list */
448 free(new_item->item);
449 free((char *)new_item);
455 /* Get Job names in Pool */
456 const char *sql_job =
457 "SELECT DISTINCT Job.Name from Job,Pool"
458 " WHERE Pool.Name='%s' AND Job.PoolId=Pool.PoolId";
460 /* Get JobIds from regex'ed Job names */
461 const char *sql_jobids_from_job =
462 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Job,Pool"
463 " WHERE Job.Name='%s' AND Pool.Name='%s' AND Job.PoolId=Pool.PoolId"
464 " ORDER by Job.StartTime";
466 /* Get Client names in Pool */
467 const char *sql_client =
468 "SELECT DISTINCT Client.Name from Client,Pool,Job"
469 " WHERE Pool.Name='%s' AND Job.ClientId=Client.ClientId AND"
470 " Job.PoolId=Pool.PoolId";
472 /* Get JobIds from regex'ed Client names */
473 const char *sql_jobids_from_client =
474 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Job,Pool,Client"
475 " WHERE Client.Name='%s' AND Pool.Name='%s' AND Job.PoolId=Pool.PoolId"
476 " AND Job.ClientId=Client.ClientId "
477 " ORDER by Job.StartTime";
479 /* Get Volume names in Pool */
480 const char *sql_vol =
481 "SELECT DISTINCT VolumeName FROM Media,Pool WHERE"
482 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
483 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'";
485 /* Get JobIds from regex'ed Volume names */
486 const char *sql_jobids_from_vol =
487 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Media,JobMedia,Job"
488 " WHERE Media.VolumeName='%s' AND Media.MediaId=JobMedia.MediaId"
489 " AND JobMedia.JobId=Job.JobId"
490 " ORDER by Job.StartTime";
493 const char *sql_smallest_vol =
494 "SELECT MediaId FROM Media,Pool WHERE"
495 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
496 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'"
497 " ORDER BY VolBytes ASC LIMIT 1";
499 const char *sql_oldest_vol =
500 "SELECT MediaId FROM Media,Pool WHERE"
501 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
502 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'"
503 " ORDER BY LastWritten ASC LIMIT 1";
505 /* Get JobIds when we have selected MediaId */
506 const char *sql_jobids_from_mediaid =
507 "SELECT DISTINCT Job.JobId,Job.StartTime FROM JobMedia,Job"
508 " WHERE JobMedia.JobId=Job.JobId AND JobMedia.MediaId=%s"
509 " ORDER by Job.StartTime";
511 /* Get tne number of bytes in the pool */
512 const char *sql_pool_bytes =
513 "SELECT SUM(VolBytes) FROM Media,Pool WHERE"
514 " VolStatus in ('Full','Used','Error','Append') AND Media.Enabled=1 AND"
515 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'";
517 /* Get tne number of bytes in the Jobs */
518 const char *sql_job_bytes =
519 "SELECT SUM(JobBytes) FROM Job WHERE JobId IN (%s)";
522 /* Get Media Ids in Pool */
523 const char *sql_mediaids =
524 "SELECT MediaId FROM Media,Pool WHERE"
525 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
526 " Media.PoolId=Pool.PoolId AND Pool.Name='%s' ORDER BY LastWritten ASC";
528 /* Get JobIds in Pool longer than specified time */
529 const char *sql_pool_time =
530 "SELECT DISTINCT Job.JobId from Pool,Job,Media,JobMedia WHERE"
531 " Pool.Name='%s' AND Media.PoolId=Pool.PoolId AND"
532 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
533 " JobMedia.JobId=Job.JobId AND Job.PoolId=Media.PoolId"
534 " AND Job.RealEndTime<='%s'";
537 * const char *sql_ujobid =
538 * "SELECT DISTINCT Job.Job from Client,Pool,Media,Job,JobMedia "
539 * " WHERE Media.PoolId=Pool.PoolId AND Pool.Name='%s' AND"
540 * " JobMedia.JobId=Job.JobId AND Job.PoolId=Media.PoolId";
547 * This is the central piece of code that finds a job or jobs
548 * actually JobIds to migrate. It first looks to see if one
549 * has been "manually" specified in jcr->MigrateJobId, and if
550 * so, it returns that JobId to be run. Otherwise, it
551 * examines the Selection Type to see what kind of migration
552 * we are doing (Volume, Job, Client, ...) and applies any
553 * Selection Pattern if appropriate to obtain a list of JobIds.
554 * Finally, it will loop over all the JobIds found, except the last
555 * one starting a new job with MigrationJobId set to that JobId, and
556 * finally, it returns the last JobId to the caller.
558 * Returns: -1 on error
559 * 0 if no jobs to migrate
560 * 1 if OK and jcr->previous_jr filled in
562 static int get_job_to_migrate(JCR *jcr)
565 POOL_MEM query(PM_MESSAGE);
570 idpkt ids, mid, jids;
575 char dt[MAX_TIME_LENGTH];
578 ids.list = get_pool_memory(PM_MESSAGE);
581 mid.list = get_pool_memory(PM_MESSAGE);
584 jids.list = get_pool_memory(PM_MESSAGE);
590 * If MigrateJobId is set, then we migrate only that Job,
591 * otherwise, we go through the full selection of jobs to
594 if (jcr->MigrateJobId != 0) {
595 Dmsg1(dbglevel, "At Job start previous jobid=%u\n", jcr->MigrateJobId);
596 edit_uint64(jcr->MigrateJobId, ids.list);
599 switch (jcr->job->selection_type) {
601 if (!regex_find_jobids(jcr, &ids, sql_job, sql_jobids_from_job, "Job")) {
606 if (!regex_find_jobids(jcr, &ids, sql_client, sql_jobids_from_client, "Client")) {
611 if (!regex_find_jobids(jcr, &ids, sql_vol, sql_jobids_from_vol, "Volume")) {
616 if (!jcr->job->selection_pattern) {
617 Jmsg(jcr, M_FATAL, 0, _("No Migration SQL selection pattern specified.\n"));
620 Dmsg1(dbglevel, "SQL=%s\n", jcr->job->selection_pattern);
621 if (!db_sql_query(jcr->db, jcr->job->selection_pattern,
622 unique_dbid_handler, (void *)&ids)) {
623 Jmsg(jcr, M_FATAL, 0,
624 _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
628 case MT_SMALLEST_VOL:
629 if (!find_mediaid_then_jobids(jcr, &ids, sql_smallest_vol, "Smallest Volume")) {
634 if (!find_mediaid_then_jobids(jcr, &ids, sql_oldest_vol, "Oldest Volume")) {
639 case MT_POOL_OCCUPANCY:
641 /* Find count of bytes in pool */
642 Mmsg(query, sql_pool_bytes, jcr->rpool->name());
643 if (!db_sql_query(jcr->db, query.c_str(), db_int64_handler, (void *)&ctx)) {
644 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
647 if (ctx.count == 0) {
648 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
651 pool_bytes = ctx.value;
652 Dmsg2(dbglevel, "highbytes=%d pool=%d\n", (int)jcr->rpool->MigrationHighBytes,
654 if (pool_bytes < (int64_t)jcr->rpool->MigrationHighBytes) {
655 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
658 Dmsg0(dbglevel, "We should do Occupation migration.\n");
661 /* Find a list of MediaIds that could be migrated */
662 Mmsg(query, sql_mediaids, jcr->rpool->name());
663 Dmsg1(dbglevel, "query=%s\n", query.c_str());
664 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)&ids)) {
665 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
668 if (ids.count == 0) {
669 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
672 Dmsg2(dbglevel, "Pool Occupancy ids=%d MediaIds=%s\n", ids.count, ids.list);
675 * Now loop over MediaIds getting more JobIds to migrate until
676 * we reduce the pool occupancy below the low water mark.
679 for (int i=0; i < (int)ids.count; i++) {
680 stat = get_next_dbid_from_list(&p, &MediaId);
681 Dmsg2(dbglevel, "get_next_dbid stat=%d MediaId=%u\n", stat, MediaId);
683 Jmsg(jcr, M_FATAL, 0, _("Invalid MediaId found.\n"));
685 } else if (stat == 0) {
689 Mmsg(mid.list, "%s", edit_int64(MediaId, ed1));
690 if (!find_jobids_from_mediaid_list(jcr, &mid, "Volumes")) {
694 pm_strcat(jids.list, ",");
696 pm_strcat(jids.list, mid.list);
697 jids.count += mid.count;
699 /* Now get the count of bytes added */
701 /* Find count of bytes from Jobs */
702 Mmsg(query, sql_job_bytes, mid.list);
703 if (!db_sql_query(jcr->db, query.c_str(), db_int64_handler, (void *)&ctx)) {
704 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
707 pool_bytes -= ctx.value;
708 Dmsg1(dbglevel, "Job bytes=%d\n", (int)ctx.value);
709 Dmsg2(dbglevel, "lowbytes=%d pool=%d\n", (int)jcr->rpool->MigrationLowBytes,
711 if (pool_bytes <= (int64_t)jcr->rpool->MigrationLowBytes) {
712 Dmsg0(dbglevel, "We should be done.\n");
717 Dmsg2(dbglevel, "Pool Occupancy ids=%d JobIds=%s\n", jids.count, jids.list);
722 ttime = time(NULL) - (time_t)jcr->rpool->MigrationTime;
723 (void)localtime_r(&ttime, &tm);
724 strftime(dt, sizeof(dt), "%Y-%m-%d %H:%M:%S", &tm);
727 Mmsg(query, sql_pool_time, jcr->rpool->name(), dt);
728 Dmsg1(dbglevel, "query=%s\n", query.c_str());
729 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)&ids)) {
730 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
733 if (ids.count == 0) {
734 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
737 Dmsg2(dbglevel, "PoolTime ids=%d JobIds=%s\n", ids.count, ids.list);
741 Jmsg(jcr, M_FATAL, 0, _("Unknown Migration Selection Type.\n"));
747 * Loop over all jobids except the last one, sending
748 * them to start_migration_job(), which will start a job
749 * for each of them. For the last JobId, we handle it below.
752 if (ids.count == 0) {
753 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
756 Jmsg(jcr, M_INFO, 0, _("The following %u JobId%s will be migrated: %s\n"),
757 ids.count, ids.count==0?"":"s", ids.list);
758 Dmsg2(dbglevel, "Before loop count=%d ids=%s\n", ids.count, ids.list);
759 for (int i=1; i < (int)ids.count; i++) {
761 stat = get_next_jobid_from_list(&p, &JobId);
762 Dmsg3(dbglevel, "get_jobid_no=%d stat=%d JobId=%u\n", i, stat, JobId);
763 jcr->MigrateJobId = JobId;
764 start_migration_job(jcr);
765 Dmsg0(dbglevel, "Back from start_migration_job\n");
767 Jmsg(jcr, M_FATAL, 0, _("Invalid JobId found.\n"));
769 } else if (stat == 0) {
770 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
775 /* Now get the last JobId and handle it in the current job */
777 stat = get_next_jobid_from_list(&p, &JobId);
778 Dmsg2(dbglevel, "Last get_next_jobid stat=%d JobId=%u\n", stat, (int)JobId);
780 Jmsg(jcr, M_FATAL, 0, _("Invalid JobId found.\n"));
782 } else if (stat == 0) {
783 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
787 jcr->previous_jr.JobId = JobId;
788 Dmsg1(dbglevel, "Previous jobid=%d\n", (int)jcr->previous_jr.JobId);
790 if (!db_get_job_record(jcr, jcr->db, &jcr->previous_jr)) {
791 Jmsg(jcr, M_FATAL, 0, _("Could not get job record for JobId %s to migrate. ERR=%s"),
792 edit_int64(jcr->previous_jr.JobId, ed1),
793 db_strerror(jcr->db));
796 Jmsg(jcr, M_INFO, 0, _("Migration using JobId=%s Job=%s\n"),
797 edit_int64(jcr->previous_jr.JobId, ed1), jcr->previous_jr.Job);
798 Dmsg3(dbglevel, "Migration JobId=%d using JobId=%s Job=%s\n",
800 edit_int64(jcr->previous_jr.JobId, ed1), jcr->previous_jr.Job);
810 free_pool_memory(ids.list);
811 free_pool_memory(mid.list);
812 free_pool_memory(jids.list);
816 static void start_migration_job(JCR *jcr)
818 UAContext *ua = new_ua_context(jcr);
821 Mmsg(ua->cmd, "run %s jobid=%s", jcr->job->hdr.name,
822 edit_uint64(jcr->MigrateJobId, ed1));
823 Dmsg1(dbglevel, "=============== Migration cmd=%s\n", ua->cmd);
824 parse_ua_args(ua); /* parse command */
825 int stat = run_cmd(ua, ua->cmd);
827 Jmsg(jcr, M_ERROR, 0, _("Could not start migration job.\n"));
829 Jmsg(jcr, M_INFO, 0, _("Migration JobId %d started.\n"), stat);
834 static bool find_mediaid_then_jobids(JCR *jcr, idpkt *ids, const char *query1,
838 POOL_MEM query(PM_MESSAGE);
841 /* Basic query for MediaId */
842 Mmsg(query, query1, jcr->rpool->name());
843 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
844 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
847 if (ids->count == 0) {
848 Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type);
850 if (ids->count != 1) {
851 Jmsg(jcr, M_FATAL, 0, _("SQL logic error. Count should be 1 but is %d\n"),
855 Dmsg1(dbglevel, "Smallest Vol Jobids=%s\n", ids->list);
857 ok = find_jobids_from_mediaid_list(jcr, ids, type);
863 static bool find_jobids_from_mediaid_list(JCR *jcr, idpkt *ids, const char *type)
866 POOL_MEM query(PM_MESSAGE);
868 Mmsg(query, sql_jobids_from_mediaid, ids->list);
870 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
871 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
874 if (ids->count == 0) {
875 Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type);
882 static bool regex_find_jobids(JCR *jcr, idpkt *ids, const char *query1,
883 const char *query2, const char *type)
887 uitem *last_item = NULL;
892 POOL_MEM query(PM_MESSAGE);
894 item_chain = New(dlist(item, &item->link));
895 if (!jcr->job->selection_pattern) {
896 Jmsg(jcr, M_FATAL, 0, _("No Migration %s selection pattern specified.\n"),
900 Dmsg1(dbglevel, "regex-sel-pattern=%s\n", jcr->job->selection_pattern);
901 /* Basic query for names */
902 Mmsg(query, query1, jcr->rpool->name());
903 Dmsg1(dbglevel, "get name query1=%s\n", query.c_str());
904 if (!db_sql_query(jcr->db, query.c_str(), unique_name_handler,
905 (void *)item_chain)) {
906 Jmsg(jcr, M_FATAL, 0,
907 _("SQL to get %s failed. ERR=%s\n"), type, db_strerror(jcr->db));
910 Dmsg1(dbglevel, "query1 returned %d names\n", item_chain->size());
911 if (item_chain->size() == 0) {
912 Jmsg(jcr, M_INFO, 0, _("Query of Pool \"%s\" returned no Jobs to migrate.\n"),
915 goto bail_out; /* skip regex match */
917 /* Compile regex expression */
918 rc = regcomp(&preg, jcr->job->selection_pattern, REG_EXTENDED);
920 regerror(rc, &preg, prbuf, sizeof(prbuf));
921 Jmsg(jcr, M_FATAL, 0, _("Could not compile regex pattern \"%s\" ERR=%s\n"),
922 jcr->job->selection_pattern, prbuf);
925 /* Now apply the regex to the names and remove any item not matched */
926 foreach_dlist(item, item_chain) {
927 const int nmatch = 30;
928 regmatch_t pmatch[nmatch];
930 Dmsg1(dbglevel, "Remove item %s\n", last_item->item);
931 free(last_item->item);
932 item_chain->remove(last_item);
934 Dmsg1(dbglevel, "get name Item=%s\n", item->item);
935 rc = regexec(&preg, item->item, nmatch, pmatch, 0);
937 last_item = NULL; /* keep this one */
943 free(last_item->item);
944 Dmsg1(dbglevel, "Remove item %s\n", last_item->item);
945 item_chain->remove(last_item);
949 if (item_chain->size() == 0) {
950 Jmsg(jcr, M_INFO, 0, _("Regex pattern matched no Jobs to migrate.\n"));
952 goto bail_out; /* skip regex match */
956 * At this point, we have a list of items in item_chain
957 * that have been matched by the regex, so now we need
958 * to look up their jobids.
961 foreach_dlist(item, item_chain) {
962 Dmsg2(dbglevel, "Got %s: %s\n", type, item->item);
963 Mmsg(query, query2, item->item, jcr->rpool->name());
964 Dmsg1(dbglevel, "get id from name query2=%s\n", query.c_str());
965 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
966 Jmsg(jcr, M_FATAL, 0,
967 _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
971 if (ids->count == 0) {
972 Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type);
977 Dmsg2(dbglevel, "Count=%d Jobids=%s\n", ids->count, ids->list);
984 * Release resources allocated during backup.
986 void migration_cleanup(JCR *jcr, int TermCode)
988 char sdt[MAX_TIME_LENGTH], edt[MAX_TIME_LENGTH];
989 char ec1[30], ec2[30], ec3[30], ec4[30], ec5[30], elapsed[50];
990 char ec6[50], ec7[50], ec8[50];
991 char term_code[100], sd_term_msg[100];
992 const char *term_msg;
993 int msg_type = M_INFO;
997 JCR *mig_jcr = jcr->mig_jcr;
998 POOL_MEM query(PM_MESSAGE);
1000 Dmsg2(100, "Enter migrate_cleanup %d %c\n", TermCode, TermCode);
1001 update_job_end(jcr, TermCode);
1002 memset(&mr, 0, sizeof(mr));
1005 * Check if we actually did something.
1006 * mig_jcr is jcr of the newly migrated job.
1009 mig_jcr->JobFiles = jcr->JobFiles = jcr->SDJobFiles;
1010 mig_jcr->JobBytes = jcr->JobBytes = jcr->SDJobBytes;
1011 mig_jcr->VolSessionId = jcr->VolSessionId;
1012 mig_jcr->VolSessionTime = jcr->VolSessionTime;
1013 mig_jcr->jr.RealEndTime = 0;
1014 mig_jcr->jr.PriorJobId = jcr->previous_jr.JobId;
1016 update_job_end(mig_jcr, TermCode);
1018 /* Update final items to set them to the previous job's values */
1019 Mmsg(query, "UPDATE Job SET StartTime='%s',EndTime='%s',"
1020 "JobTDate=%s WHERE JobId=%s",
1021 jcr->previous_jr.cStartTime, jcr->previous_jr.cEndTime,
1022 edit_uint64(jcr->previous_jr.JobTDate, ec1),
1023 edit_uint64(mig_jcr->jr.JobId, ec2));
1024 db_sql_query(mig_jcr->db, query.c_str(), NULL, NULL);
1026 /* Now mark the previous job as migrated if it terminated normally */
1027 if (jcr->JobStatus == JS_Terminated) {
1028 Mmsg(query, "UPDATE Job SET Type='%c' WHERE JobId=%s",
1029 (char)JT_MIGRATED_JOB, edit_uint64(jcr->previous_jr.JobId, ec1));
1030 db_sql_query(mig_jcr->db, query.c_str(), NULL, NULL);
1033 if (!db_get_job_record(jcr, jcr->db, &jcr->jr)) {
1034 Jmsg(jcr, M_WARNING, 0, _("Error getting job record for stats: %s"),
1035 db_strerror(jcr->db));
1036 set_jcr_job_status(jcr, JS_ErrorTerminated);
1039 bstrncpy(mr.VolumeName, jcr->VolumeName, sizeof(mr.VolumeName));
1040 if (!db_get_media_record(jcr, jcr->db, &mr)) {
1041 Jmsg(jcr, M_WARNING, 0, _("Error getting Media record for Volume \"%s\": ERR=%s"),
1042 mr.VolumeName, db_strerror(jcr->db));
1043 set_jcr_job_status(jcr, JS_ErrorTerminated);
1046 update_bootstrap_file(mig_jcr);
1048 if (!db_get_job_volume_names(mig_jcr, mig_jcr->db, mig_jcr->jr.JobId, &mig_jcr->VolumeName)) {
1050 * Note, if the job has failed, most likely it did not write any
1051 * tape, so suppress this "error" message since in that case
1052 * it is normal. Or look at it the other way, only for a
1053 * normal exit should we complain about this error.
1055 if (jcr->JobStatus == JS_Terminated && jcr->jr.JobBytes) {
1056 Jmsg(jcr, M_ERROR, 0, "%s", db_strerror(mig_jcr->db));
1058 mig_jcr->VolumeName[0] = 0; /* none */
1060 switch (jcr->JobStatus) {
1062 if (jcr->Errors || jcr->SDErrors) {
1063 term_msg = _("%s OK -- with warnings");
1065 term_msg = _("%s OK");
1069 case JS_ErrorTerminated:
1070 term_msg = _("*** %s Error ***");
1071 msg_type = M_ERROR; /* Generate error message */
1072 if (jcr->store_bsock) {
1073 bnet_sig(jcr->store_bsock, BNET_TERMINATE);
1074 if (jcr->SD_msg_chan) {
1075 pthread_cancel(jcr->SD_msg_chan);
1080 term_msg = _("%s Canceled");
1081 if (jcr->store_bsock) {
1082 bnet_sig(jcr->store_bsock, BNET_TERMINATE);
1083 if (jcr->SD_msg_chan) {
1084 pthread_cancel(jcr->SD_msg_chan);
1089 term_msg = _("Inappropriate %s term code");
1093 term_msg = _("%s -- no files to migrate");
1096 bsnprintf(term_code, sizeof(term_code), term_msg, "Migration");
1097 bstrftimes(sdt, sizeof(sdt), jcr->jr.StartTime);
1098 bstrftimes(edt, sizeof(edt), jcr->jr.EndTime);
1099 RunTime = jcr->jr.EndTime - jcr->jr.StartTime;
1103 kbps = (double)jcr->SDJobBytes / (1000 * RunTime);
1107 jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg));
1109 Jmsg(jcr, msg_type, 0, _("Bacula %s (%s): %s\n"
1110 " Prev Backup JobId: %s\n"
1111 " New Backup JobId: %s\n"
1112 " Migration JobId: %s\n"
1113 " Migration Job: %s\n"
1114 " Backup Level: %s%s\n"
1116 " FileSet: \"%s\" %s\n"
1117 " Read Pool: \"%s\" (From %s)\n"
1118 " Read Storage: \"%s\" (From %s)\n"
1119 " Write Pool: \"%s\" (From %s)\n"
1120 " Write Storage: \"%s\" (From %s)\n"
1123 " Elapsed time: %s\n"
1125 " SD Files Written: %s\n"
1126 " SD Bytes Written: %s (%sB)\n"
1127 " Rate: %.1f KB/s\n"
1128 " Volume name(s): %s\n"
1129 " Volume Session Id: %d\n"
1130 " Volume Session Time: %d\n"
1131 " Last Volume Bytes: %s (%sB)\n"
1133 " SD termination status: %s\n"
1134 " Termination: %s\n\n"),
1138 edit_uint64(jcr->previous_jr.JobId, ec6),
1139 mig_jcr ? edit_uint64(mig_jcr->jr.JobId, ec7) : "0",
1140 edit_uint64(jcr->jr.JobId, ec8),
1142 level_to_str(jcr->JobLevel), jcr->since,
1143 jcr->client->name(),
1144 jcr->fileset->name(), jcr->FSCreateTime,
1145 jcr->rpool->name(), jcr->rpool_source,
1146 jcr->rstore?jcr->rstore->name():"*None*",
1147 NPRT(jcr->rstore_source),
1148 jcr->pool->name(), jcr->pool_source,
1149 jcr->wstore?jcr->wstore->name():"*None*",
1150 NPRT(jcr->wstore_source),
1153 edit_utime(RunTime, elapsed, sizeof(elapsed)),
1155 edit_uint64_with_commas(jcr->SDJobFiles, ec1),
1156 edit_uint64_with_commas(jcr->SDJobBytes, ec2),
1157 edit_uint64_with_suffix(jcr->SDJobBytes, ec3),
1159 mig_jcr ? mig_jcr->VolumeName : "",
1161 jcr->VolSessionTime,
1162 edit_uint64_with_commas(mr.VolBytes, ec4),
1163 edit_uint64_with_suffix(mr.VolBytes, ec5),
1168 Dmsg1(100, "migrate_cleanup() mig_jcr=0x%x\n", jcr->mig_jcr);
1170 free_jcr(jcr->mig_jcr);
1171 jcr->mig_jcr = NULL;
1173 Dmsg0(100, "Leave migrate_cleanup()\n");
1177 * Return next DBId from comma separated list
1180 * 1 if next DBId returned
1181 * 0 if no more DBIds are in list
1182 * -1 there is an error
1184 static int get_next_dbid_from_list(char **p, DBId_t *DBId)
1190 for (int i=0; i<(int)sizeof(id); i++) {
1193 } else if (*q == ',') {
1202 } else if (!is_a_number(id)) {
1203 return -1; /* error */
1206 *DBId = str_to_int64(id);
1210 bool set_migration_wstorage(JCR *jcr, POOL *pool)
1212 POOL *wpool = pool->NextPool;
1215 Jmsg(jcr, M_FATAL, 0, _("No Next Pool specification found in Pool \"%s\".\n"),
1220 if (!wpool->storage || wpool->storage->size() == 0) {
1221 Jmsg(jcr, M_FATAL, 0, _("No Storage specification found in Next Pool \"%s\".\n"),
1226 /* If pool storage specified, use it instead of job storage for backup */
1227 copy_wstorage(jcr, wpool->storage, _("Storage from Pool's NextPool resource"));