3 * Bacula Director -- migrate.c -- responsible for doing
6 * Kern Sibbald, September MMIV
8 * Basic tasks done here:
9 * Open DB and create records for this job.
10 * Open Message Channel with Storage daemon to tell him a job will be starting.
11 * Open connection with Storage daemon and pass him commands
13 * When the Storage daemon finishes the job, update the DB.
18 Bacula® - The Network Backup Solution
20 Copyright (C) 2004-2006 Free Software Foundation Europe e.V.
22 The main author of Bacula is Kern Sibbald, with contributions from
23 many others, a complete list can be found in the file AUTHORS.
24 This program is Free Software; you can redistribute it and/or
25 modify it under the terms of version two of the GNU General Public
26 License as published by the Free Software Foundation plus additions
27 that are listed in the file LICENSE.
29 This program is distributed in the hope that it will be useful, but
30 WITHOUT ANY WARRANTY; without even the implied warranty of
31 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
32 General Public License for more details.
34 You should have received a copy of the GNU General Public License
35 along with this program; if not, write to the Free Software
36 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
39 Bacula® is a registered trademark of John Walker.
40 The licensor of Bacula is the Free Software Foundation Europe
41 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
42 Switzerland, email:ftf@fsfeurope.org.
49 #include "lib/bregex.h"
54 static const int dbglevel = 10;
56 static char OKbootstrap[] = "3000 OK bootstrap\n";
57 static int get_job_to_migrate(JCR *jcr);
59 static bool regex_find_jobids(JCR *jcr, idpkt *ids, const char *query1,
60 const char *query2, const char *type);
61 static bool find_mediaid_then_jobids(JCR *jcr, idpkt *ids, const char *query1,
63 static bool find_jobids_from_mediaid_list(JCR *jcr, idpkt *ids, const char *type);
64 static void start_migration_job(JCR *jcr);
65 static int get_next_dbid_from_list(char **p, DBId_t *DBId);
68 * Called here before the job is run to do the job
69 * specific setup. Note, one of the important things to
70 * complete in this init code is to make the definitive
71 * choice of input and output storage devices. This is
72 * because immediately after the init, the job is queued
73 * in the jobq.c code, and it checks that all the resources
74 * (storage resources in particular) are available, so these
75 * must all be properly defined.
77 * previous_jr refers to the job DB record of the Job that is
78 * going to be migrated.
79 * prev_job refers to the job resource of the Job that is
80 * going to be migrated.
81 * jcr is the jcr for the current "migration" job. It is a
82 * control job that is put in the DB as a migration job, which
83 * means that this job migrated a previous job to a new job.
84 * No Volume or File data is associated with this control
86 * mig_jcr refers to the newly migrated job that is run by
87 * the current jcr. It is a backup job that moves (migrates) the
88 * data written for the previous_jr into the new pool. This
89 * job (mig_jcr) becomes the new backup job that replaces
90 * the original backup job. Note, this jcr is not really run. It
91 * is simply attached to the current jcr. It will show up in
92 * the Director's status output, but not in the SD or FD, both of
93 * which deal only with the current migration job (i.e. jcr).
95 bool do_migration_init(JCR *jcr)
101 JCR *mig_jcr; /* newly migrated job */
105 apply_pool_overrides(jcr);
107 jcr->jr.PoolId = get_or_create_pool_record(jcr, jcr->pool->name());
108 if (jcr->jr.PoolId == 0) {
109 Dmsg1(dbglevel, "JobId=%d no PoolId\n", (int)jcr->JobId);
110 Jmsg(jcr, M_FATAL, 0, _("Could not get or create a Pool record.\n"));
114 * Note, at this point, pool is the pool for this job. We
115 * transfer it to rpool (read pool), and a bit later,
116 * pool will be changed to point to the write pool,
117 * which comes from pool->NextPool.
119 jcr->rpool = jcr->pool; /* save read pool */
120 pm_strcpy(jcr->rpool_source, jcr->pool_source);
123 Dmsg2(dbglevel, "Read pool=%s (From %s)\n", jcr->rpool->name(), jcr->rpool_source);
125 /* If we find a job or jobs to migrate it is previous_jr.JobId */
126 count = get_job_to_migrate(jcr);
134 Dmsg1(dbglevel, "Back from get_job_to_migrate JobId=%d\n", (int)jcr->JobId);
136 if (jcr->previous_jr.JobId == 0) {
137 Dmsg1(dbglevel, "JobId=%d no previous JobId\n", (int)jcr->JobId);
138 Jmsg(jcr, M_INFO, 0, _("No previous Job found to migrate.\n"));
139 return true; /* no work */
142 if (!get_or_create_fileset_record(jcr)) {
143 Dmsg1(dbglevel, "JobId=%d no FileSet\n", (int)jcr->JobId);
144 Jmsg(jcr, M_FATAL, 0, _("Could not get or create the FileSet record.\n"));
148 create_restore_bootstrap_file(jcr);
150 if (jcr->previous_jr.JobId == 0 || jcr->ExpectedFiles == 0) {
151 set_jcr_job_status(jcr, JS_Terminated);
152 Dmsg1(dbglevel, "JobId=%d expected files == 0\n", (int)jcr->JobId);
153 if (jcr->previous_jr.JobId == 0) {
154 Jmsg(jcr, M_INFO, 0, _("No previous Job found to migrate.\n"));
156 Jmsg(jcr, M_INFO, 0, _("Previous Job has no data to migrate.\n"));
158 return true; /* no work */
161 Dmsg5(dbglevel, "JobId=%d: Previous: Name=%s JobId=%d Type=%c Level=%c\n",
163 jcr->previous_jr.Name, (int)jcr->previous_jr.JobId,
164 jcr->previous_jr.JobType, jcr->previous_jr.JobLevel);
166 Dmsg5(dbglevel, "JobId=%d: Current: Name=%s JobId=%d Type=%c Level=%c\n",
168 jcr->jr.Name, (int)jcr->jr.JobId,
169 jcr->jr.JobType, jcr->jr.JobLevel);
172 job = (JOB *)GetResWithName(R_JOB, jcr->jr.Name);
173 prev_job = (JOB *)GetResWithName(R_JOB, jcr->previous_jr.Name);
176 Jmsg(jcr, M_FATAL, 0, _("Job resource not found for \"%s\".\n"), jcr->jr.Name);
180 Jmsg(jcr, M_FATAL, 0, _("Previous Job resource not found for \"%s\".\n"),
181 jcr->previous_jr.Name);
185 /* Create a migation jcr */
186 mig_jcr = jcr->mig_jcr = new_jcr(sizeof(JCR), dird_free_jcr);
187 memcpy(&mig_jcr->previous_jr, &jcr->previous_jr, sizeof(mig_jcr->previous_jr));
190 * Turn the mig_jcr into a "real" job that takes on the aspects of
191 * the previous backup job "prev_job".
193 set_jcr_defaults(mig_jcr, prev_job);
194 if (!setup_job(mig_jcr)) {
195 Jmsg(jcr, M_FATAL, 0, _("setup job failed.\n"));
199 /* Now reset the job record from the previous job */
200 memcpy(&mig_jcr->jr, &jcr->previous_jr, sizeof(mig_jcr->jr));
201 /* Update the jr to reflect the new values of PoolId, FileSetId, and JobId. */
202 mig_jcr->jr.PoolId = jcr->jr.PoolId;
203 mig_jcr->jr.FileSetId = jcr->jr.FileSetId;
204 mig_jcr->jr.JobId = mig_jcr->JobId;
206 Dmsg4(dbglevel, "mig_jcr: Name=%s JobId=%d Type=%c Level=%c\n",
207 mig_jcr->jr.Name, (int)mig_jcr->jr.JobId,
208 mig_jcr->jr.JobType, mig_jcr->jr.JobLevel);
211 * Get the PoolId used with the original job. Then
212 * find the pool name from the database record.
214 memset(&pr, 0, sizeof(pr));
215 pr.PoolId = mig_jcr->previous_jr.PoolId;
216 if (!db_get_pool_record(jcr, jcr->db, &pr)) {
217 Jmsg(jcr, M_FATAL, 0, _("Pool for JobId %s not in database. ERR=%s\n"),
218 edit_int64(pr.PoolId, ed1), db_strerror(jcr->db));
221 /* Get the pool resource corresponding to the original job */
222 pool = (POOL *)GetResWithName(R_POOL, pr.Name);
224 Jmsg(jcr, M_FATAL, 0, _("Pool resource \"%s\" not found.\n"), pr.Name);
228 /* If pool storage specified, use it for restore */
229 copy_rstorage(mig_jcr, pool->storage, _("Pool resource"));
230 copy_rstorage(jcr, pool->storage, _("Pool resource"));
233 * If the original backup pool has a NextPool, make sure a
234 * record exists in the database. Note, in this case, we
235 * will be migrating from pool to pool->NextPool.
237 if (pool->NextPool) {
238 jcr->jr.PoolId = get_or_create_pool_record(jcr, pool->NextPool->name());
239 if (jcr->jr.PoolId == 0) {
243 if (!set_migration_wstorage(jcr, pool)) {
246 mig_jcr->pool = jcr->pool = pool->NextPool;
247 pm_strcpy(jcr->pool_source, _("Job Pool's NextPool resource"));
248 mig_jcr->jr.PoolId = jcr->jr.PoolId;
250 Dmsg2(dbglevel, "Write pool=%s read rpool=%s\n", jcr->pool->name(), jcr->rpool->name());
255 * Do a Migration of a previous job
257 * Returns: false on failure
260 bool do_migration(JCR *jcr)
264 JCR *mig_jcr = jcr->mig_jcr; /* newly migrated job */
267 * If mig_jcr is NULL, there is nothing to do for this job,
268 * so set a normal status, cleanup and return OK.
271 set_jcr_job_status(jcr, JS_Terminated);
272 migration_cleanup(jcr, jcr->JobStatus);
276 /* Print Job Start message */
277 Jmsg(jcr, M_INFO, 0, _("Start Migration JobId %s, Job=%s\n"),
278 edit_uint64(jcr->JobId, ed1), jcr->Job);
280 set_jcr_job_status(jcr, JS_Running);
281 set_jcr_job_status(mig_jcr, JS_Running);
282 Dmsg2(dbglevel, "JobId=%d JobLevel=%c\n", (int)jcr->jr.JobId, jcr->jr.JobLevel);
284 /* Update job start record for this migration control job */
285 if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) {
286 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db));
290 Dmsg4(dbglevel, "mig_jcr: Name=%s JobId=%d Type=%c Level=%c\n",
291 mig_jcr->jr.Name, (int)mig_jcr->jr.JobId,
292 mig_jcr->jr.JobType, mig_jcr->jr.JobLevel);
294 /* Update job start record for the real migration backup job */
295 if (!db_update_job_start_record(mig_jcr, mig_jcr->db, &mig_jcr->jr)) {
296 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(mig_jcr->db));
302 * Open a message channel connection with the Storage
303 * daemon. This is to let him know that our client
304 * will be contacting him for a backup session.
307 Dmsg0(110, "Open connection with storage daemon\n");
308 set_jcr_job_status(jcr, JS_WaitSD);
309 set_jcr_job_status(mig_jcr, JS_WaitSD);
311 * Start conversation with Storage daemon
313 if (!connect_to_storage_daemon(jcr, 10, SDConnectTimeout, 1)) {
316 sd = jcr->store_bsock;
318 * Now start a job with the Storage daemon
320 Dmsg2(dbglevel, "Read store=%s, write store=%s\n",
321 ((STORE *)jcr->rstorage->first())->name(),
322 ((STORE *)jcr->wstorage->first())->name());
323 if (((STORE *)jcr->rstorage->first())->name() == ((STORE *)jcr->wstorage->first())->name()) {
324 Jmsg(jcr, M_FATAL, 0, _("Read storage \"%s\" same as write storage.\n"),
325 ((STORE *)jcr->rstorage->first())->name());
328 if (!start_storage_daemon_job(jcr, jcr->rstorage, jcr->wstorage)) {
331 Dmsg0(150, "Storage daemon connection OK\n");
333 if (!send_bootstrap_file(jcr, sd) ||
334 !response(jcr, sd, OKbootstrap, "Bootstrap", DISPLAY_ERROR)) {
338 if (!bnet_fsend(sd, "run")) {
343 * Now start a Storage daemon message thread
345 if (!start_storage_daemon_message_thread(jcr)) {
350 set_jcr_job_status(jcr, JS_Running);
351 set_jcr_job_status(mig_jcr, JS_Running);
353 /* Pickup Job termination data */
354 /* Note, the SD stores in jcr->JobFiles/ReadBytes/JobBytes/Errors */
355 wait_for_storage_daemon_termination(jcr);
357 set_jcr_job_status(jcr, jcr->SDJobStatus);
358 if (jcr->JobStatus != JS_Terminated) {
362 migration_cleanup(jcr, jcr->JobStatus);
364 UAContext *ua = new_ua_context(jcr);
365 purge_job_records_from_catalog(ua, jcr->previous_jr.JobId);
376 /* Add an item to the list if it is unique */
377 static void add_unique_id(idpkt *ids, char *item)
382 /* Walk through current list to see if each item is the same as item */
385 for (int i=0; i<(int)sizeof(id); i++) {
388 } else if (*q == ',') {
395 if (strcmp(item, id) == 0) {
399 /* Did not find item, so add it to list */
400 if (ids->count == 0) {
403 pm_strcat(ids->list, ",");
405 pm_strcat(ids->list, item);
407 // Dmsg3(0, "add_uniq count=%d Ids=%p %s\n", ids->count, ids->list, ids->list);
412 * Callback handler make list of DB Ids
414 static int unique_dbid_handler(void *ctx, int num_fields, char **row)
416 idpkt *ids = (idpkt *)ctx;
418 add_unique_id(ids, row[0]);
419 Dmsg3(dbglevel, "dbid_hdlr count=%d Ids=%p %s\n", ids->count, ids->list, ids->list);
429 static int item_compare(void *item1, void *item2)
431 uitem *i1 = (uitem *)item1;
432 uitem *i2 = (uitem *)item2;
433 return strcmp(i1->item, i2->item);
436 static int unique_name_handler(void *ctx, int num_fields, char **row)
438 dlist *list = (dlist *)ctx;
440 uitem *new_item = (uitem *)malloc(sizeof(uitem));
443 memset(new_item, 0, sizeof(uitem));
444 new_item->item = bstrdup(row[0]);
445 Dmsg1(dbglevel, "Unique_name_hdlr Item=%s\n", row[0]);
446 item = (uitem *)list->binary_insert((void *)new_item, item_compare);
447 if (item != new_item) { /* already in list */
448 free(new_item->item);
449 free((char *)new_item);
455 /* Get Job names in Pool */
456 const char *sql_job =
457 "SELECT DISTINCT Job.Name from Job,Pool"
458 " WHERE Pool.Name='%s' AND Job.PoolId=Pool.PoolId";
460 /* Get JobIds from regex'ed Job names */
461 const char *sql_jobids_from_job =
462 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Job,Pool"
463 " WHERE Job.Name='%s' AND Pool.Name='%s' AND Job.PoolId=Pool.PoolId"
464 " ORDER by Job.StartTime";
466 /* Get Client names in Pool */
467 const char *sql_client =
468 "SELECT DISTINCT Client.Name from Client,Pool,Job"
469 " WHERE Pool.Name='%s' AND Job.ClientId=Client.ClientId AND"
470 " Job.PoolId=Pool.PoolId";
472 /* Get JobIds from regex'ed Client names */
473 const char *sql_jobids_from_client =
474 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Job,Pool,Client"
475 " WHERE Client.Name='%s' AND Pool.Name='%s' AND Job.PoolId=Pool.PoolId"
476 " AND Job.ClientId=Client.ClientId AND Job.Type='B'"
477 " ORDER by Job.StartTime";
479 /* Get Volume names in Pool */
480 const char *sql_vol =
481 "SELECT DISTINCT VolumeName FROM Media,Pool WHERE"
482 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
483 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'";
485 /* Get JobIds from regex'ed Volume names */
486 const char *sql_jobids_from_vol =
487 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Media,JobMedia,Job"
488 " WHERE Media.VolumeName='%s' AND Media.MediaId=JobMedia.MediaId"
489 " AND JobMedia.JobId=Job.JobId AND Job.Type='B'"
490 " ORDER by Job.StartTime";
493 const char *sql_smallest_vol =
494 "SELECT MediaId FROM Media,Pool WHERE"
495 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
496 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'"
497 " ORDER BY VolBytes ASC LIMIT 1";
499 const char *sql_oldest_vol =
500 "SELECT MediaId FROM Media,Pool WHERE"
501 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
502 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'"
503 " ORDER BY LastWritten ASC LIMIT 1";
505 /* Get JobIds when we have selected MediaId */
506 const char *sql_jobids_from_mediaid =
507 "SELECT DISTINCT Job.JobId,Job.StartTime FROM JobMedia,Job"
508 " WHERE JobMedia.JobId=Job.JobId AND JobMedia.MediaId=%s"
510 " ORDER by Job.StartTime";
512 /* Get tne number of bytes in the pool */
513 const char *sql_pool_bytes =
514 "SELECT SUM(VolBytes) FROM Media,Pool WHERE"
515 " VolStatus in ('Full','Used','Error','Append') AND Media.Enabled=1 AND"
516 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'";
518 /* Get tne number of bytes in the Jobs */
519 const char *sql_job_bytes =
520 "SELECT SUM(JobBytes) FROM Job WHERE JobId IN (%s)";
523 /* Get Media Ids in Pool */
524 const char *sql_mediaids =
525 "SELECT MediaId FROM Media,Pool WHERE"
526 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
527 " Media.PoolId=Pool.PoolId AND Pool.Name='%s' ORDER BY LastWritten ASC";
529 /* Get JobIds in Pool longer than specified time */
530 const char *sql_pool_time =
531 "SELECT DISTINCT Job.JobId from Pool,Job,Media,JobMedia WHERE"
532 " Pool.Name='%s' AND Media.PoolId=Pool.PoolId AND"
533 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
535 " JobMedia.JobId=Job.JobId AND Job.PoolId=Media.PoolId"
536 " AND Job.RealEndTime<='%s'";
539 * const char *sql_ujobid =
540 * "SELECT DISTINCT Job.Job from Client,Pool,Media,Job,JobMedia "
541 * " WHERE Media.PoolId=Pool.PoolId AND Pool.Name='%s' AND"
542 * " JobMedia.JobId=Job.JobId AND Job.PoolId=Media.PoolId";
549 * This is the central piece of code that finds a job or jobs
550 * actually JobIds to migrate. It first looks to see if one
551 * has been "manually" specified in jcr->MigrateJobId, and if
552 * so, it returns that JobId to be run. Otherwise, it
553 * examines the Selection Type to see what kind of migration
554 * we are doing (Volume, Job, Client, ...) and applies any
555 * Selection Pattern if appropriate to obtain a list of JobIds.
556 * Finally, it will loop over all the JobIds found, except the last
557 * one starting a new job with MigrationJobId set to that JobId, and
558 * finally, it returns the last JobId to the caller.
560 * Returns: -1 on error
561 * 0 if no jobs to migrate
562 * 1 if OK and jcr->previous_jr filled in
564 static int get_job_to_migrate(JCR *jcr)
567 POOL_MEM query(PM_MESSAGE);
572 idpkt ids, mid, jids;
577 char dt[MAX_TIME_LENGTH];
580 ids.list = get_pool_memory(PM_MESSAGE);
583 mid.list = get_pool_memory(PM_MESSAGE);
586 jids.list = get_pool_memory(PM_MESSAGE);
592 * If MigrateJobId is set, then we migrate only that Job,
593 * otherwise, we go through the full selection of jobs to
596 if (jcr->MigrateJobId != 0) {
597 Dmsg1(dbglevel, "At Job start previous jobid=%u\n", jcr->MigrateJobId);
598 edit_uint64(jcr->MigrateJobId, ids.list);
601 switch (jcr->job->selection_type) {
603 if (!regex_find_jobids(jcr, &ids, sql_job, sql_jobids_from_job, "Job")) {
608 if (!regex_find_jobids(jcr, &ids, sql_client, sql_jobids_from_client, "Client")) {
613 if (!regex_find_jobids(jcr, &ids, sql_vol, sql_jobids_from_vol, "Volume")) {
618 if (!jcr->job->selection_pattern) {
619 Jmsg(jcr, M_FATAL, 0, _("No Migration SQL selection pattern specified.\n"));
622 Dmsg1(dbglevel, "SQL=%s\n", jcr->job->selection_pattern);
623 if (!db_sql_query(jcr->db, jcr->job->selection_pattern,
624 unique_dbid_handler, (void *)&ids)) {
625 Jmsg(jcr, M_FATAL, 0,
626 _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
630 case MT_SMALLEST_VOL:
631 if (!find_mediaid_then_jobids(jcr, &ids, sql_smallest_vol, "Smallest Volume")) {
636 if (!find_mediaid_then_jobids(jcr, &ids, sql_oldest_vol, "Oldest Volume")) {
641 case MT_POOL_OCCUPANCY:
643 /* Find count of bytes in pool */
644 Mmsg(query, sql_pool_bytes, jcr->rpool->name());
645 if (!db_sql_query(jcr->db, query.c_str(), db_int64_handler, (void *)&ctx)) {
646 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
649 if (ctx.count == 0) {
650 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
653 pool_bytes = ctx.value;
654 Dmsg2(dbglevel, "highbytes=%d pool=%d\n", (int)jcr->rpool->MigrationHighBytes,
656 if (pool_bytes < (int64_t)jcr->rpool->MigrationHighBytes) {
657 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
660 Dmsg0(dbglevel, "We should do Occupation migration.\n");
663 /* Find a list of MediaIds that could be migrated */
664 Mmsg(query, sql_mediaids, jcr->rpool->name());
665 Dmsg1(dbglevel, "query=%s\n", query.c_str());
666 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)&ids)) {
667 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
670 if (ids.count == 0) {
671 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
674 Dmsg2(dbglevel, "Pool Occupancy ids=%d MediaIds=%s\n", ids.count, ids.list);
677 * Now loop over MediaIds getting more JobIds to migrate until
678 * we reduce the pool occupancy below the low water mark.
681 for (int i=0; i < (int)ids.count; i++) {
682 stat = get_next_dbid_from_list(&p, &MediaId);
683 Dmsg2(dbglevel, "get_next_dbid stat=%d MediaId=%u\n", stat, MediaId);
685 Jmsg(jcr, M_FATAL, 0, _("Invalid MediaId found.\n"));
687 } else if (stat == 0) {
691 Mmsg(mid.list, "%s", edit_int64(MediaId, ed1));
692 if (!find_jobids_from_mediaid_list(jcr, &mid, "Volumes")) {
696 pm_strcat(jids.list, ",");
698 pm_strcat(jids.list, mid.list);
699 jids.count += mid.count;
701 /* Now get the count of bytes added */
703 /* Find count of bytes from Jobs */
704 Mmsg(query, sql_job_bytes, mid.list);
705 if (!db_sql_query(jcr->db, query.c_str(), db_int64_handler, (void *)&ctx)) {
706 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
709 pool_bytes -= ctx.value;
710 Dmsg1(dbglevel, "Job bytes=%d\n", (int)ctx.value);
711 Dmsg2(dbglevel, "lowbytes=%d pool=%d\n", (int)jcr->rpool->MigrationLowBytes,
713 if (pool_bytes <= (int64_t)jcr->rpool->MigrationLowBytes) {
714 Dmsg0(dbglevel, "We should be done.\n");
719 Dmsg2(dbglevel, "Pool Occupancy ids=%d JobIds=%s\n", jids.count, jids.list);
724 ttime = time(NULL) - (time_t)jcr->rpool->MigrationTime;
725 (void)localtime_r(&ttime, &tm);
726 strftime(dt, sizeof(dt), "%Y-%m-%d %H:%M:%S", &tm);
729 Mmsg(query, sql_pool_time, jcr->rpool->name(), dt);
730 Dmsg1(dbglevel, "query=%s\n", query.c_str());
731 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)&ids)) {
732 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
735 if (ids.count == 0) {
736 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
739 Dmsg2(dbglevel, "PoolTime ids=%d JobIds=%s\n", ids.count, ids.list);
743 Jmsg(jcr, M_FATAL, 0, _("Unknown Migration Selection Type.\n"));
749 * Loop over all jobids except the last one, sending
750 * them to start_migration_job(), which will start a job
751 * for each of them. For the last JobId, we handle it below.
754 if (ids.count == 0) {
755 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
758 Jmsg(jcr, M_INFO, 0, _("The following %u JobId%s will be migrated: %s\n"),
759 ids.count, ids.count==0?"":"s", ids.list);
760 Dmsg2(dbglevel, "Before loop count=%d ids=%s\n", ids.count, ids.list);
761 for (int i=1; i < (int)ids.count; i++) {
763 stat = get_next_jobid_from_list(&p, &JobId);
764 Dmsg3(dbglevel, "get_jobid_no=%d stat=%d JobId=%u\n", i, stat, JobId);
765 jcr->MigrateJobId = JobId;
766 start_migration_job(jcr);
767 Dmsg0(dbglevel, "Back from start_migration_job\n");
769 Jmsg(jcr, M_FATAL, 0, _("Invalid JobId found.\n"));
771 } else if (stat == 0) {
772 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
777 /* Now get the last JobId and handle it in the current job */
779 stat = get_next_jobid_from_list(&p, &JobId);
780 Dmsg2(dbglevel, "Last get_next_jobid stat=%d JobId=%u\n", stat, (int)JobId);
782 Jmsg(jcr, M_FATAL, 0, _("Invalid JobId found.\n"));
784 } else if (stat == 0) {
785 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
789 jcr->previous_jr.JobId = JobId;
790 Dmsg1(dbglevel, "Previous jobid=%d\n", (int)jcr->previous_jr.JobId);
792 if (!db_get_job_record(jcr, jcr->db, &jcr->previous_jr)) {
793 Jmsg(jcr, M_FATAL, 0, _("Could not get job record for JobId %s to migrate. ERR=%s"),
794 edit_int64(jcr->previous_jr.JobId, ed1),
795 db_strerror(jcr->db));
798 Jmsg(jcr, M_INFO, 0, _("Migration using JobId=%s Job=%s\n"),
799 edit_int64(jcr->previous_jr.JobId, ed1), jcr->previous_jr.Job);
800 Dmsg3(dbglevel, "Migration JobId=%d using JobId=%s Job=%s\n",
802 edit_int64(jcr->previous_jr.JobId, ed1), jcr->previous_jr.Job);
812 free_pool_memory(ids.list);
813 free_pool_memory(mid.list);
814 free_pool_memory(jids.list);
818 static void start_migration_job(JCR *jcr)
820 UAContext *ua = new_ua_context(jcr);
823 Mmsg(ua->cmd, "run %s jobid=%s", jcr->job->hdr.name,
824 edit_uint64(jcr->MigrateJobId, ed1));
825 Dmsg1(dbglevel, "=============== Migration cmd=%s\n", ua->cmd);
826 parse_ua_args(ua); /* parse command */
827 int stat = run_cmd(ua, ua->cmd);
829 Jmsg(jcr, M_ERROR, 0, _("Could not start migration job.\n"));
831 Jmsg(jcr, M_INFO, 0, _("Migration JobId %d started.\n"), stat);
836 static bool find_mediaid_then_jobids(JCR *jcr, idpkt *ids, const char *query1,
840 POOL_MEM query(PM_MESSAGE);
843 /* Basic query for MediaId */
844 Mmsg(query, query1, jcr->rpool->name());
845 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
846 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
849 if (ids->count == 0) {
850 Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type);
852 if (ids->count != 1) {
853 Jmsg(jcr, M_FATAL, 0, _("SQL logic error. Count should be 1 but is %d\n"),
857 Dmsg1(dbglevel, "Smallest Vol Jobids=%s\n", ids->list);
859 ok = find_jobids_from_mediaid_list(jcr, ids, type);
865 static bool find_jobids_from_mediaid_list(JCR *jcr, idpkt *ids, const char *type)
868 POOL_MEM query(PM_MESSAGE);
870 Mmsg(query, sql_jobids_from_mediaid, ids->list);
872 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
873 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
876 if (ids->count == 0) {
877 Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type);
884 static bool regex_find_jobids(JCR *jcr, idpkt *ids, const char *query1,
885 const char *query2, const char *type)
889 uitem *last_item = NULL;
894 POOL_MEM query(PM_MESSAGE);
896 item_chain = New(dlist(item, &item->link));
897 if (!jcr->job->selection_pattern) {
898 Jmsg(jcr, M_FATAL, 0, _("No Migration %s selection pattern specified.\n"),
902 Dmsg1(dbglevel, "regex-sel-pattern=%s\n", jcr->job->selection_pattern);
903 /* Basic query for names */
904 Mmsg(query, query1, jcr->rpool->name());
905 Dmsg1(dbglevel, "get name query1=%s\n", query.c_str());
906 if (!db_sql_query(jcr->db, query.c_str(), unique_name_handler,
907 (void *)item_chain)) {
908 Jmsg(jcr, M_FATAL, 0,
909 _("SQL to get %s failed. ERR=%s\n"), type, db_strerror(jcr->db));
912 Dmsg1(dbglevel, "query1 returned %d names\n", item_chain->size());
913 if (item_chain->size() == 0) {
914 Jmsg(jcr, M_INFO, 0, _("Query of Pool \"%s\" returned no Jobs to migrate.\n"),
917 goto bail_out; /* skip regex match */
919 /* Compile regex expression */
920 rc = regcomp(&preg, jcr->job->selection_pattern, REG_EXTENDED);
922 regerror(rc, &preg, prbuf, sizeof(prbuf));
923 Jmsg(jcr, M_FATAL, 0, _("Could not compile regex pattern \"%s\" ERR=%s\n"),
924 jcr->job->selection_pattern, prbuf);
927 /* Now apply the regex to the names and remove any item not matched */
928 foreach_dlist(item, item_chain) {
929 const int nmatch = 30;
930 regmatch_t pmatch[nmatch];
932 Dmsg1(dbglevel, "Remove item %s\n", last_item->item);
933 free(last_item->item);
934 item_chain->remove(last_item);
936 Dmsg1(dbglevel, "get name Item=%s\n", item->item);
937 rc = regexec(&preg, item->item, nmatch, pmatch, 0);
939 last_item = NULL; /* keep this one */
945 free(last_item->item);
946 Dmsg1(dbglevel, "Remove item %s\n", last_item->item);
947 item_chain->remove(last_item);
951 if (item_chain->size() == 0) {
952 Jmsg(jcr, M_INFO, 0, _("Regex pattern matched no Jobs to migrate.\n"));
954 goto bail_out; /* skip regex match */
958 * At this point, we have a list of items in item_chain
959 * that have been matched by the regex, so now we need
960 * to look up their jobids.
963 foreach_dlist(item, item_chain) {
964 Dmsg2(dbglevel, "Got %s: %s\n", type, item->item);
965 Mmsg(query, query2, item->item, jcr->rpool->name());
966 Dmsg1(dbglevel, "get id from name query2=%s\n", query.c_str());
967 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
968 Jmsg(jcr, M_FATAL, 0,
969 _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
973 if (ids->count == 0) {
974 Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type);
979 Dmsg2(dbglevel, "Count=%d Jobids=%s\n", ids->count, ids->list);
986 * Release resources allocated during backup.
988 void migration_cleanup(JCR *jcr, int TermCode)
990 char sdt[MAX_TIME_LENGTH], edt[MAX_TIME_LENGTH];
991 char ec1[30], ec2[30], ec3[30], ec4[30], ec5[30], elapsed[50];
992 char ec6[50], ec7[50], ec8[50];
993 char term_code[100], sd_term_msg[100];
994 const char *term_msg;
995 int msg_type = M_INFO;
999 JCR *mig_jcr = jcr->mig_jcr;
1000 POOL_MEM query(PM_MESSAGE);
1002 Dmsg2(100, "Enter migrate_cleanup %d %c\n", TermCode, TermCode);
1003 update_job_end(jcr, TermCode);
1004 memset(&mr, 0, sizeof(mr));
1007 * Check if we actually did something.
1008 * mig_jcr is jcr of the newly migrated job.
1011 mig_jcr->JobFiles = jcr->JobFiles = jcr->SDJobFiles;
1012 mig_jcr->JobBytes = jcr->JobBytes = jcr->SDJobBytes;
1013 mig_jcr->VolSessionId = jcr->VolSessionId;
1014 mig_jcr->VolSessionTime = jcr->VolSessionTime;
1015 mig_jcr->jr.RealEndTime = 0;
1016 mig_jcr->jr.PriorJobId = jcr->previous_jr.JobId;
1018 update_job_end(mig_jcr, TermCode);
1020 /* Update final items to set them to the previous job's values */
1021 Mmsg(query, "UPDATE Job SET StartTime='%s',EndTime='%s',"
1022 "JobTDate=%s WHERE JobId=%s",
1023 jcr->previous_jr.cStartTime, jcr->previous_jr.cEndTime,
1024 edit_uint64(jcr->previous_jr.JobTDate, ec1),
1025 edit_uint64(mig_jcr->jr.JobId, ec2));
1026 db_sql_query(mig_jcr->db, query.c_str(), NULL, NULL);
1028 /* Now mark the previous job as migrated if it terminated normally */
1029 if (jcr->JobStatus == JS_Terminated) {
1030 Mmsg(query, "UPDATE Job SET Type='%c' WHERE JobId=%s",
1031 (char)JT_MIGRATED_JOB, edit_uint64(jcr->previous_jr.JobId, ec1));
1032 db_sql_query(mig_jcr->db, query.c_str(), NULL, NULL);
1035 if (!db_get_job_record(jcr, jcr->db, &jcr->jr)) {
1036 Jmsg(jcr, M_WARNING, 0, _("Error getting job record for stats: %s"),
1037 db_strerror(jcr->db));
1038 set_jcr_job_status(jcr, JS_ErrorTerminated);
1041 bstrncpy(mr.VolumeName, jcr->VolumeName, sizeof(mr.VolumeName));
1042 if (!db_get_media_record(jcr, jcr->db, &mr)) {
1043 Jmsg(jcr, M_WARNING, 0, _("Error getting Media record for Volume \"%s\": ERR=%s"),
1044 mr.VolumeName, db_strerror(jcr->db));
1045 set_jcr_job_status(jcr, JS_ErrorTerminated);
1048 update_bootstrap_file(mig_jcr);
1050 if (!db_get_job_volume_names(mig_jcr, mig_jcr->db, mig_jcr->jr.JobId, &mig_jcr->VolumeName)) {
1052 * Note, if the job has failed, most likely it did not write any
1053 * tape, so suppress this "error" message since in that case
1054 * it is normal. Or look at it the other way, only for a
1055 * normal exit should we complain about this error.
1057 if (jcr->JobStatus == JS_Terminated && jcr->jr.JobBytes) {
1058 Jmsg(jcr, M_ERROR, 0, "%s", db_strerror(mig_jcr->db));
1060 mig_jcr->VolumeName[0] = 0; /* none */
1062 switch (jcr->JobStatus) {
1064 if (jcr->Errors || jcr->SDErrors) {
1065 term_msg = _("%s OK -- with warnings");
1067 term_msg = _("%s OK");
1071 case JS_ErrorTerminated:
1072 term_msg = _("*** %s Error ***");
1073 msg_type = M_ERROR; /* Generate error message */
1074 if (jcr->store_bsock) {
1075 bnet_sig(jcr->store_bsock, BNET_TERMINATE);
1076 if (jcr->SD_msg_chan) {
1077 pthread_cancel(jcr->SD_msg_chan);
1082 term_msg = _("%s Canceled");
1083 if (jcr->store_bsock) {
1084 bnet_sig(jcr->store_bsock, BNET_TERMINATE);
1085 if (jcr->SD_msg_chan) {
1086 pthread_cancel(jcr->SD_msg_chan);
1091 term_msg = _("Inappropriate %s term code");
1095 if (jcr->previous_jr.JobId != 0) {
1096 /* Mark previous job as migrated */
1097 Mmsg(query, "UPDATE Job SET Type='%c' WHERE JobId=%s",
1098 (char)JT_MIGRATED_JOB, edit_uint64(jcr->previous_jr.JobId, ec1));
1099 Dmsg1(000, "Mark: %s\n", query.c_str());
1100 db_sql_query(jcr->db, query.c_str(), NULL, NULL);
1102 term_msg = _("%s -- no files to migrate");
1105 bsnprintf(term_code, sizeof(term_code), term_msg, "Migration");
1106 bstrftimes(sdt, sizeof(sdt), jcr->jr.StartTime);
1107 bstrftimes(edt, sizeof(edt), jcr->jr.EndTime);
1108 RunTime = jcr->jr.EndTime - jcr->jr.StartTime;
1112 kbps = (double)jcr->SDJobBytes / (1000 * RunTime);
1116 jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg));
1118 Jmsg(jcr, msg_type, 0, _("Bacula %s (%s): %s\n"
1119 " Prev Backup JobId: %s\n"
1120 " New Backup JobId: %s\n"
1121 " Migration JobId: %s\n"
1122 " Migration Job: %s\n"
1123 " Backup Level: %s%s\n"
1125 " FileSet: \"%s\" %s\n"
1126 " Read Pool: \"%s\" (From %s)\n"
1127 " Read Storage: \"%s\" (From %s)\n"
1128 " Write Pool: \"%s\" (From %s)\n"
1129 " Write Storage: \"%s\" (From %s)\n"
1132 " Elapsed time: %s\n"
1134 " SD Files Written: %s\n"
1135 " SD Bytes Written: %s (%sB)\n"
1136 " Rate: %.1f KB/s\n"
1137 " Volume name(s): %s\n"
1138 " Volume Session Id: %d\n"
1139 " Volume Session Time: %d\n"
1140 " Last Volume Bytes: %s (%sB)\n"
1142 " SD termination status: %s\n"
1143 " Termination: %s\n\n"),
1147 edit_uint64(jcr->previous_jr.JobId, ec6),
1148 mig_jcr ? edit_uint64(mig_jcr->jr.JobId, ec7) : "0",
1149 edit_uint64(jcr->jr.JobId, ec8),
1151 level_to_str(jcr->JobLevel), jcr->since,
1152 jcr->client->name(),
1153 jcr->fileset->name(), jcr->FSCreateTime,
1154 jcr->rpool->name(), jcr->rpool_source,
1155 jcr->rstore?jcr->rstore->name():"*None*",
1156 NPRT(jcr->rstore_source),
1157 jcr->pool->name(), jcr->pool_source,
1158 jcr->wstore?jcr->wstore->name():"*None*",
1159 NPRT(jcr->wstore_source),
1162 edit_utime(RunTime, elapsed, sizeof(elapsed)),
1164 edit_uint64_with_commas(jcr->SDJobFiles, ec1),
1165 edit_uint64_with_commas(jcr->SDJobBytes, ec2),
1166 edit_uint64_with_suffix(jcr->SDJobBytes, ec3),
1168 mig_jcr ? mig_jcr->VolumeName : "",
1170 jcr->VolSessionTime,
1171 edit_uint64_with_commas(mr.VolBytes, ec4),
1172 edit_uint64_with_suffix(mr.VolBytes, ec5),
1177 Dmsg1(100, "migrate_cleanup() mig_jcr=0x%x\n", jcr->mig_jcr);
1179 free_jcr(jcr->mig_jcr);
1180 jcr->mig_jcr = NULL;
1182 Dmsg0(100, "Leave migrate_cleanup()\n");
1186 * Return next DBId from comma separated list
1189 * 1 if next DBId returned
1190 * 0 if no more DBIds are in list
1191 * -1 there is an error
1193 static int get_next_dbid_from_list(char **p, DBId_t *DBId)
1199 for (int i=0; i<(int)sizeof(id); i++) {
1202 } else if (*q == ',') {
1211 } else if (!is_a_number(id)) {
1212 return -1; /* error */
1215 *DBId = str_to_int64(id);
1219 bool set_migration_wstorage(JCR *jcr, POOL *pool)
1221 POOL *wpool = pool->NextPool;
1224 Jmsg(jcr, M_FATAL, 0, _("No Next Pool specification found in Pool \"%s\".\n"),
1229 if (!wpool->storage || wpool->storage->size() == 0) {
1230 Jmsg(jcr, M_FATAL, 0, _("No Storage specification found in Next Pool \"%s\".\n"),
1235 /* If pool storage specified, use it instead of job storage for backup */
1236 copy_wstorage(jcr, wpool->storage, _("Storage from Pool's NextPool resource"));