2 Bacula® - The Network Backup Solution
4 Copyright (C) 2004-2007 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of John Walker.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
30 * Bacula Director -- migrate.c -- responsible for doing
33 * Kern Sibbald, September MMIV
35 * Basic tasks done here:
36 * Open DB and create records for this job.
37 * Open Message Channel with Storage daemon to tell him a job will be starting.
38 * Open connection with Storage daemon and pass him commands
40 * When the Storage daemon finishes the job, update the DB.
49 #include "lib/bregex.h"
54 static const int dbglevel = 10;
56 static char OKbootstrap[] = "3000 OK bootstrap\n";
57 static int get_job_to_migrate(JCR *jcr);
59 static bool regex_find_jobids(JCR *jcr, idpkt *ids, const char *query1,
60 const char *query2, const char *type);
61 static bool find_mediaid_then_jobids(JCR *jcr, idpkt *ids, const char *query1,
63 static bool find_jobids_from_mediaid_list(JCR *jcr, idpkt *ids, const char *type);
64 static void start_migration_job(JCR *jcr);
65 static int get_next_dbid_from_list(char **p, DBId_t *DBId);
68 * Called here before the job is run to do the job
69 * specific setup. Note, one of the important things to
70 * complete in this init code is to make the definitive
71 * choice of input and output storage devices. This is
72 * because immediately after the init, the job is queued
73 * in the jobq.c code, and it checks that all the resources
74 * (storage resources in particular) are available, so these
75 * must all be properly defined.
77 * previous_jr refers to the job DB record of the Job that is
78 * going to be migrated.
79 * prev_job refers to the job resource of the Job that is
80 * going to be migrated.
81 * jcr is the jcr for the current "migration" job. It is a
82 * control job that is put in the DB as a migration job, which
83 * means that this job migrated a previous job to a new job.
84 * No Volume or File data is associated with this control
86 * mig_jcr refers to the newly migrated job that is run by
87 * the current jcr. It is a backup job that moves (migrates) the
88 * data written for the previous_jr into the new pool. This
89 * job (mig_jcr) becomes the new backup job that replaces
90 * the original backup job. Note, this jcr is not really run. It
91 * is simply attached to the current jcr. It will show up in
92 * the Director's status output, but not in the SD or FD, both of
93 * which deal only with the current migration job (i.e. jcr).
95 bool do_migration_init(JCR *jcr)
101 JCR *mig_jcr; /* newly migrated job */
105 apply_pool_overrides(jcr);
107 jcr->jr.PoolId = get_or_create_pool_record(jcr, jcr->pool->name());
108 if (jcr->jr.PoolId == 0) {
109 Dmsg1(dbglevel, "JobId=%d no PoolId\n", (int)jcr->JobId);
110 Jmsg(jcr, M_FATAL, 0, _("Could not get or create a Pool record.\n"));
114 * Note, at this point, pool is the pool for this job. We
115 * transfer it to rpool (read pool), and a bit later,
116 * pool will be changed to point to the write pool,
117 * which comes from pool->NextPool.
119 jcr->rpool = jcr->pool; /* save read pool */
120 pm_strcpy(jcr->rpool_source, jcr->pool_source);
123 Dmsg2(dbglevel, "Read pool=%s (From %s)\n", jcr->rpool->name(), jcr->rpool_source);
125 /* If we find a job or jobs to migrate it is previous_jr.JobId */
126 count = get_job_to_migrate(jcr);
134 Dmsg1(dbglevel, "Back from get_job_to_migrate JobId=%d\n", (int)jcr->JobId);
136 if (jcr->previous_jr.JobId == 0) {
137 Dmsg1(dbglevel, "JobId=%d no previous JobId\n", (int)jcr->JobId);
138 Jmsg(jcr, M_INFO, 0, _("No previous Job found to migrate.\n"));
139 return true; /* no work */
142 if (!get_or_create_fileset_record(jcr)) {
143 Dmsg1(dbglevel, "JobId=%d no FileSet\n", (int)jcr->JobId);
144 Jmsg(jcr, M_FATAL, 0, _("Could not get or create the FileSet record.\n"));
148 create_restore_bootstrap_file(jcr);
150 if (jcr->previous_jr.JobId == 0 || jcr->ExpectedFiles == 0) {
151 set_jcr_job_status(jcr, JS_Terminated);
152 Dmsg1(dbglevel, "JobId=%d expected files == 0\n", (int)jcr->JobId);
153 if (jcr->previous_jr.JobId == 0) {
154 Jmsg(jcr, M_INFO, 0, _("No previous Job found to migrate.\n"));
156 Jmsg(jcr, M_INFO, 0, _("Previous Job has no data to migrate.\n"));
158 return true; /* no work */
161 Dmsg5(dbglevel, "JobId=%d: Previous: Name=%s JobId=%d Type=%c Level=%c\n",
163 jcr->previous_jr.Name, (int)jcr->previous_jr.JobId,
164 jcr->previous_jr.JobType, jcr->previous_jr.JobLevel);
166 Dmsg5(dbglevel, "JobId=%d: Current: Name=%s JobId=%d Type=%c Level=%c\n",
168 jcr->jr.Name, (int)jcr->jr.JobId,
169 jcr->jr.JobType, jcr->jr.JobLevel);
172 job = (JOB *)GetResWithName(R_JOB, jcr->jr.Name);
173 prev_job = (JOB *)GetResWithName(R_JOB, jcr->previous_jr.Name);
176 Jmsg(jcr, M_FATAL, 0, _("Job resource not found for \"%s\".\n"), jcr->jr.Name);
180 Jmsg(jcr, M_FATAL, 0, _("Previous Job resource not found for \"%s\".\n"),
181 jcr->previous_jr.Name);
185 jcr->spool_data = job->spool_data; /* turn on spooling if requested in job */
187 /* Create a migation jcr */
188 mig_jcr = jcr->mig_jcr = new_jcr(sizeof(JCR), dird_free_jcr);
189 memcpy(&mig_jcr->previous_jr, &jcr->previous_jr, sizeof(mig_jcr->previous_jr));
192 * Turn the mig_jcr into a "real" job that takes on the aspects of
193 * the previous backup job "prev_job".
195 set_jcr_defaults(mig_jcr, prev_job);
196 if (!setup_job(mig_jcr)) {
197 Jmsg(jcr, M_FATAL, 0, _("setup job failed.\n"));
201 /* Now reset the job record from the previous job */
202 memcpy(&mig_jcr->jr, &jcr->previous_jr, sizeof(mig_jcr->jr));
203 /* Update the jr to reflect the new values of PoolId and JobId. */
204 mig_jcr->jr.PoolId = jcr->jr.PoolId;
205 mig_jcr->jr.JobId = mig_jcr->JobId;
207 Dmsg4(dbglevel, "mig_jcr: Name=%s JobId=%d Type=%c Level=%c\n",
208 mig_jcr->jr.Name, (int)mig_jcr->jr.JobId,
209 mig_jcr->jr.JobType, mig_jcr->jr.JobLevel);
212 * Get the PoolId used with the original job. Then
213 * find the pool name from the database record.
215 memset(&pr, 0, sizeof(pr));
216 pr.PoolId = mig_jcr->previous_jr.PoolId;
217 if (!db_get_pool_record(jcr, jcr->db, &pr)) {
218 Jmsg(jcr, M_FATAL, 0, _("Pool for JobId %s not in database. ERR=%s\n"),
219 edit_int64(pr.PoolId, ed1), db_strerror(jcr->db));
222 /* Get the pool resource corresponding to the original job */
223 pool = (POOL *)GetResWithName(R_POOL, pr.Name);
225 Jmsg(jcr, M_FATAL, 0, _("Pool resource \"%s\" not found.\n"), pr.Name);
229 /* If pool storage specified, use it for restore */
230 copy_rstorage(mig_jcr, pool->storage, _("Pool resource"));
231 copy_rstorage(jcr, pool->storage, _("Pool resource"));
234 * If the original backup pool has a NextPool, make sure a
235 * record exists in the database. Note, in this case, we
236 * will be migrating from pool to pool->NextPool.
238 if (pool->NextPool) {
239 jcr->jr.PoolId = get_or_create_pool_record(jcr, pool->NextPool->name());
240 if (jcr->jr.PoolId == 0) {
244 if (!set_migration_wstorage(jcr, pool)) {
247 mig_jcr->pool = jcr->pool = pool->NextPool;
248 pm_strcpy(jcr->pool_source, _("Job Pool's NextPool resource"));
249 mig_jcr->jr.PoolId = jcr->jr.PoolId;
251 Dmsg2(dbglevel, "Write pool=%s read rpool=%s\n", jcr->pool->name(), jcr->rpool->name());
256 * Do a Migration of a previous job
258 * Returns: false on failure
261 bool do_migration(JCR *jcr)
265 JCR *mig_jcr = jcr->mig_jcr; /* newly migrated job */
268 * If mig_jcr is NULL, there is nothing to do for this job,
269 * so set a normal status, cleanup and return OK.
272 set_jcr_job_status(jcr, JS_Terminated);
273 migration_cleanup(jcr, jcr->JobStatus);
277 /* Print Job Start message */
278 Jmsg(jcr, M_INFO, 0, _("Start Migration JobId %s, Job=%s\n"),
279 edit_uint64(jcr->JobId, ed1), jcr->Job);
284 * Open a message channel connection with the Storage
285 * daemon. This is to let him know that our client
286 * will be contacting him for a backup session.
289 Dmsg0(110, "Open connection with storage daemon\n");
290 set_jcr_job_status(jcr, JS_WaitSD);
291 set_jcr_job_status(mig_jcr, JS_WaitSD);
293 * Start conversation with Storage daemon
295 if (!connect_to_storage_daemon(jcr, 10, SDConnectTimeout, 1)) {
298 sd = jcr->store_bsock;
300 * Now start a job with the Storage daemon
302 Dmsg2(dbglevel, "Read store=%s, write store=%s\n",
303 ((STORE *)jcr->rstorage->first())->name(),
304 ((STORE *)jcr->wstorage->first())->name());
305 if (((STORE *)jcr->rstorage->first())->name() == ((STORE *)jcr->wstorage->first())->name()) {
306 Jmsg(jcr, M_FATAL, 0, _("Read storage \"%s\" same as write storage.\n"),
307 ((STORE *)jcr->rstorage->first())->name());
310 if (!start_storage_daemon_job(jcr, jcr->rstorage, jcr->wstorage)) {
313 Dmsg0(150, "Storage daemon connection OK\n");
315 if (!send_bootstrap_file(jcr, sd) ||
316 !response(jcr, sd, OKbootstrap, "Bootstrap", DISPLAY_ERROR)) {
321 * We re-update the job start record so that the start
322 * time is set after the run before job. This avoids
323 * that any files created by the run before job will
324 * be saved twice. They will be backed up in the current
325 * job, but not in the next one unless they are changed.
326 * Without this, they will be backed up in this job and
327 * in the next job run because in that case, their date
328 * is after the start of this run.
330 jcr->start_time = time(NULL);
331 jcr->jr.StartTime = jcr->start_time;
332 jcr->jr.JobTDate = jcr->start_time;
333 set_jcr_job_status(jcr, JS_Running);
335 /* Update job start record for this migration control job */
336 if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) {
337 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db));
342 mig_jcr->start_time = time(NULL);
343 mig_jcr->jr.StartTime = mig_jcr->start_time;
344 mig_jcr->jr.JobTDate = mig_jcr->start_time;
345 set_jcr_job_status(mig_jcr, JS_Running);
347 /* Update job start record for the real migration backup job */
348 if (!db_update_job_start_record(mig_jcr, mig_jcr->db, &mig_jcr->jr)) {
349 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(mig_jcr->db));
353 Dmsg4(dbglevel, "mig_jcr: Name=%s JobId=%d Type=%c Level=%c\n",
354 mig_jcr->jr.Name, (int)mig_jcr->jr.JobId,
355 mig_jcr->jr.JobType, mig_jcr->jr.JobLevel);
359 * Start the job prior to starting the message thread below
360 * to avoid two threads from using the BSOCK structure at
363 if (!bnet_fsend(sd, "run")) {
368 * Now start a Storage daemon message thread
370 if (!start_storage_daemon_message_thread(jcr)) {
375 set_jcr_job_status(jcr, JS_Running);
376 set_jcr_job_status(mig_jcr, JS_Running);
378 /* Pickup Job termination data */
379 /* Note, the SD stores in jcr->JobFiles/ReadBytes/JobBytes/Errors */
380 wait_for_storage_daemon_termination(jcr);
381 set_jcr_job_status(jcr, jcr->SDJobStatus);
382 db_write_batch_file_records(jcr); /* used by bulk batch file insert */
383 if (jcr->JobStatus != JS_Terminated) {
387 migration_cleanup(jcr, jcr->JobStatus);
390 UAContext *ua = new_ua_context(jcr);
391 edit_uint64(jcr->previous_jr.JobId, jobid);
392 /* Purge all old file records, but leave Job record */
393 purge_files_from_jobs(ua, jobid);
404 /* Add an item to the list if it is unique */
405 static void add_unique_id(idpkt *ids, char *item)
410 /* Walk through current list to see if each item is the same as item */
413 for (int i=0; i<(int)sizeof(id); i++) {
416 } else if (*q == ',') {
423 if (strcmp(item, id) == 0) {
427 /* Did not find item, so add it to list */
428 if (ids->count == 0) {
431 pm_strcat(ids->list, ",");
433 pm_strcat(ids->list, item);
435 // Dmsg3(0, "add_uniq count=%d Ids=%p %s\n", ids->count, ids->list, ids->list);
440 * Callback handler make list of DB Ids
442 static int unique_dbid_handler(void *ctx, int num_fields, char **row)
444 idpkt *ids = (idpkt *)ctx;
446 add_unique_id(ids, row[0]);
447 Dmsg3(dbglevel, "dbid_hdlr count=%d Ids=%p %s\n", ids->count, ids->list, ids->list);
457 static int item_compare(void *item1, void *item2)
459 uitem *i1 = (uitem *)item1;
460 uitem *i2 = (uitem *)item2;
461 return strcmp(i1->item, i2->item);
464 static int unique_name_handler(void *ctx, int num_fields, char **row)
466 dlist *list = (dlist *)ctx;
468 uitem *new_item = (uitem *)malloc(sizeof(uitem));
471 memset(new_item, 0, sizeof(uitem));
472 new_item->item = bstrdup(row[0]);
473 Dmsg1(dbglevel, "Unique_name_hdlr Item=%s\n", row[0]);
474 item = (uitem *)list->binary_insert((void *)new_item, item_compare);
475 if (item != new_item) { /* already in list */
476 free(new_item->item);
477 free((char *)new_item);
483 /* Get Job names in Pool */
484 const char *sql_job =
485 "SELECT DISTINCT Job.Name from Job,Pool"
486 " WHERE Pool.Name='%s' AND Job.PoolId=Pool.PoolId";
488 /* Get JobIds from regex'ed Job names */
489 const char *sql_jobids_from_job =
490 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Job,Pool"
491 " WHERE Job.Name='%s' AND Pool.Name='%s' AND Job.PoolId=Pool.PoolId"
492 " ORDER by Job.StartTime";
494 /* Get Client names in Pool */
495 const char *sql_client =
496 "SELECT DISTINCT Client.Name from Client,Pool,Job"
497 " WHERE Pool.Name='%s' AND Job.ClientId=Client.ClientId AND"
498 " Job.PoolId=Pool.PoolId";
500 /* Get JobIds from regex'ed Client names */
501 const char *sql_jobids_from_client =
502 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Job,Pool,Client"
503 " WHERE Client.Name='%s' AND Pool.Name='%s' AND Job.PoolId=Pool.PoolId"
504 " AND Job.ClientId=Client.ClientId AND Job.Type='B'"
505 " ORDER by Job.StartTime";
507 /* Get Volume names in Pool */
508 const char *sql_vol =
509 "SELECT DISTINCT VolumeName FROM Media,Pool WHERE"
510 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
511 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'";
513 /* Get JobIds from regex'ed Volume names */
514 const char *sql_jobids_from_vol =
515 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Media,JobMedia,Job"
516 " WHERE Media.VolumeName='%s' AND Media.MediaId=JobMedia.MediaId"
517 " AND JobMedia.JobId=Job.JobId AND Job.Type='B'"
518 " ORDER by Job.StartTime";
521 const char *sql_smallest_vol =
522 "SELECT Media.MediaId FROM Media,Pool,JobMedia WHERE"
523 " Media.MediaId in (SELECT DISTINCT MediaId from JobMedia) AND"
524 " Media.VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
525 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'"
526 " ORDER BY VolBytes ASC LIMIT 1";
528 const char *sql_oldest_vol =
529 "SELECT Media.MediaId FROM Media,Pool,JobMedia WHERE"
530 " Media.MediaId in (SELECT DISTINCT MediaId from JobMedia) AND"
531 " Media.VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
532 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'"
533 " ORDER BY LastWritten ASC LIMIT 1";
535 /* Get JobIds when we have selected MediaId */
536 const char *sql_jobids_from_mediaid =
537 "SELECT DISTINCT Job.JobId,Job.StartTime FROM JobMedia,Job"
538 " WHERE JobMedia.JobId=Job.JobId AND JobMedia.MediaId=%s"
540 " ORDER by Job.StartTime";
542 /* Get tne number of bytes in the pool */
543 const char *sql_pool_bytes =
544 "SELECT SUM(VolBytes) FROM Media,Pool WHERE"
545 " VolStatus in ('Full','Used','Error','Append') AND Media.Enabled=1 AND"
546 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'";
548 /* Get tne number of bytes in the Jobs */
549 const char *sql_job_bytes =
550 "SELECT SUM(JobBytes) FROM Job WHERE JobId IN (%s)";
553 /* Get Media Ids in Pool */
554 const char *sql_mediaids =
555 "SELECT MediaId FROM Media,Pool WHERE"
556 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
557 " Media.PoolId=Pool.PoolId AND Pool.Name='%s' ORDER BY LastWritten ASC";
559 /* Get JobIds in Pool longer than specified time */
560 const char *sql_pool_time =
561 "SELECT DISTINCT Job.JobId from Pool,Job,Media,JobMedia WHERE"
562 " Pool.Name='%s' AND Media.PoolId=Pool.PoolId AND"
563 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
565 " JobMedia.JobId=Job.JobId AND Job.PoolId=Media.PoolId"
566 " AND Job.RealEndTime<='%s'";
569 * const char *sql_ujobid =
570 * "SELECT DISTINCT Job.Job from Client,Pool,Media,Job,JobMedia "
571 * " WHERE Media.PoolId=Pool.PoolId AND Pool.Name='%s' AND"
572 * " JobMedia.JobId=Job.JobId AND Job.PoolId=Media.PoolId";
579 * This is the central piece of code that finds a job or jobs
580 * actually JobIds to migrate. It first looks to see if one
581 * has been "manually" specified in jcr->MigrateJobId, and if
582 * so, it returns that JobId to be run. Otherwise, it
583 * examines the Selection Type to see what kind of migration
584 * we are doing (Volume, Job, Client, ...) and applies any
585 * Selection Pattern if appropriate to obtain a list of JobIds.
586 * Finally, it will loop over all the JobIds found, except the last
587 * one starting a new job with MigrationJobId set to that JobId, and
588 * finally, it returns the last JobId to the caller.
590 * Returns: -1 on error
591 * 0 if no jobs to migrate
592 * 1 if OK and jcr->previous_jr filled in
594 static int get_job_to_migrate(JCR *jcr)
596 char ed1[30], ed2[30];
597 POOL_MEM query(PM_MESSAGE);
602 idpkt ids, mid, jids;
607 char dt[MAX_TIME_LENGTH];
610 ids.list = get_pool_memory(PM_MESSAGE);
613 mid.list = get_pool_memory(PM_MESSAGE);
616 jids.list = get_pool_memory(PM_MESSAGE);
622 * If MigrateJobId is set, then we migrate only that Job,
623 * otherwise, we go through the full selection of jobs to
626 if (jcr->MigrateJobId != 0) {
627 Dmsg1(dbglevel, "At Job start previous jobid=%u\n", jcr->MigrateJobId);
628 edit_uint64(jcr->MigrateJobId, ids.list);
631 switch (jcr->job->selection_type) {
633 if (!regex_find_jobids(jcr, &ids, sql_job, sql_jobids_from_job, "Job")) {
638 if (!regex_find_jobids(jcr, &ids, sql_client, sql_jobids_from_client, "Client")) {
643 if (!regex_find_jobids(jcr, &ids, sql_vol, sql_jobids_from_vol, "Volume")) {
648 if (!jcr->job->selection_pattern) {
649 Jmsg(jcr, M_FATAL, 0, _("No Migration SQL selection pattern specified.\n"));
652 Dmsg1(dbglevel, "SQL=%s\n", jcr->job->selection_pattern);
653 if (!db_sql_query(jcr->db, jcr->job->selection_pattern,
654 unique_dbid_handler, (void *)&ids)) {
655 Jmsg(jcr, M_FATAL, 0,
656 _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
660 case MT_SMALLEST_VOL:
661 if (!find_mediaid_then_jobids(jcr, &ids, sql_smallest_vol, "Smallest Volume")) {
666 if (!find_mediaid_then_jobids(jcr, &ids, sql_oldest_vol, "Oldest Volume")) {
671 case MT_POOL_OCCUPANCY:
673 /* Find count of bytes in pool */
674 Mmsg(query, sql_pool_bytes, jcr->rpool->name());
675 if (!db_sql_query(jcr->db, query.c_str(), db_int64_handler, (void *)&ctx)) {
676 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
679 if (ctx.count == 0) {
680 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
683 pool_bytes = ctx.value;
684 Dmsg2(dbglevel, "highbytes=%lld pool=%lld\n", jcr->rpool->MigrationHighBytes,
686 if (pool_bytes < (int64_t)jcr->rpool->MigrationHighBytes) {
687 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
690 Dmsg0(dbglevel, "We should do Occupation migration.\n");
693 /* Find a list of MediaIds that could be migrated */
694 Mmsg(query, sql_mediaids, jcr->rpool->name());
695 Dmsg1(dbglevel, "query=%s\n", query.c_str());
696 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)&ids)) {
697 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
700 if (ids.count == 0) {
701 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
704 Dmsg2(dbglevel, "Pool Occupancy ids=%d MediaIds=%s\n", ids.count, ids.list);
706 if (!find_jobids_from_mediaid_list(jcr, &ids, "Volumes")) {
709 /* ids == list of jobs */
711 for (int i=0; i < (int)ids.count; i++) {
712 stat = get_next_dbid_from_list(&p, &DBId);
713 Dmsg2(dbglevel, "get_next_dbid stat=%d JobId=%u\n", stat, (uint32_t)DBId);
715 Jmsg(jcr, M_FATAL, 0, _("Invalid JobId found.\n"));
717 } else if (stat == 0) {
722 Mmsg(mid.list, "%s", edit_int64(DBId, ed1));
723 if (jids.count > 0) {
724 pm_strcat(jids.list, ",");
726 pm_strcat(jids.list, mid.list);
727 jids.count += mid.count;
729 /* Find count of bytes from Jobs */
730 Mmsg(query, sql_job_bytes, mid.list);
731 Dmsg1(dbglevel, "Jobbytes query: %s\n", query.c_str());
732 if (!db_sql_query(jcr->db, query.c_str(), db_int64_handler, (void *)&ctx)) {
733 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
736 pool_bytes -= ctx.value;
737 Dmsg1(dbglevel, "Total migrate Job bytes=%s\n", edit_int64(ctx.value, ed1));
738 Dmsg2(dbglevel, "lowbytes=%s poolafter=%s\n",
739 edit_int64(jcr->rpool->MigrationLowBytes, ed1),
740 edit_int64(pool_bytes, ed2));
741 if (pool_bytes <= (int64_t)jcr->rpool->MigrationLowBytes) {
742 Dmsg0(dbglevel, "We should be done.\n");
746 /* Transfer jids to ids, where the jobs list is expected */
747 ids.count = jids.count;
748 pm_strcpy(ids.list, jids.list);
749 Dmsg2(dbglevel, "Pool Occupancy ids=%d JobIds=%s\n", ids.count, ids.list);
753 ttime = time(NULL) - (time_t)jcr->rpool->MigrationTime;
754 (void)localtime_r(&ttime, &tm);
755 strftime(dt, sizeof(dt), "%Y-%m-%d %H:%M:%S", &tm);
758 Mmsg(query, sql_pool_time, jcr->rpool->name(), dt);
759 Dmsg1(dbglevel, "query=%s\n", query.c_str());
760 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)&ids)) {
761 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
764 if (ids.count == 0) {
765 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
768 Dmsg2(dbglevel, "PoolTime ids=%d JobIds=%s\n", ids.count, ids.list);
772 Jmsg(jcr, M_FATAL, 0, _("Unknown Migration Selection Type.\n"));
778 * Loop over all jobids except the last one, sending
779 * them to start_migration_job(), which will start a job
780 * for each of them. For the last JobId, we handle it below.
783 if (ids.count == 0) {
784 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
788 Jmsg(jcr, M_INFO, 0, _("The following %u JobId%s were chosen to be migrated: %s\n"),
789 ids.count, ids.count==0?"":"s", ids.list);
791 Dmsg2(dbglevel, "Before loop count=%d ids=%s\n", ids.count, ids.list);
792 for (int i=1; i < (int)ids.count; i++) {
794 stat = get_next_jobid_from_list(&p, &JobId);
795 Dmsg3(dbglevel, "get_jobid_no=%d stat=%d JobId=%u\n", i, stat, JobId);
796 jcr->MigrateJobId = JobId;
797 start_migration_job(jcr);
798 Dmsg0(dbglevel, "Back from start_migration_job\n");
800 Jmsg(jcr, M_FATAL, 0, _("Invalid JobId found.\n"));
802 } else if (stat == 0) {
803 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
808 /* Now get the last JobId and handle it in the current job */
810 stat = get_next_jobid_from_list(&p, &JobId);
811 Dmsg2(dbglevel, "Last get_next_jobid stat=%d JobId=%u\n", stat, (int)JobId);
813 Jmsg(jcr, M_FATAL, 0, _("Invalid JobId found.\n"));
815 } else if (stat == 0) {
816 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
820 jcr->previous_jr.JobId = JobId;
821 Dmsg1(dbglevel, "Previous jobid=%d\n", (int)jcr->previous_jr.JobId);
823 if (!db_get_job_record(jcr, jcr->db, &jcr->previous_jr)) {
824 Jmsg(jcr, M_FATAL, 0, _("Could not get job record for JobId %s to migrate. ERR=%s"),
825 edit_int64(jcr->previous_jr.JobId, ed1),
826 db_strerror(jcr->db));
829 Jmsg(jcr, M_INFO, 0, _("Migration using JobId=%s Job=%s\n"),
830 edit_int64(jcr->previous_jr.JobId, ed1), jcr->previous_jr.Job);
831 Dmsg3(dbglevel, "Migration JobId=%d using JobId=%s Job=%s\n",
833 edit_int64(jcr->previous_jr.JobId, ed1), jcr->previous_jr.Job);
843 free_pool_memory(ids.list);
844 free_pool_memory(mid.list);
845 free_pool_memory(jids.list);
849 static void start_migration_job(JCR *jcr)
851 UAContext *ua = new_ua_context(jcr);
854 Mmsg(ua->cmd, "run %s jobid=%s", jcr->job->hdr.name,
855 edit_uint64(jcr->MigrateJobId, ed1));
856 Dmsg1(dbglevel, "=============== Migration cmd=%s\n", ua->cmd);
857 parse_ua_args(ua); /* parse command */
858 int jobid = run_cmd(ua, ua->cmd);
860 Jmsg(jcr, M_ERROR, 0, _("Could not start migration job.\n"));
862 Jmsg(jcr, M_INFO, 0, _("Migration JobId %d started.\n"), jobid);
867 static bool find_mediaid_then_jobids(JCR *jcr, idpkt *ids, const char *query1,
871 POOL_MEM query(PM_MESSAGE);
874 /* Basic query for MediaId */
875 Mmsg(query, query1, jcr->rpool->name());
876 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
877 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
880 if (ids->count == 0) {
881 Jmsg(jcr, M_INFO, 0, _("No %s found to migrate.\n"), type);
882 ok = true; /* Not an error */
884 } else if (ids->count != 1) {
885 Jmsg(jcr, M_FATAL, 0, _("SQL error. Expected 1 MediaId got %d\n"), ids->count);
888 Dmsg2(dbglevel, "%s MediaIds=%s\n", type, ids->list);
890 ok = find_jobids_from_mediaid_list(jcr, ids, type);
897 * This routine returns:
898 * false if an error occurred
900 * ids.count number of jobids found (may be zero)
902 static bool find_jobids_from_mediaid_list(JCR *jcr, idpkt *ids, const char *type)
905 POOL_MEM query(PM_MESSAGE);
907 Mmsg(query, sql_jobids_from_mediaid, ids->list);
909 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
910 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
913 if (ids->count == 0) {
914 Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type);
922 static bool regex_find_jobids(JCR *jcr, idpkt *ids, const char *query1,
923 const char *query2, const char *type)
927 uitem *last_item = NULL;
932 POOL_MEM query(PM_MESSAGE);
934 item_chain = New(dlist(item, &item->link));
935 if (!jcr->job->selection_pattern) {
936 Jmsg(jcr, M_FATAL, 0, _("No Migration %s selection pattern specified.\n"),
940 Dmsg1(dbglevel, "regex-sel-pattern=%s\n", jcr->job->selection_pattern);
941 /* Basic query for names */
942 Mmsg(query, query1, jcr->rpool->name());
943 Dmsg1(dbglevel, "get name query1=%s\n", query.c_str());
944 if (!db_sql_query(jcr->db, query.c_str(), unique_name_handler,
945 (void *)item_chain)) {
946 Jmsg(jcr, M_FATAL, 0,
947 _("SQL to get %s failed. ERR=%s\n"), type, db_strerror(jcr->db));
950 Dmsg1(dbglevel, "query1 returned %d names\n", item_chain->size());
951 if (item_chain->size() == 0) {
952 Jmsg(jcr, M_INFO, 0, _("Query of Pool \"%s\" returned no Jobs to migrate.\n"),
955 goto bail_out; /* skip regex match */
957 /* Compile regex expression */
958 rc = regcomp(&preg, jcr->job->selection_pattern, REG_EXTENDED);
960 regerror(rc, &preg, prbuf, sizeof(prbuf));
961 Jmsg(jcr, M_FATAL, 0, _("Could not compile regex pattern \"%s\" ERR=%s\n"),
962 jcr->job->selection_pattern, prbuf);
965 /* Now apply the regex to the names and remove any item not matched */
966 foreach_dlist(item, item_chain) {
967 const int nmatch = 30;
968 regmatch_t pmatch[nmatch];
970 Dmsg1(dbglevel, "Remove item %s\n", last_item->item);
971 free(last_item->item);
972 item_chain->remove(last_item);
974 Dmsg1(dbglevel, "get name Item=%s\n", item->item);
975 rc = regexec(&preg, item->item, nmatch, pmatch, 0);
977 last_item = NULL; /* keep this one */
983 free(last_item->item);
984 Dmsg1(dbglevel, "Remove item %s\n", last_item->item);
985 item_chain->remove(last_item);
989 if (item_chain->size() == 0) {
990 Jmsg(jcr, M_INFO, 0, _("Regex pattern matched no Jobs to migrate.\n"));
992 goto bail_out; /* skip regex match */
996 * At this point, we have a list of items in item_chain
997 * that have been matched by the regex, so now we need
998 * to look up their jobids.
1001 foreach_dlist(item, item_chain) {
1002 Dmsg2(dbglevel, "Got %s: %s\n", type, item->item);
1003 Mmsg(query, query2, item->item, jcr->rpool->name());
1004 Dmsg1(dbglevel, "get id from name query2=%s\n", query.c_str());
1005 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
1006 Jmsg(jcr, M_FATAL, 0,
1007 _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
1011 if (ids->count == 0) {
1012 Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type);
1017 Dmsg2(dbglevel, "Count=%d Jobids=%s\n", ids->count, ids->list);
1018 foreach_dlist(item, item_chain) {
1027 * Release resources allocated during backup.
1029 void migration_cleanup(JCR *jcr, int TermCode)
1031 char sdt[MAX_TIME_LENGTH], edt[MAX_TIME_LENGTH];
1032 char ec1[30], ec2[30], ec3[30], ec4[30], ec5[30], elapsed[50];
1033 char ec6[50], ec7[50], ec8[50];
1034 char term_code[100], sd_term_msg[100];
1035 const char *term_msg;
1036 int msg_type = M_INFO;
1040 JCR *mig_jcr = jcr->mig_jcr;
1041 POOL_MEM query(PM_MESSAGE);
1043 Dmsg2(100, "Enter migrate_cleanup %d %c\n", TermCode, TermCode);
1044 update_job_end(jcr, TermCode);
1045 memset(&mr, 0, sizeof(mr));
1048 * Check if we actually did something.
1049 * mig_jcr is jcr of the newly migrated job.
1052 mig_jcr->JobFiles = jcr->JobFiles = jcr->SDJobFiles;
1053 mig_jcr->JobBytes = jcr->JobBytes = jcr->SDJobBytes;
1054 mig_jcr->VolSessionId = jcr->VolSessionId;
1055 mig_jcr->VolSessionTime = jcr->VolSessionTime;
1056 mig_jcr->jr.RealEndTime = 0;
1057 mig_jcr->jr.PriorJobId = jcr->previous_jr.JobId;
1059 update_job_end(mig_jcr, TermCode);
1061 /* Update final items to set them to the previous job's values */
1062 Mmsg(query, "UPDATE Job SET StartTime='%s',EndTime='%s',"
1063 "JobTDate=%s WHERE JobId=%s",
1064 jcr->previous_jr.cStartTime, jcr->previous_jr.cEndTime,
1065 edit_uint64(jcr->previous_jr.JobTDate, ec1),
1066 edit_uint64(mig_jcr->jr.JobId, ec2));
1067 db_sql_query(mig_jcr->db, query.c_str(), NULL, NULL);
1069 /* Now mark the previous job as migrated if it terminated normally */
1070 if (jcr->JobStatus == JS_Terminated) {
1071 Mmsg(query, "UPDATE Job SET Type='%c' WHERE JobId=%s",
1072 (char)JT_MIGRATED_JOB, edit_uint64(jcr->previous_jr.JobId, ec1));
1073 db_sql_query(mig_jcr->db, query.c_str(), NULL, NULL);
1076 if (!db_get_job_record(jcr, jcr->db, &jcr->jr)) {
1077 Jmsg(jcr, M_WARNING, 0, _("Error getting Job record for Job report: ERR=%s"),
1078 db_strerror(jcr->db));
1079 set_jcr_job_status(jcr, JS_ErrorTerminated);
1083 update_bootstrap_file(mig_jcr);
1085 if (!db_get_job_volume_names(mig_jcr, mig_jcr->db, mig_jcr->jr.JobId, &mig_jcr->VolumeName)) {
1087 * Note, if the job has failed, most likely it did not write any
1088 * tape, so suppress this "error" message since in that case
1089 * it is normal. Or look at it the other way, only for a
1090 * normal exit should we complain about this error.
1092 if (jcr->JobStatus == JS_Terminated && jcr->jr.JobBytes) {
1093 Jmsg(jcr, M_ERROR, 0, "%s", db_strerror(mig_jcr->db));
1095 mig_jcr->VolumeName[0] = 0; /* none */
1098 if (mig_jcr->VolumeName[0]) {
1099 /* Find last volume name. Multiple vols are separated by | */
1100 char *p = strrchr(mig_jcr->VolumeName, '|');
1102 p = mig_jcr->VolumeName;
1104 bstrncpy(mr.VolumeName, p, sizeof(mr.VolumeName));
1105 if (!db_get_media_record(jcr, jcr->db, &mr)) {
1106 Jmsg(jcr, M_WARNING, 0, _("Error getting Media record for Volume \"%s\": ERR=%s"),
1107 mr.VolumeName, db_strerror(jcr->db));
1111 switch (jcr->JobStatus) {
1113 if (jcr->Errors || jcr->SDErrors) {
1114 term_msg = _("%s OK -- with warnings");
1116 term_msg = _("%s OK");
1120 case JS_ErrorTerminated:
1121 term_msg = _("*** %s Error ***");
1122 msg_type = M_ERROR; /* Generate error message */
1123 if (jcr->store_bsock) {
1124 bnet_sig(jcr->store_bsock, BNET_TERMINATE);
1125 if (jcr->SD_msg_chan) {
1126 pthread_cancel(jcr->SD_msg_chan);
1131 term_msg = _("%s Canceled");
1132 if (jcr->store_bsock) {
1133 bnet_sig(jcr->store_bsock, BNET_TERMINATE);
1134 if (jcr->SD_msg_chan) {
1135 pthread_cancel(jcr->SD_msg_chan);
1140 term_msg = _("Inappropriate %s term code");
1144 if (jcr->previous_jr.JobId != 0) {
1145 /* Mark previous job as migrated */
1146 Mmsg(query, "UPDATE Job SET Type='%c' WHERE JobId=%s",
1147 (char)JT_MIGRATED_JOB, edit_uint64(jcr->previous_jr.JobId, ec1));
1148 Dmsg1(000, "Mark: %s\n", query.c_str());
1149 db_sql_query(jcr->db, query.c_str(), NULL, NULL);
1151 term_msg = _("%s -- no files to migrate");
1154 bsnprintf(term_code, sizeof(term_code), term_msg, "Migration");
1155 bstrftimes(sdt, sizeof(sdt), jcr->jr.StartTime);
1156 bstrftimes(edt, sizeof(edt), jcr->jr.EndTime);
1157 RunTime = jcr->jr.EndTime - jcr->jr.StartTime;
1161 kbps = (double)jcr->SDJobBytes / (1000 * RunTime);
1165 jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg));
1167 Jmsg(jcr, msg_type, 0, _("Bacula %s %s (%s): %s\n"
1168 " Build OS: %s %s %s\n"
1169 " Prev Backup JobId: %s\n"
1170 " New Backup JobId: %s\n"
1171 " Migration JobId: %s\n"
1172 " Migration Job: %s\n"
1173 " Backup Level: %s%s\n"
1175 " FileSet: \"%s\" %s\n"
1176 " Read Pool: \"%s\" (From %s)\n"
1177 " Read Storage: \"%s\" (From %s)\n"
1178 " Write Pool: \"%s\" (From %s)\n"
1179 " Write Storage: \"%s\" (From %s)\n"
1182 " Elapsed time: %s\n"
1184 " SD Files Written: %s\n"
1185 " SD Bytes Written: %s (%sB)\n"
1186 " Rate: %.1f KB/s\n"
1187 " Volume name(s): %s\n"
1188 " Volume Session Id: %d\n"
1189 " Volume Session Time: %d\n"
1190 " Last Volume Bytes: %s (%sB)\n"
1192 " SD termination status: %s\n"
1193 " Termination: %s\n\n"),
1194 my_name, VERSION, LSMDATE, edt,
1195 HOST_OS, DISTNAME, DISTVER,
1196 edit_uint64(jcr->previous_jr.JobId, ec6),
1197 mig_jcr ? edit_uint64(mig_jcr->jr.JobId, ec7) : "0",
1198 edit_uint64(jcr->jr.JobId, ec8),
1200 level_to_str(jcr->JobLevel), jcr->since,
1201 jcr->client->name(),
1202 jcr->fileset->name(), jcr->FSCreateTime,
1203 jcr->rpool->name(), jcr->rpool_source,
1204 jcr->rstore?jcr->rstore->name():"*None*",
1205 NPRT(jcr->rstore_source),
1206 jcr->pool->name(), jcr->pool_source,
1207 jcr->wstore?jcr->wstore->name():"*None*",
1208 NPRT(jcr->wstore_source),
1211 edit_utime(RunTime, elapsed, sizeof(elapsed)),
1213 edit_uint64_with_commas(jcr->SDJobFiles, ec1),
1214 edit_uint64_with_commas(jcr->SDJobBytes, ec2),
1215 edit_uint64_with_suffix(jcr->SDJobBytes, ec3),
1217 mig_jcr ? mig_jcr->VolumeName : "",
1219 jcr->VolSessionTime,
1220 edit_uint64_with_commas(mr.VolBytes, ec4),
1221 edit_uint64_with_suffix(mr.VolBytes, ec5),
1226 Dmsg1(100, "migrate_cleanup() mig_jcr=0x%x\n", jcr->mig_jcr);
1228 free_jcr(jcr->mig_jcr);
1229 jcr->mig_jcr = NULL;
1231 Dmsg0(100, "Leave migrate_cleanup()\n");
1235 * Return next DBId from comma separated list
1238 * 1 if next DBId returned
1239 * 0 if no more DBIds are in list
1240 * -1 there is an error
1242 static int get_next_dbid_from_list(char **p, DBId_t *DBId)
1248 for (int i=0; i<(int)sizeof(id); i++) {
1251 } else if (*q == ',') {
1260 } else if (!is_a_number(id)) {
1261 return -1; /* error */
1264 *DBId = str_to_int64(id);
1268 bool set_migration_wstorage(JCR *jcr, POOL *pool)
1270 POOL *wpool = pool->NextPool;
1273 Jmsg(jcr, M_FATAL, 0, _("No Next Pool specification found in Pool \"%s\".\n"),
1278 if (!wpool->storage || wpool->storage->size() == 0) {
1279 Jmsg(jcr, M_FATAL, 0, _("No Storage specification found in Next Pool \"%s\".\n"),
1284 /* If pool storage specified, use it instead of job storage for backup */
1285 copy_wstorage(jcr, wpool->storage, _("Storage from Pool's NextPool resource"));