2 Bacula® - The Network Backup Solution
4 Copyright (C) 2004-2008 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation and included
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of Kern Sibbald.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
30 * Bacula Director -- migrate.c -- responsible for doing
31 * migration and copy jobs.
33 * Also handles Copy jobs (March MMVIII)
35 * Kern Sibbald, September MMIV
37 * Basic tasks done here:
38 * Open DB and create records for this job.
39 * Open Message Channel with Storage daemon to tell him a job will be starting.
40 * Open connection with Storage daemon and pass him commands
42 * When the Storage daemon finishes the job, update the DB.
51 #include "lib/bregex.h"
56 static const int dbglevel = 10;
58 static char OKbootstrap[] = "3000 OK bootstrap\n";
59 static int get_job_to_migrate(JCR *jcr);
61 static bool regex_find_jobids(JCR *jcr, idpkt *ids, const char *query1,
62 const char *query2, const char *type);
63 static bool find_mediaid_then_jobids(JCR *jcr, idpkt *ids, const char *query1,
65 static bool find_jobids_from_mediaid_list(JCR *jcr, idpkt *ids, const char *type);
66 static void start_migration_job(JCR *jcr);
67 static int get_next_dbid_from_list(char **p, DBId_t *DBId);
70 * Called here before the job is run to do the job
71 * specific setup. Note, one of the important things to
72 * complete in this init code is to make the definitive
73 * choice of input and output storage devices. This is
74 * because immediately after the init, the job is queued
75 * in the jobq.c code, and it checks that all the resources
76 * (storage resources in particular) are available, so these
77 * must all be properly defined.
79 * previous_jr refers to the job DB record of the Job that is
80 * going to be migrated.
81 * prev_job refers to the job resource of the Job that is
82 * going to be migrated.
83 * jcr is the jcr for the current "migration" job. It is a
84 * control job that is put in the DB as a migration job, which
85 * means that this job migrated a previous job to a new job.
86 * No Volume or File data is associated with this control
88 * mig_jcr refers to the newly migrated job that is run by
89 * the current jcr. It is a backup job that moves (migrates) the
90 * data written for the previous_jr into the new pool. This
91 * job (mig_jcr) becomes the new backup job that replaces
92 * the original backup job. Note, this jcr is not really run. It
93 * is simply attached to the current jcr. It will show up in
94 * the Director's status output, but not in the SD or FD, both of
95 * which deal only with the current migration job (i.e. jcr).
97 bool do_migration_init(JCR *jcr)
103 JCR *mig_jcr; /* newly migrated job */
107 apply_pool_overrides(jcr);
109 if (!allow_duplicate_job(jcr)) {
113 jcr->jr.PoolId = get_or_create_pool_record(jcr, jcr->pool->name());
114 if (jcr->jr.PoolId == 0) {
115 Dmsg1(dbglevel, "JobId=%d no PoolId\n", (int)jcr->JobId);
116 Jmsg(jcr, M_FATAL, 0, _("Could not get or create a Pool record.\n"));
120 * Note, at this point, pool is the pool for this job. We
121 * transfer it to rpool (read pool), and a bit later,
122 * pool will be changed to point to the write pool,
123 * which comes from pool->NextPool.
125 jcr->rpool = jcr->pool; /* save read pool */
126 pm_strcpy(jcr->rpool_source, jcr->pool_source);
129 Dmsg2(dbglevel, "Read pool=%s (From %s)\n", jcr->rpool->name(), jcr->rpool_source);
131 /* If we find a job or jobs to migrate it is previous_jr.JobId */
132 count = get_job_to_migrate(jcr);
140 Dmsg1(dbglevel, "Back from get_job_to_migrate JobId=%d\n", (int)jcr->JobId);
142 if (jcr->previous_jr.JobId == 0) {
143 Dmsg1(dbglevel, "JobId=%d no previous JobId\n", (int)jcr->JobId);
144 Jmsg(jcr, M_INFO, 0, _("No previous Job found to migrate.\n"));
145 return true; /* no work */
148 if (!get_or_create_fileset_record(jcr)) {
149 Dmsg1(dbglevel, "JobId=%d no FileSet\n", (int)jcr->JobId);
150 Jmsg(jcr, M_FATAL, 0, _("Could not get or create the FileSet record.\n"));
154 create_restore_bootstrap_file(jcr);
156 if (jcr->previous_jr.JobId == 0 || jcr->ExpectedFiles == 0) {
157 set_jcr_job_status(jcr, JS_Terminated);
158 Dmsg1(dbglevel, "JobId=%d expected files == 0\n", (int)jcr->JobId);
159 if (jcr->previous_jr.JobId == 0) {
160 Jmsg(jcr, M_INFO, 0, _("No previous Job found to migrate.\n"));
162 Jmsg(jcr, M_INFO, 0, _("Previous Job has no data to migrate.\n"));
164 return true; /* no work */
168 Dmsg5(dbglevel, "JobId=%d: Current: Name=%s JobId=%d Type=%c Level=%c\n",
170 jcr->jr.Name, (int)jcr->jr.JobId,
171 jcr->jr.JobType, jcr->jr.JobLevel);
174 job = (JOB *)GetResWithName(R_JOB, jcr->jr.Name);
175 prev_job = (JOB *)GetResWithName(R_JOB, jcr->previous_jr.Name);
178 Jmsg(jcr, M_FATAL, 0, _("Job resource not found for \"%s\".\n"), jcr->jr.Name);
182 Jmsg(jcr, M_FATAL, 0, _("Previous Job resource not found for \"%s\".\n"),
183 jcr->previous_jr.Name);
187 jcr->spool_data = job->spool_data; /* turn on spooling if requested in job */
189 /* Create a migation jcr */
190 mig_jcr = jcr->mig_jcr = new_jcr(sizeof(JCR), dird_free_jcr);
191 memcpy(&mig_jcr->previous_jr, &jcr->previous_jr, sizeof(mig_jcr->previous_jr));
194 * Turn the mig_jcr into a "real" job that takes on the aspects of
195 * the previous backup job "prev_job".
197 set_jcr_defaults(mig_jcr, prev_job);
198 if (!setup_job(mig_jcr)) {
199 Jmsg(jcr, M_FATAL, 0, _("setup job failed.\n"));
203 /* Now reset the job record from the previous job */
204 memcpy(&mig_jcr->jr, &jcr->previous_jr, sizeof(mig_jcr->jr));
205 /* Update the jr to reflect the new values of PoolId and JobId. */
206 mig_jcr->jr.PoolId = jcr->jr.PoolId;
207 mig_jcr->jr.JobId = mig_jcr->JobId;
209 Dmsg4(dbglevel, "mig_jcr: Name=%s JobId=%d Type=%c Level=%c\n",
210 mig_jcr->jr.Name, (int)mig_jcr->jr.JobId,
211 mig_jcr->jr.JobType, mig_jcr->jr.JobLevel);
214 * Get the PoolId used with the original job. Then
215 * find the pool name from the database record.
217 memset(&pr, 0, sizeof(pr));
218 pr.PoolId = mig_jcr->previous_jr.PoolId;
219 if (!db_get_pool_record(jcr, jcr->db, &pr)) {
220 Jmsg(jcr, M_FATAL, 0, _("Pool for JobId %s not in database. ERR=%s\n"),
221 edit_int64(pr.PoolId, ed1), db_strerror(jcr->db));
224 /* Get the pool resource corresponding to the original job */
225 pool = (POOL *)GetResWithName(R_POOL, pr.Name);
227 Jmsg(jcr, M_FATAL, 0, _("Pool resource \"%s\" not found.\n"), pr.Name);
231 /* If pool storage specified, use it for restore */
232 copy_rstorage(mig_jcr, pool->storage, _("Pool resource"));
233 copy_rstorage(jcr, pool->storage, _("Pool resource"));
236 * If the original backup pool has a NextPool, make sure a
237 * record exists in the database. Note, in this case, we
238 * will be migrating from pool to pool->NextPool.
240 if (pool->NextPool) {
241 jcr->jr.PoolId = get_or_create_pool_record(jcr, pool->NextPool->name());
242 if (jcr->jr.PoolId == 0) {
246 if (!set_migration_wstorage(jcr, pool)) {
249 mig_jcr->pool = jcr->pool = pool->NextPool;
250 pm_strcpy(jcr->pool_source, _("Job Pool's NextPool resource"));
251 mig_jcr->jr.PoolId = jcr->jr.PoolId;
253 Dmsg2(dbglevel, "Write pool=%s read rpool=%s\n", jcr->pool->name(), jcr->rpool->name());
258 * Do a Migration of a previous job
260 * Returns: false on failure
263 bool do_migration(JCR *jcr)
267 JCR *mig_jcr = jcr->mig_jcr; /* newly migrated job */
270 * If mig_jcr is NULL, there is nothing to do for this job,
271 * so set a normal status, cleanup and return OK.
274 set_jcr_job_status(jcr, JS_Terminated);
275 migration_cleanup(jcr, jcr->JobStatus);
279 if (!db_get_job_record(jcr, jcr->db, &jcr->previous_jr)) {
280 Jmsg(jcr, M_FATAL, 0, _("Could not get job record for JobId %s to migrate. ERR=%s"),
281 edit_int64(jcr->previous_jr.JobId, ed1),
282 db_strerror(jcr->db));
283 set_jcr_job_status(jcr, JS_Terminated);
284 migration_cleanup(jcr, jcr->JobStatus);
287 /* Make sure this job was not already migrated */
288 if (jcr->previous_jr.JobType != JT_BACKUP) {
289 set_jcr_job_status(jcr, JS_Terminated);
290 migration_cleanup(jcr, jcr->JobStatus);
294 /* Print Job Start message */
295 Jmsg(jcr, M_INFO, 0, _("Start %s JobId %s, Job=%s\n"),
296 jcr->get_JobType() == JT_MIGRATE ? "Migration" : "Copy",
297 edit_uint64(jcr->JobId, ed1), jcr->Job);
300 * Open a message channel connection with the Storage
301 * daemon. This is to let him know that our client
302 * will be contacting him for a backup session.
305 Dmsg0(110, "Open connection with storage daemon\n");
306 set_jcr_job_status(jcr, JS_WaitSD);
307 set_jcr_job_status(mig_jcr, JS_WaitSD);
309 * Start conversation with Storage daemon
311 if (!connect_to_storage_daemon(jcr, 10, SDConnectTimeout, 1)) {
314 sd = jcr->store_bsock;
316 * Now start a job with the Storage daemon
318 Dmsg2(dbglevel, "Read store=%s, write store=%s\n",
319 ((STORE *)jcr->rstorage->first())->name(),
320 ((STORE *)jcr->wstorage->first())->name());
321 if (((STORE *)jcr->rstorage->first())->name() == ((STORE *)jcr->wstorage->first())->name()) {
322 Jmsg(jcr, M_FATAL, 0, _("Read storage \"%s\" same as write storage.\n"),
323 ((STORE *)jcr->rstorage->first())->name());
326 if (!start_storage_daemon_job(jcr, jcr->rstorage, jcr->wstorage)) {
329 Dmsg0(150, "Storage daemon connection OK\n");
331 if (!send_bootstrap_file(jcr, sd) ||
332 !response(jcr, sd, OKbootstrap, "Bootstrap", DISPLAY_ERROR)) {
337 * We re-update the job start record so that the start
338 * time is set after the run before job. This avoids
339 * that any files created by the run before job will
340 * be saved twice. They will be backed up in the current
341 * job, but not in the next one unless they are changed.
342 * Without this, they will be backed up in this job and
343 * in the next job run because in that case, their date
344 * is after the start of this run.
346 jcr->start_time = time(NULL);
347 jcr->jr.StartTime = jcr->start_time;
348 jcr->jr.JobTDate = jcr->start_time;
349 set_jcr_job_status(jcr, JS_Running);
351 /* Update job start record for this migration control job */
352 if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) {
353 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db));
358 mig_jcr->start_time = time(NULL);
359 mig_jcr->jr.StartTime = mig_jcr->start_time;
360 mig_jcr->jr.JobTDate = mig_jcr->start_time;
361 set_jcr_job_status(mig_jcr, JS_Running);
363 /* Update job start record for the real migration backup job */
364 if (!db_update_job_start_record(mig_jcr, mig_jcr->db, &mig_jcr->jr)) {
365 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(mig_jcr->db));
369 Dmsg4(dbglevel, "mig_jcr: Name=%s JobId=%d Type=%c Level=%c\n",
370 mig_jcr->jr.Name, (int)mig_jcr->jr.JobId,
371 mig_jcr->jr.JobType, mig_jcr->jr.JobLevel);
375 * Start the job prior to starting the message thread below
376 * to avoid two threads from using the BSOCK structure at
379 if (!sd->fsend("run")) {
384 * Now start a Storage daemon message thread
386 if (!start_storage_daemon_message_thread(jcr)) {
391 set_jcr_job_status(jcr, JS_Running);
392 set_jcr_job_status(mig_jcr, JS_Running);
394 /* Pickup Job termination data */
395 /* Note, the SD stores in jcr->JobFiles/ReadBytes/JobBytes/Errors */
396 wait_for_storage_daemon_termination(jcr);
397 set_jcr_job_status(jcr, jcr->SDJobStatus);
398 db_write_batch_file_records(jcr); /* used by bulk batch file insert */
399 if (jcr->JobStatus != JS_Terminated) {
403 migration_cleanup(jcr, jcr->JobStatus);
404 if (jcr->get_JobType() == JT_MIGRATE && mig_jcr) {
406 UAContext *ua = new_ua_context(jcr);
407 edit_uint64(jcr->previous_jr.JobId, jobid);
408 /* Purge all old file records, but leave Job record */
409 purge_files_from_jobs(ua, jobid);
420 /* Add an item to the list if it is unique */
421 static void add_unique_id(idpkt *ids, char *item)
423 const int maxlen = 30;
427 /* Walk through current list to see if each item is the same as item */
430 for (int i=0; i<maxlen; i++) {
433 } else if (*q == ',') {
440 if (strcmp(item, id) == 0) {
444 /* Did not find item, so add it to list */
445 if (ids->count == 0) {
448 pm_strcat(ids->list, ",");
450 pm_strcat(ids->list, item);
452 // Dmsg3(0, "add_uniq count=%d Ids=%p %s\n", ids->count, ids->list, ids->list);
457 * Callback handler make list of DB Ids
459 static int unique_dbid_handler(void *ctx, int num_fields, char **row)
461 idpkt *ids = (idpkt *)ctx;
463 add_unique_id(ids, row[0]);
464 Dmsg3(dbglevel, "dbid_hdlr count=%d Ids=%p %s\n", ids->count, ids->list, ids->list);
474 static int item_compare(void *item1, void *item2)
476 uitem *i1 = (uitem *)item1;
477 uitem *i2 = (uitem *)item2;
478 return strcmp(i1->item, i2->item);
481 static int unique_name_handler(void *ctx, int num_fields, char **row)
483 dlist *list = (dlist *)ctx;
485 uitem *new_item = (uitem *)malloc(sizeof(uitem));
488 memset(new_item, 0, sizeof(uitem));
489 new_item->item = bstrdup(row[0]);
490 Dmsg1(dbglevel, "Unique_name_hdlr Item=%s\n", row[0]);
491 item = (uitem *)list->binary_insert((void *)new_item, item_compare);
492 if (item != new_item) { /* already in list */
493 free(new_item->item);
494 free((char *)new_item);
500 /* Get Job names in Pool */
501 const char *sql_job =
502 "SELECT DISTINCT Job.Name from Job,Pool"
503 " WHERE Pool.Name='%s' AND Job.PoolId=Pool.PoolId";
505 /* Get JobIds from regex'ed Job names */
506 const char *sql_jobids_from_job =
507 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Job,Pool"
508 " WHERE Job.Name='%s' AND Pool.Name='%s' AND Job.PoolId=Pool.PoolId"
509 " ORDER by Job.StartTime";
511 /* Get Client names in Pool */
512 const char *sql_client =
513 "SELECT DISTINCT Client.Name from Client,Pool,Job"
514 " WHERE Pool.Name='%s' AND Job.ClientId=Client.ClientId AND"
515 " Job.PoolId=Pool.PoolId";
517 /* Get JobIds from regex'ed Client names */
518 const char *sql_jobids_from_client =
519 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Job,Pool,Client"
520 " WHERE Client.Name='%s' AND Pool.Name='%s' AND Job.PoolId=Pool.PoolId"
521 " AND Job.ClientId=Client.ClientId AND Job.Type='B'"
522 " ORDER by Job.StartTime";
524 /* Get Volume names in Pool */
525 const char *sql_vol =
526 "SELECT DISTINCT VolumeName FROM Media,Pool WHERE"
527 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
528 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'";
530 /* Get JobIds from regex'ed Volume names */
531 const char *sql_jobids_from_vol =
532 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Media,JobMedia,Job"
533 " WHERE Media.VolumeName='%s' AND Media.MediaId=JobMedia.MediaId"
534 " AND JobMedia.JobId=Job.JobId AND Job.Type='B'"
535 " ORDER by Job.StartTime";
538 const char *sql_smallest_vol =
539 "SELECT Media.MediaId FROM Media,Pool,JobMedia WHERE"
540 " Media.MediaId in (SELECT DISTINCT MediaId from JobMedia) AND"
541 " Media.VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
542 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'"
543 " ORDER BY VolBytes ASC LIMIT 1";
545 const char *sql_oldest_vol =
546 "SELECT Media.MediaId FROM Media,Pool,JobMedia WHERE"
547 " Media.MediaId in (SELECT DISTINCT MediaId from JobMedia) AND"
548 " Media.VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
549 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'"
550 " ORDER BY LastWritten ASC LIMIT 1";
552 /* Get JobIds when we have selected MediaId */
553 const char *sql_jobids_from_mediaid =
554 "SELECT DISTINCT Job.JobId,Job.StartTime FROM JobMedia,Job"
555 " WHERE JobMedia.JobId=Job.JobId AND JobMedia.MediaId IN (%s)"
557 " ORDER by Job.StartTime";
559 /* Get tne number of bytes in the pool */
560 const char *sql_pool_bytes =
561 "SELECT SUM(VolBytes) FROM Media,Pool WHERE"
562 " VolStatus in ('Full','Used','Error','Append') AND Media.Enabled=1 AND"
563 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'";
565 /* Get tne number of bytes in the Jobs */
566 const char *sql_job_bytes =
567 "SELECT SUM(JobBytes) FROM Job WHERE JobId IN (%s)";
570 /* Get Media Ids in Pool */
571 const char *sql_mediaids =
572 "SELECT MediaId FROM Media,Pool WHERE"
573 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
574 " Media.PoolId=Pool.PoolId AND Pool.Name='%s' ORDER BY LastWritten ASC";
576 /* Get JobIds in Pool longer than specified time */
577 const char *sql_pool_time =
578 "SELECT DISTINCT Job.JobId from Pool,Job,Media,JobMedia WHERE"
579 " Pool.Name='%s' AND Media.PoolId=Pool.PoolId AND"
580 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
582 " JobMedia.JobId=Job.JobId AND Job.PoolId=Media.PoolId"
583 " AND Job.RealEndTime<='%s'";
586 * const char *sql_ujobid =
587 * "SELECT DISTINCT Job.Job from Client,Pool,Media,Job,JobMedia "
588 * " WHERE Media.PoolId=Pool.PoolId AND Pool.Name='%s' AND"
589 * " JobMedia.JobId=Job.JobId AND Job.PoolId=Media.PoolId";
596 * This is the central piece of code that finds a job or jobs
597 * actually JobIds to migrate. It first looks to see if one
598 * has been "manually" specified in jcr->MigrateJobId, and if
599 * so, it returns that JobId to be run. Otherwise, it
600 * examines the Selection Type to see what kind of migration
601 * we are doing (Volume, Job, Client, ...) and applies any
602 * Selection Pattern if appropriate to obtain a list of JobIds.
603 * Finally, it will loop over all the JobIds found, except the last
604 * one starting a new job with MigrationJobId set to that JobId, and
605 * finally, it returns the last JobId to the caller.
607 * Returns: -1 on error
608 * 0 if no jobs to migrate
609 * 1 if OK and jcr->previous_jr filled in
611 static int get_job_to_migrate(JCR *jcr)
613 char ed1[30], ed2[30];
614 POOL_MEM query(PM_MESSAGE);
619 idpkt ids, mid, jids;
624 char dt[MAX_TIME_LENGTH];
627 ids.list = get_pool_memory(PM_MESSAGE);
630 mid.list = get_pool_memory(PM_MESSAGE);
633 jids.list = get_pool_memory(PM_MESSAGE);
639 * If MigrateJobId is set, then we migrate only that Job,
640 * otherwise, we go through the full selection of jobs to
643 if (jcr->MigrateJobId != 0) {
644 Dmsg1(dbglevel, "At Job start previous jobid=%u\n", jcr->MigrateJobId);
645 edit_uint64(jcr->MigrateJobId, ids.list);
648 switch (jcr->job->selection_type) {
650 if (!regex_find_jobids(jcr, &ids, sql_job, sql_jobids_from_job, "Job")) {
655 if (!regex_find_jobids(jcr, &ids, sql_client, sql_jobids_from_client, "Client")) {
660 if (!regex_find_jobids(jcr, &ids, sql_vol, sql_jobids_from_vol, "Volume")) {
665 if (!jcr->job->selection_pattern) {
666 Jmsg(jcr, M_FATAL, 0, _("No Migration SQL selection pattern specified.\n"));
669 Dmsg1(dbglevel, "SQL=%s\n", jcr->job->selection_pattern);
670 if (!db_sql_query(jcr->db, jcr->job->selection_pattern,
671 unique_dbid_handler, (void *)&ids)) {
672 Jmsg(jcr, M_FATAL, 0,
673 _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
677 case MT_SMALLEST_VOL:
678 if (!find_mediaid_then_jobids(jcr, &ids, sql_smallest_vol, "Smallest Volume")) {
683 if (!find_mediaid_then_jobids(jcr, &ids, sql_oldest_vol, "Oldest Volume")) {
688 case MT_POOL_OCCUPANCY:
690 /* Find count of bytes in pool */
691 Mmsg(query, sql_pool_bytes, jcr->rpool->name());
692 if (!db_sql_query(jcr->db, query.c_str(), db_int64_handler, (void *)&ctx)) {
693 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
696 if (ctx.count == 0) {
697 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
700 pool_bytes = ctx.value;
701 Dmsg2(dbglevel, "highbytes=%lld pool=%lld\n", jcr->rpool->MigrationHighBytes,
703 if (pool_bytes < (int64_t)jcr->rpool->MigrationHighBytes) {
704 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
707 Dmsg0(dbglevel, "We should do Occupation migration.\n");
710 /* Find a list of MediaIds that could be migrated */
711 Mmsg(query, sql_mediaids, jcr->rpool->name());
712 Dmsg1(dbglevel, "query=%s\n", query.c_str());
713 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)&ids)) {
714 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
717 if (ids.count == 0) {
718 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
721 Dmsg2(dbglevel, "Pool Occupancy ids=%d MediaIds=%s\n", ids.count, ids.list);
723 if (!find_jobids_from_mediaid_list(jcr, &ids, "Volumes")) {
726 /* ids == list of jobs */
728 for (int i=0; i < (int)ids.count; i++) {
729 stat = get_next_dbid_from_list(&p, &DBId);
730 Dmsg2(dbglevel, "get_next_dbid stat=%d JobId=%u\n", stat, (uint32_t)DBId);
732 Jmsg(jcr, M_FATAL, 0, _("Invalid JobId found.\n"));
734 } else if (stat == 0) {
739 Mmsg(mid.list, "%s", edit_int64(DBId, ed1));
740 if (jids.count > 0) {
741 pm_strcat(jids.list, ",");
743 pm_strcat(jids.list, mid.list);
744 jids.count += mid.count;
746 /* Find count of bytes from Jobs */
747 Mmsg(query, sql_job_bytes, mid.list);
748 Dmsg1(dbglevel, "Jobbytes query: %s\n", query.c_str());
749 if (!db_sql_query(jcr->db, query.c_str(), db_int64_handler, (void *)&ctx)) {
750 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
753 pool_bytes -= ctx.value;
754 Dmsg1(dbglevel, "Total migrate Job bytes=%s\n", edit_int64(ctx.value, ed1));
755 Dmsg2(dbglevel, "lowbytes=%s poolafter=%s\n",
756 edit_int64(jcr->rpool->MigrationLowBytes, ed1),
757 edit_int64(pool_bytes, ed2));
758 if (pool_bytes <= (int64_t)jcr->rpool->MigrationLowBytes) {
759 Dmsg0(dbglevel, "We should be done.\n");
763 /* Transfer jids to ids, where the jobs list is expected */
764 ids.count = jids.count;
765 pm_strcpy(ids.list, jids.list);
766 Dmsg2(dbglevel, "Pool Occupancy ids=%d JobIds=%s\n", ids.count, ids.list);
770 ttime = time(NULL) - (time_t)jcr->rpool->MigrationTime;
771 (void)localtime_r(&ttime, &tm);
772 strftime(dt, sizeof(dt), "%Y-%m-%d %H:%M:%S", &tm);
775 Mmsg(query, sql_pool_time, jcr->rpool->name(), dt);
776 Dmsg1(dbglevel, "query=%s\n", query.c_str());
777 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)&ids)) {
778 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
781 if (ids.count == 0) {
782 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
785 Dmsg2(dbglevel, "PoolTime ids=%d JobIds=%s\n", ids.count, ids.list);
789 Jmsg(jcr, M_FATAL, 0, _("Unknown Migration Selection Type.\n"));
795 * Loop over all jobids except the last one, sending
796 * them to start_migration_job(), which will start a job
797 * for each of them. For the last JobId, we handle it below.
800 if (ids.count == 0) {
801 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
805 Jmsg(jcr, M_INFO, 0, _("The following %u JobId%s were chosen to be migrated: %s\n"),
806 ids.count, ids.count==0?"":"s", ids.list);
808 Dmsg2(dbglevel, "Before loop count=%d ids=%s\n", ids.count, ids.list);
809 for (int i=1; i < (int)ids.count; i++) {
811 stat = get_next_jobid_from_list(&p, &JobId);
812 Dmsg3(dbglevel, "get_jobid_no=%d stat=%d JobId=%u\n", i, stat, JobId);
813 jcr->MigrateJobId = JobId;
814 start_migration_job(jcr);
815 Dmsg0(dbglevel, "Back from start_migration_job\n");
817 Jmsg(jcr, M_FATAL, 0, _("Invalid JobId found.\n"));
819 } else if (stat == 0) {
820 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
825 /* Now get the last JobId and handle it in the current job */
827 stat = get_next_jobid_from_list(&p, &JobId);
828 Dmsg2(dbglevel, "Last get_next_jobid stat=%d JobId=%u\n", stat, (int)JobId);
830 Jmsg(jcr, M_FATAL, 0, _("Invalid JobId found.\n"));
832 } else if (stat == 0) {
833 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
837 jcr->previous_jr.JobId = JobId;
838 Dmsg1(dbglevel, "Previous jobid=%d\n", (int)jcr->previous_jr.JobId);
840 if (!db_get_job_record(jcr, jcr->db, &jcr->previous_jr)) {
841 Jmsg(jcr, M_FATAL, 0, _("Could not get job record for JobId %s to migrate. ERR=%s"),
842 edit_int64(jcr->previous_jr.JobId, ed1),
843 db_strerror(jcr->db));
846 Jmsg(jcr, M_INFO, 0, _("Migration using JobId=%s Job=%s\n"),
847 edit_int64(jcr->previous_jr.JobId, ed1), jcr->previous_jr.Job);
848 Dmsg3(dbglevel, "Migration JobId=%d using JobId=%s Job=%s\n",
850 edit_int64(jcr->previous_jr.JobId, ed1), jcr->previous_jr.Job);
860 free_pool_memory(ids.list);
861 free_pool_memory(mid.list);
862 free_pool_memory(jids.list);
866 static void start_migration_job(JCR *jcr)
868 UAContext *ua = new_ua_context(jcr);
871 Mmsg(ua->cmd, "run %s jobid=%s", jcr->job->hdr.name,
872 edit_uint64(jcr->MigrateJobId, ed1));
873 Dmsg1(dbglevel, "=============== Migration cmd=%s\n", ua->cmd);
874 parse_ua_args(ua); /* parse command */
875 JobId_t jobid = run_cmd(ua, ua->cmd);
877 Jmsg(jcr, M_ERROR, 0, _("Could not start migration job.\n"));
879 Jmsg(jcr, M_INFO, 0, _("Migration JobId %d started.\n"), (int)jobid);
884 static bool find_mediaid_then_jobids(JCR *jcr, idpkt *ids, const char *query1,
888 POOL_MEM query(PM_MESSAGE);
891 /* Basic query for MediaId */
892 Mmsg(query, query1, jcr->rpool->name());
893 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
894 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
897 if (ids->count == 0) {
898 Jmsg(jcr, M_INFO, 0, _("No %s found to migrate.\n"), type);
899 ok = true; /* Not an error */
901 } else if (ids->count != 1) {
902 Jmsg(jcr, M_FATAL, 0, _("SQL error. Expected 1 MediaId got %d\n"), ids->count);
905 Dmsg2(dbglevel, "%s MediaIds=%s\n", type, ids->list);
907 ok = find_jobids_from_mediaid_list(jcr, ids, type);
914 * This routine returns:
915 * false if an error occurred
917 * ids.count number of jobids found (may be zero)
919 static bool find_jobids_from_mediaid_list(JCR *jcr, idpkt *ids, const char *type)
922 POOL_MEM query(PM_MESSAGE);
924 Mmsg(query, sql_jobids_from_mediaid, ids->list);
926 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
927 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
930 if (ids->count == 0) {
931 Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type);
939 static bool regex_find_jobids(JCR *jcr, idpkt *ids, const char *query1,
940 const char *query2, const char *type)
944 uitem *last_item = NULL;
949 POOL_MEM query(PM_MESSAGE);
951 item_chain = New(dlist(item, &item->link));
952 if (!jcr->job->selection_pattern) {
953 Jmsg(jcr, M_FATAL, 0, _("No Migration %s selection pattern specified.\n"),
957 Dmsg1(dbglevel, "regex-sel-pattern=%s\n", jcr->job->selection_pattern);
958 /* Basic query for names */
959 Mmsg(query, query1, jcr->rpool->name());
960 Dmsg1(dbglevel, "get name query1=%s\n", query.c_str());
961 if (!db_sql_query(jcr->db, query.c_str(), unique_name_handler,
962 (void *)item_chain)) {
963 Jmsg(jcr, M_FATAL, 0,
964 _("SQL to get %s failed. ERR=%s\n"), type, db_strerror(jcr->db));
967 Dmsg1(dbglevel, "query1 returned %d names\n", item_chain->size());
968 if (item_chain->size() == 0) {
969 Jmsg(jcr, M_INFO, 0, _("Query of Pool \"%s\" returned no Jobs to migrate.\n"),
972 goto bail_out; /* skip regex match */
974 /* Compile regex expression */
975 rc = regcomp(&preg, jcr->job->selection_pattern, REG_EXTENDED);
977 regerror(rc, &preg, prbuf, sizeof(prbuf));
978 Jmsg(jcr, M_FATAL, 0, _("Could not compile regex pattern \"%s\" ERR=%s\n"),
979 jcr->job->selection_pattern, prbuf);
982 /* Now apply the regex to the names and remove any item not matched */
983 foreach_dlist(item, item_chain) {
984 const int nmatch = 30;
985 regmatch_t pmatch[nmatch];
987 Dmsg1(dbglevel, "Remove item %s\n", last_item->item);
988 free(last_item->item);
989 item_chain->remove(last_item);
991 Dmsg1(dbglevel, "get name Item=%s\n", item->item);
992 rc = regexec(&preg, item->item, nmatch, pmatch, 0);
994 last_item = NULL; /* keep this one */
1000 free(last_item->item);
1001 Dmsg1(dbglevel, "Remove item %s\n", last_item->item);
1002 item_chain->remove(last_item);
1006 if (item_chain->size() == 0) {
1007 Jmsg(jcr, M_INFO, 0, _("Regex pattern matched no Jobs to migrate.\n"));
1009 goto bail_out; /* skip regex match */
1013 * At this point, we have a list of items in item_chain
1014 * that have been matched by the regex, so now we need
1015 * to look up their jobids.
1018 foreach_dlist(item, item_chain) {
1019 Dmsg2(dbglevel, "Got %s: %s\n", type, item->item);
1020 Mmsg(query, query2, item->item, jcr->rpool->name());
1021 Dmsg1(dbglevel, "get id from name query2=%s\n", query.c_str());
1022 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
1023 Jmsg(jcr, M_FATAL, 0,
1024 _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
1028 if (ids->count == 0) {
1029 Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type);
1034 Dmsg2(dbglevel, "Count=%d Jobids=%s\n", ids->count, ids->list);
1035 foreach_dlist(item, item_chain) {
1044 * Release resources allocated during backup.
1046 void migration_cleanup(JCR *jcr, int TermCode)
1048 char sdt[MAX_TIME_LENGTH], edt[MAX_TIME_LENGTH];
1049 char ec1[30], ec2[30], ec3[30], ec4[30], ec5[30], elapsed[50];
1050 char ec6[50], ec7[50], ec8[50];
1051 char term_code[100], sd_term_msg[100];
1052 const char *term_msg;
1053 int msg_type = M_INFO;
1057 JCR *mig_jcr = jcr->mig_jcr;
1058 POOL_MEM query(PM_MESSAGE);
1060 Dmsg2(100, "Enter migrate_cleanup %d %c\n", TermCode, TermCode);
1061 update_job_end(jcr, TermCode);
1062 memset(&mr, 0, sizeof(mr));
1065 * Check if we actually did something.
1066 * mig_jcr is jcr of the newly migrated job.
1069 mig_jcr->JobFiles = jcr->JobFiles = jcr->SDJobFiles;
1070 mig_jcr->JobBytes = jcr->JobBytes = jcr->SDJobBytes;
1071 mig_jcr->VolSessionId = jcr->VolSessionId;
1072 mig_jcr->VolSessionTime = jcr->VolSessionTime;
1073 mig_jcr->jr.RealEndTime = 0;
1074 mig_jcr->jr.PriorJobId = jcr->previous_jr.JobId;
1076 update_job_end(mig_jcr, TermCode);
1078 /* Update final items to set them to the previous job's values */
1079 Mmsg(query, "UPDATE Job SET StartTime='%s',EndTime='%s',"
1080 "JobTDate=%s WHERE JobId=%s",
1081 jcr->previous_jr.cStartTime, jcr->previous_jr.cEndTime,
1082 edit_uint64(jcr->previous_jr.JobTDate, ec1),
1083 edit_uint64(mig_jcr->jr.JobId, ec2));
1084 db_sql_query(mig_jcr->db, query.c_str(), NULL, NULL);
1086 /* Now mark the previous job as migrated if it terminated normally */
1087 if (jcr->get_JobType() == JT_MIGRATE && jcr->JobStatus == JS_Terminated) {
1088 Mmsg(query, "UPDATE Job SET Type='%c' WHERE JobId=%s",
1089 (char)JT_MIGRATED_JOB, edit_uint64(jcr->previous_jr.JobId, ec1));
1090 db_sql_query(mig_jcr->db, query.c_str(), NULL, NULL);
1093 if (!db_get_job_record(jcr, jcr->db, &jcr->jr)) {
1094 Jmsg(jcr, M_WARNING, 0, _("Error getting Job record for Job report: ERR=%s"),
1095 db_strerror(jcr->db));
1096 set_jcr_job_status(jcr, JS_ErrorTerminated);
1099 update_bootstrap_file(mig_jcr);
1101 if (!db_get_job_volume_names(mig_jcr, mig_jcr->db, mig_jcr->jr.JobId, &mig_jcr->VolumeName)) {
1103 * Note, if the job has failed, most likely it did not write any
1104 * tape, so suppress this "error" message since in that case
1105 * it is normal. Or look at it the other way, only for a
1106 * normal exit should we complain about this error.
1108 if (jcr->JobStatus == JS_Terminated && jcr->jr.JobBytes) {
1109 Jmsg(jcr, M_ERROR, 0, "%s", db_strerror(mig_jcr->db));
1111 mig_jcr->VolumeName[0] = 0; /* none */
1114 if (mig_jcr->VolumeName[0]) {
1115 /* Find last volume name. Multiple vols are separated by | */
1116 char *p = strrchr(mig_jcr->VolumeName, '|');
1120 p = mig_jcr->VolumeName; /* no |, take full name */
1122 bstrncpy(mr.VolumeName, p, sizeof(mr.VolumeName));
1123 if (!db_get_media_record(jcr, jcr->db, &mr)) {
1124 Jmsg(jcr, M_WARNING, 0, _("Error getting Media record for Volume \"%s\": ERR=%s"),
1125 mr.VolumeName, db_strerror(jcr->db));
1129 switch (jcr->JobStatus) {
1131 if (jcr->Errors || jcr->SDErrors) {
1132 term_msg = _("%s OK -- with warnings");
1134 term_msg = _("%s OK");
1138 case JS_ErrorTerminated:
1139 term_msg = _("*** %s Error ***");
1140 msg_type = M_ERROR; /* Generate error message */
1141 if (jcr->store_bsock) {
1142 bnet_sig(jcr->store_bsock, BNET_TERMINATE);
1143 if (jcr->SD_msg_chan) {
1144 pthread_cancel(jcr->SD_msg_chan);
1149 term_msg = _("%s Canceled");
1150 if (jcr->store_bsock) {
1151 bnet_sig(jcr->store_bsock, BNET_TERMINATE);
1152 if (jcr->SD_msg_chan) {
1153 pthread_cancel(jcr->SD_msg_chan);
1158 term_msg = _("Inappropriate %s term code");
1162 if (jcr->get_JobType() == JT_MIGRATE && jcr->previous_jr.JobId != 0) {
1163 /* Mark previous job as migrated */
1164 Mmsg(query, "UPDATE Job SET Type='%c' WHERE JobId=%s",
1165 (char)JT_MIGRATED_JOB, edit_uint64(jcr->previous_jr.JobId, ec1));
1166 db_sql_query(jcr->db, query.c_str(), NULL, NULL);
1168 term_msg = _("%s -- no files to migrate");
1171 bsnprintf(term_code, sizeof(term_code), term_msg, "Migration");
1172 bstrftimes(sdt, sizeof(sdt), jcr->jr.StartTime);
1173 bstrftimes(edt, sizeof(edt), jcr->jr.EndTime);
1174 RunTime = jcr->jr.EndTime - jcr->jr.StartTime;
1178 kbps = (double)jcr->SDJobBytes / (1000 * RunTime);
1182 jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg));
1184 Jmsg(jcr, msg_type, 0, _("Bacula %s %s (%s): %s\n"
1185 " Build OS: %s %s %s\n"
1186 " Prev Backup JobId: %s\n"
1187 " New Backup JobId: %s\n"
1188 " Current JobId: %s\n"
1189 " Current Job: %s\n"
1190 " Backup Level: %s%s\n"
1192 " FileSet: \"%s\" %s\n"
1193 " Read Pool: \"%s\" (From %s)\n"
1194 " Read Storage: \"%s\" (From %s)\n"
1195 " Write Pool: \"%s\" (From %s)\n"
1196 " Write Storage: \"%s\" (From %s)\n"
1197 " Catalog: \"%s\" (From %s)\n"
1200 " Elapsed time: %s\n"
1202 " SD Files Written: %s\n"
1203 " SD Bytes Written: %s (%sB)\n"
1204 " Rate: %.1f KB/s\n"
1205 " Volume name(s): %s\n"
1206 " Volume Session Id: %d\n"
1207 " Volume Session Time: %d\n"
1208 " Last Volume Bytes: %s (%sB)\n"
1210 " SD termination status: %s\n"
1211 " Termination: %s\n\n"),
1212 my_name, VERSION, LSMDATE, edt,
1213 HOST_OS, DISTNAME, DISTVER,
1214 edit_uint64(jcr->previous_jr.JobId, ec6),
1215 mig_jcr ? edit_uint64(mig_jcr->jr.JobId, ec7) : "0",
1216 edit_uint64(jcr->jr.JobId, ec8),
1218 level_to_str(jcr->get_JobLevel()), jcr->since,
1219 jcr->client->name(),
1220 jcr->fileset->name(), jcr->FSCreateTime,
1221 jcr->rpool->name(), jcr->rpool_source,
1222 jcr->rstore?jcr->rstore->name():"*None*",
1223 NPRT(jcr->rstore_source),
1224 jcr->pool->name(), jcr->pool_source,
1225 jcr->wstore?jcr->wstore->name():"*None*",
1226 NPRT(jcr->wstore_source),
1227 jcr->catalog->name(), jcr->catalog_source,
1230 edit_utime(RunTime, elapsed, sizeof(elapsed)),
1232 edit_uint64_with_commas(jcr->SDJobFiles, ec1),
1233 edit_uint64_with_commas(jcr->SDJobBytes, ec2),
1234 edit_uint64_with_suffix(jcr->SDJobBytes, ec3),
1236 mig_jcr ? mig_jcr->VolumeName : "",
1238 jcr->VolSessionTime,
1239 edit_uint64_with_commas(mr.VolBytes, ec4),
1240 edit_uint64_with_suffix(mr.VolBytes, ec5),
1245 Dmsg1(100, "migrate_cleanup() mig_jcr=0x%x\n", jcr->mig_jcr);
1247 free_jcr(jcr->mig_jcr);
1248 jcr->mig_jcr = NULL;
1250 Dmsg0(100, "Leave migrate_cleanup()\n");
1254 * Return next DBId from comma separated list
1257 * 1 if next DBId returned
1258 * 0 if no more DBIds are in list
1259 * -1 there is an error
1261 static int get_next_dbid_from_list(char **p, DBId_t *DBId)
1263 const int maxlen = 30;
1268 for (int i=0; i<maxlen; i++) {
1271 } else if (*q == ',') {
1280 } else if (!is_a_number(id)) {
1281 return -1; /* error */
1284 *DBId = str_to_int64(id);
1288 bool set_migration_wstorage(JCR *jcr, POOL *pool)
1290 POOL *wpool = pool->NextPool;
1293 Jmsg(jcr, M_FATAL, 0, _("No Next Pool specification found in Pool \"%s\".\n"),
1298 if (!wpool->storage || wpool->storage->size() == 0) {
1299 Jmsg(jcr, M_FATAL, 0, _("No Storage specification found in Next Pool \"%s\".\n"),
1304 /* If pool storage specified, use it instead of job storage for backup */
1305 copy_wstorage(jcr, wpool->storage, _("Storage from Pool's NextPool resource"));