3 * Bacula Director -- migrate.c -- responsible for doing
6 * Kern Sibbald, September MMIV
8 * Basic tasks done here:
9 * Open DB and create records for this job.
10 * Open Message Channel with Storage daemon to tell him a job will be starting.
11 * Open connection with Storage daemon and pass him commands
13 * When the Storage daemon finishes the job, update the DB.
18 Bacula® - The Network Backup Solution
20 Copyright (C) 2004-2006 Free Software Foundation Europe e.V.
22 The main author of Bacula is Kern Sibbald, with contributions from
23 many others, a complete list can be found in the file AUTHORS.
24 This program is Free Software; you can redistribute it and/or
25 modify it under the terms of version two of the GNU General Public
26 License as published by the Free Software Foundation plus additions
27 that are listed in the file LICENSE.
29 This program is distributed in the hope that it will be useful, but
30 WITHOUT ANY WARRANTY; without even the implied warranty of
31 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
32 General Public License for more details.
34 You should have received a copy of the GNU General Public License
35 along with this program; if not, write to the Free Software
36 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
39 Bacula® is a registered trademark of John Walker.
40 The licensor of Bacula is the Free Software Foundation Europe
41 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
42 Switzerland, email:ftf@fsfeurope.org.
49 #include "lib/bregex.h"
54 static const int dbglevel = 10;
56 static char OKbootstrap[] = "3000 OK bootstrap\n";
57 static int get_job_to_migrate(JCR *jcr);
59 static bool regex_find_jobids(JCR *jcr, idpkt *ids, const char *query1,
60 const char *query2, const char *type);
61 static bool find_mediaid_then_jobids(JCR *jcr, idpkt *ids, const char *query1,
63 static bool find_jobids_from_mediaid_list(JCR *jcr, idpkt *ids, const char *type);
64 static void start_migration_job(JCR *jcr);
65 static int get_next_dbid_from_list(char **p, DBId_t *DBId);
68 * Called here before the job is run to do the job
69 * specific setup. Note, one of the important things to
70 * complete in this init code is to make the definitive
71 * choice of input and output storage devices. This is
72 * because immediately after the init, the job is queued
73 * in the jobq.c code, and it checks that all the resources
74 * (storage resources in particular) are available, so these
75 * must all be properly defined.
77 * previous_jr refers to the job DB record of the Job that is
78 * going to be migrated.
79 * prev_job refers to the job resource of the Job that is
80 * going to be migrated.
81 * jcr is the jcr for the current "migration" job. It is a
82 * control job that is put in the DB as a migration job, which
83 * means that this job migrated a previous job to a new job.
84 * No Volume or File data is associated with this control
86 * mig_jcr refers to the newly migrated job that is run by
87 * the current jcr. It is a backup job that moves (migrates) the
88 * data written for the previous_jr into the new pool. This
89 * job (mig_jcr) becomes the new backup job that replaces
90 * the original backup job.
92 bool do_migration_init(JCR *jcr)
98 JCR *mig_jcr; /* newly migrated job */
101 /* If we find a job or jobs to migrate it is previous_jr.JobId */
102 count = get_job_to_migrate(jcr);
110 Dmsg1(dbglevel, "Back from get_job_to_migrate JobId=%d\n", (int)jcr->JobId);
112 if (jcr->previous_jr.JobId == 0) {
113 Dmsg1(dbglevel, "JobId=%d no previous JobId\n", (int)jcr->JobId);
114 Jmsg(jcr, M_INFO, 0, _("No previous Job found to migrate.\n"));
115 return true; /* no work */
118 if (!get_or_create_fileset_record(jcr)) {
119 Dmsg1(dbglevel, "JobId=%d no FileSet\n", (int)jcr->JobId);
120 Jmsg(jcr, M_FATAL, 0, _("Could not get or create the FileSet record.\n"));
124 apply_pool_overrides(jcr);
126 jcr->jr.PoolId = get_or_create_pool_record(jcr, jcr->pool->hdr.name);
127 if (jcr->jr.PoolId == 0) {
128 Dmsg1(dbglevel, "JobId=%d no PoolId\n", (int)jcr->JobId);
129 Jmsg(jcr, M_FATAL, 0, _("Could not get or create a Pool record.\n"));
133 create_restore_bootstrap_file(jcr);
135 if (jcr->previous_jr.JobId == 0 || jcr->ExpectedFiles == 0) {
136 set_jcr_job_status(jcr, JS_Terminated);
137 Dmsg1(dbglevel, "JobId=%d expected files == 0\n", (int)jcr->JobId);
138 if (jcr->previous_jr.JobId == 0) {
139 Jmsg(jcr, M_INFO, 0, _("No previous Job found to migrate.\n"));
141 Jmsg(jcr, M_INFO, 0, _("Previous Job has no data to migrate.\n"));
143 return true; /* no work */
146 Dmsg5(dbglevel, "JobId=%d: Previous: Name=%s JobId=%d Type=%c Level=%c\n",
148 jcr->previous_jr.Name, (int)jcr->previous_jr.JobId,
149 jcr->previous_jr.JobType, jcr->previous_jr.JobLevel);
151 Dmsg5(dbglevel, "JobId=%d: Current: Name=%s JobId=%d Type=%c Level=%c\n",
153 jcr->jr.Name, (int)jcr->jr.JobId,
154 jcr->jr.JobType, jcr->jr.JobLevel);
157 job = (JOB *)GetResWithName(R_JOB, jcr->jr.Name);
158 prev_job = (JOB *)GetResWithName(R_JOB, jcr->previous_jr.Name);
161 Jmsg(jcr, M_FATAL, 0, _("Job resource not found for \"%s\".\n"), jcr->jr.Name);
165 Jmsg(jcr, M_FATAL, 0, _("Previous Job resource not found for \"%s\".\n"),
166 jcr->previous_jr.Name);
170 /* Create a migation jcr */
171 mig_jcr = jcr->mig_jcr = new_jcr(sizeof(JCR), dird_free_jcr);
172 memcpy(&mig_jcr->previous_jr, &jcr->previous_jr, sizeof(mig_jcr->previous_jr));
175 * Turn the mig_jcr into a "real" job that takes on the aspects of
176 * the previous backup job "prev_job".
178 set_jcr_defaults(mig_jcr, prev_job);
179 if (!setup_job(mig_jcr)) {
180 Jmsg(jcr, M_FATAL, 0, _("setup job failed.\n"));
184 /* Now reset the job record from the previous job */
185 memcpy(&mig_jcr->jr, &jcr->previous_jr, sizeof(mig_jcr->jr));
186 /* Update the jr to reflect the new values of PoolId, FileSetId, and JobId. */
187 mig_jcr->jr.PoolId = jcr->jr.PoolId;
188 mig_jcr->jr.FileSetId = jcr->jr.FileSetId;
189 mig_jcr->jr.JobId = mig_jcr->JobId;
191 Dmsg4(dbglevel, "mig_jcr: Name=%s JobId=%d Type=%c Level=%c\n",
192 mig_jcr->jr.Name, (int)mig_jcr->jr.JobId,
193 mig_jcr->jr.JobType, mig_jcr->jr.JobLevel);
196 * Get the PoolId used with the original job. Then
197 * find the pool name from the database record.
199 memset(&pr, 0, sizeof(pr));
200 pr.PoolId = mig_jcr->previous_jr.PoolId;
201 if (!db_get_pool_record(jcr, jcr->db, &pr)) {
202 Jmsg(jcr, M_FATAL, 0, _("Pool for JobId %s not in database. ERR=%s\n"),
203 edit_int64(pr.PoolId, ed1), db_strerror(jcr->db));
206 /* Get the pool resource corresponding to the original job */
207 pool = (POOL *)GetResWithName(R_POOL, pr.Name);
209 Jmsg(jcr, M_FATAL, 0, _("Pool resource \"%s\" not found.\n"), pr.Name);
213 /* If pool storage specified, use it for restore */
214 copy_rstorage(mig_jcr, pool->storage, _("Pool resource"));
215 copy_rstorage(jcr, pool->storage, _("Pool resource"));
218 * If the original backup pool has a NextPool, make sure a
219 * record exists in the database. Note, in this case, we
220 * will be migrating from pool to pool->NextPool.
222 if (pool->NextPool) {
223 jcr->jr.PoolId = get_or_create_pool_record(jcr, pool->NextPool->hdr.name);
224 if (jcr->jr.PoolId == 0) {
228 * put the "NextPool" resource pointer in our jcr so that we
229 * can pull the Storage reference from it.
231 mig_jcr->pool = jcr->pool = pool->NextPool;
232 mig_jcr->jr.PoolId = jcr->jr.PoolId;
233 pm_strcpy(jcr->pool_source, _("NextPool in Pool resource"));
235 Jmsg(jcr, M_FATAL, 0, _("No Next Pool specification found in Pool \"%s\".\n"),
240 if (!jcr->pool->storage || jcr->pool->storage->size() == 0) {
241 Jmsg(jcr, M_FATAL, 0, _("No Storage specification found in Next Pool \"%s\".\n"),
242 jcr->pool->hdr.name);
246 /* If pool storage specified, use it instead of job storage for backup */
247 copy_wstorage(jcr, jcr->pool->storage, _("NextPool in Pool resource"));
253 * Do a Migration of a previous job
255 * Returns: false on failure
258 bool do_migration(JCR *jcr)
262 JCR *mig_jcr = jcr->mig_jcr; /* newly migrated job */
265 * If mig_jcr is NULL, there is nothing to do for this job,
266 * so set a normal status, cleanup and return OK.
269 set_jcr_job_status(jcr, JS_Terminated);
270 migration_cleanup(jcr, jcr->JobStatus);
274 /* Print Job Start message */
275 Jmsg(jcr, M_INFO, 0, _("Start Migration JobId %s, Job=%s\n"),
276 edit_uint64(jcr->JobId, ed1), jcr->Job);
278 set_jcr_job_status(jcr, JS_Running);
279 set_jcr_job_status(mig_jcr, JS_Running);
280 Dmsg2(dbglevel, "JobId=%d JobLevel=%c\n", (int)jcr->jr.JobId, jcr->jr.JobLevel);
282 /* Update job start record for this migration control job */
283 if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) {
284 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db));
288 Dmsg4(dbglevel, "mig_jcr: Name=%s JobId=%d Type=%c Level=%c\n",
289 mig_jcr->jr.Name, (int)mig_jcr->jr.JobId,
290 mig_jcr->jr.JobType, mig_jcr->jr.JobLevel);
292 /* Update job start record for the real migration backup job */
293 if (!db_update_job_start_record(mig_jcr, mig_jcr->db, &mig_jcr->jr)) {
294 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(mig_jcr->db));
300 * Open a message channel connection with the Storage
301 * daemon. This is to let him know that our client
302 * will be contacting him for a backup session.
305 Dmsg0(110, "Open connection with storage daemon\n");
306 set_jcr_job_status(jcr, JS_WaitSD);
307 set_jcr_job_status(mig_jcr, JS_WaitSD);
309 * Start conversation with Storage daemon
311 if (!connect_to_storage_daemon(jcr, 10, SDConnectTimeout, 1)) {
314 sd = jcr->store_bsock;
316 * Now start a job with the Storage daemon
318 Dmsg2(dbglevel, "Read store=%s, write store=%s\n",
319 ((STORE *)jcr->rstorage->first())->name(),
320 ((STORE *)jcr->wstorage->first())->name());
321 if (((STORE *)jcr->rstorage->first())->name() == ((STORE *)jcr->wstorage->first())->name()) {
322 Jmsg(jcr, M_FATAL, 0, _("Read storage \"%s\" same as write storage.\n"),
323 ((STORE *)jcr->rstorage->first())->name());
326 if (!start_storage_daemon_job(jcr, jcr->rstorage, jcr->wstorage)) {
329 Dmsg0(150, "Storage daemon connection OK\n");
331 if (!send_bootstrap_file(jcr, sd) ||
332 !response(jcr, sd, OKbootstrap, "Bootstrap", DISPLAY_ERROR)) {
336 if (!bnet_fsend(sd, "run")) {
341 * Now start a Storage daemon message thread
343 if (!start_storage_daemon_message_thread(jcr)) {
348 set_jcr_job_status(jcr, JS_Running);
349 set_jcr_job_status(mig_jcr, JS_Running);
351 /* Pickup Job termination data */
352 /* Note, the SD stores in jcr->JobFiles/ReadBytes/JobBytes/Errors */
353 wait_for_storage_daemon_termination(jcr);
355 set_jcr_job_status(jcr, jcr->SDJobStatus);
356 if (jcr->JobStatus != JS_Terminated) {
360 migration_cleanup(jcr, jcr->JobStatus);
362 UAContext *ua = new_ua_context(jcr);
363 purge_job_records_from_catalog(ua, jcr->previous_jr.JobId);
374 /* Add an item to the list if it is unique */
375 static void add_unique_id(idpkt *ids, char *item)
380 /* Walk through current list to see if each item is the same as item */
383 for (int i=0; i<(int)sizeof(id); i++) {
386 } else if (*q == ',') {
393 if (strcmp(item, id) == 0) {
397 /* Did not find item, so add it to list */
398 if (ids->count == 0) {
401 pm_strcat(ids->list, ",");
403 pm_strcat(ids->list, item);
405 // Dmsg3(0, "add_uniq count=%d Ids=%p %s\n", ids->count, ids->list, ids->list);
410 * Callback handler make list of DB Ids
412 static int unique_dbid_handler(void *ctx, int num_fields, char **row)
414 idpkt *ids = (idpkt *)ctx;
416 add_unique_id(ids, row[0]);
417 Dmsg3(dbglevel, "dbid_hdlr count=%d Ids=%p %s\n", ids->count, ids->list, ids->list);
427 static int item_compare(void *item1, void *item2)
429 uitem *i1 = (uitem *)item1;
430 uitem *i2 = (uitem *)item2;
431 return strcmp(i1->item, i2->item);
434 static int unique_name_handler(void *ctx, int num_fields, char **row)
436 dlist *list = (dlist *)ctx;
438 uitem *new_item = (uitem *)malloc(sizeof(uitem));
441 memset(new_item, 0, sizeof(uitem));
442 new_item->item = bstrdup(row[0]);
443 Dmsg1(dbglevel, "Unique_name_hdlr Item=%s\n", row[0]);
444 item = (uitem *)list->binary_insert((void *)new_item, item_compare);
445 if (item != new_item) { /* already in list */
446 free(new_item->item);
447 free((char *)new_item);
453 /* Get Job names in Pool */
454 const char *sql_job =
455 "SELECT DISTINCT Job.Name from Job,Pool"
456 " WHERE Pool.Name='%s' AND Job.PoolId=Pool.PoolId";
458 /* Get JobIds from regex'ed Job names */
459 const char *sql_jobids_from_job =
460 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Job,Pool"
461 " WHERE Job.Name='%s' AND Pool.Name='%s' AND Job.PoolId=Pool.PoolId"
462 " ORDER by Job.StartTime";
464 /* Get Client names in Pool */
465 const char *sql_client =
466 "SELECT DISTINCT Client.Name from Client,Pool,Job"
467 " WHERE Pool.Name='%s' AND Job.ClientId=Client.ClientId AND"
468 " Job.PoolId=Pool.PoolId";
470 /* Get JobIds from regex'ed Client names */
471 const char *sql_jobids_from_client =
472 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Job,Pool,Client"
473 " WHERE Client.Name='%s' AND Pool.Name='%s' AND Job.PoolId=Pool.PoolId"
474 " AND Job.ClientId=Client.ClientId "
475 " ORDER by Job.StartTime";
477 /* Get Volume names in Pool */
478 const char *sql_vol =
479 "SELECT DISTINCT VolumeName FROM Media,Pool WHERE"
480 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
481 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'";
483 /* Get JobIds from regex'ed Volume names */
484 const char *sql_jobids_from_vol =
485 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Media,JobMedia,Job"
486 " WHERE Media.VolumeName='%s' AND Media.MediaId=JobMedia.MediaId"
487 " AND JobMedia.JobId=Job.JobId"
488 " ORDER by Job.StartTime";
491 const char *sql_smallest_vol =
492 "SELECT MediaId FROM Media,Pool WHERE"
493 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
494 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'"
495 " ORDER BY VolBytes ASC LIMIT 1";
497 const char *sql_oldest_vol =
498 "SELECT MediaId FROM Media,Pool WHERE"
499 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
500 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'"
501 " ORDER BY LastWritten ASC LIMIT 1";
503 /* Get JobIds when we have selected MediaId */
504 const char *sql_jobids_from_mediaid =
505 "SELECT DISTINCT Job.JobId,Job.StartTime FROM JobMedia,Job"
506 " WHERE JobMedia.JobId=Job.JobId AND JobMedia.MediaId=%s"
507 " ORDER by Job.StartTime";
509 /* Get tne number of bytes in the pool */
510 const char *sql_pool_bytes =
511 "SELECT SUM(VolBytes) FROM Media,Pool WHERE"
512 " VolStatus in ('Full','Used','Error','Append') AND Media.Enabled=1 AND"
513 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'";
515 /* Get tne number of bytes in the Jobs */
516 const char *sql_job_bytes =
517 "SELECT SUM(JobBytes) FROM Job WHERE JobId IN (%s)";
520 /* Get Media Ids in Pool */
521 const char *sql_mediaids =
522 "SELECT MediaId FROM Media,Pool WHERE"
523 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
524 " Media.PoolId=Pool.PoolId AND Pool.Name='%s' ORDER BY LastWritten ASC";
526 /* Get JobIds in Pool longer than specified time */
527 const char *sql_pool_time =
528 "SELECT DISTINCT Job.JobId from Pool,Job,Media,JobMedia WHERE"
529 " Pool.Name='%s' AND Media.PoolId=Pool.PoolId AND"
530 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
531 " JobMedia.JobId=Job.JobId AND Job.PoolId=Media.PoolId"
532 " AND Job.RealEndTime<='%s'";
535 * const char *sql_ujobid =
536 * "SELECT DISTINCT Job.Job from Client,Pool,Media,Job,JobMedia "
537 * " WHERE Media.PoolId=Pool.PoolId AND Pool.Name='%s' AND"
538 * " JobMedia.JobId=Job.JobId AND Job.PoolId=Media.PoolId";
545 * This is the central piece of code that finds a job or jobs
546 * actually JobIds to migrate. It first looks to see if one
547 * has been "manually" specified in jcr->MigrateJobId, and if
548 * so, it returns that JobId to be run. Otherwise, it
549 * examines the Selection Type to see what kind of migration
550 * we are doing (Volume, Job, Client, ...) and applies any
551 * Selection Pattern if appropriate to obtain a list of JobIds.
552 * Finally, it will loop over all the JobIds found, except the last
553 * one starting a new job with MigrationJobId set to that JobId, and
554 * finally, it returns the last JobId to the caller.
556 * Returns: -1 on error
557 * 0 if no jobs to migrate
558 * 1 if OK and jcr->previous_jr filled in
560 static int get_job_to_migrate(JCR *jcr)
563 POOL_MEM query(PM_MESSAGE);
568 idpkt ids, mid, jids;
573 char dt[MAX_TIME_LENGTH];
576 ids.list = get_pool_memory(PM_MESSAGE);
579 mid.list = get_pool_memory(PM_MESSAGE);
582 jids.list = get_pool_memory(PM_MESSAGE);
588 * If MigrateJobId is set, then we migrate only that Job,
589 * otherwise, we go through the full selection of jobs to
592 if (jcr->MigrateJobId != 0) {
593 Dmsg1(dbglevel, "At Job start previous jobid=%u\n", jcr->MigrateJobId);
594 edit_uint64(jcr->MigrateJobId, ids.list);
597 switch (jcr->job->selection_type) {
599 if (!regex_find_jobids(jcr, &ids, sql_job, sql_jobids_from_job, "Job")) {
604 if (!regex_find_jobids(jcr, &ids, sql_client, sql_jobids_from_client, "Client")) {
609 if (!regex_find_jobids(jcr, &ids, sql_vol, sql_jobids_from_vol, "Volume")) {
614 if (!jcr->job->selection_pattern) {
615 Jmsg(jcr, M_FATAL, 0, _("No Migration SQL selection pattern specified.\n"));
618 Dmsg1(dbglevel, "SQL=%s\n", jcr->job->selection_pattern);
619 if (!db_sql_query(jcr->db, jcr->job->selection_pattern,
620 unique_dbid_handler, (void *)&ids)) {
621 Jmsg(jcr, M_FATAL, 0,
622 _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
626 case MT_SMALLEST_VOL:
627 if (!find_mediaid_then_jobids(jcr, &ids, sql_smallest_vol, "Smallest Volume")) {
632 if (!find_mediaid_then_jobids(jcr, &ids, sql_oldest_vol, "Oldest Volume")) {
637 case MT_POOL_OCCUPANCY:
639 /* Find count of bytes in pool */
640 Mmsg(query, sql_pool_bytes, jcr->pool->hdr.name);
641 if (!db_sql_query(jcr->db, query.c_str(), db_int64_handler, (void *)&ctx)) {
642 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
645 if (ctx.count == 0) {
646 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
649 pool_bytes = ctx.value;
650 Dmsg2(dbglevel, "highbytes=%d pool=%d\n", (int)jcr->pool->MigrationHighBytes,
652 if (pool_bytes < (int64_t)jcr->pool->MigrationHighBytes) {
653 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
656 Dmsg0(dbglevel, "We should do Occupation migration.\n");
659 /* Find a list of MediaIds that could be migrated */
660 Mmsg(query, sql_mediaids, jcr->pool->hdr.name);
661 Dmsg1(dbglevel, "query=%s\n", query.c_str());
662 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)&ids)) {
663 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
666 if (ids.count == 0) {
667 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
670 Dmsg2(dbglevel, "Pool Occupancy ids=%d MediaIds=%s\n", ids.count, ids.list);
673 * Now loop over MediaIds getting more JobIds to migrate until
674 * we reduce the pool occupancy below the low water mark.
677 for (int i=0; i < (int)ids.count; i++) {
678 stat = get_next_dbid_from_list(&p, &MediaId);
679 Dmsg2(dbglevel, "get_next_dbid stat=%d MediaId=%u\n", stat, MediaId);
681 Jmsg(jcr, M_FATAL, 0, _("Invalid MediaId found.\n"));
683 } else if (stat == 0) {
687 Mmsg(mid.list, "%s", edit_int64(MediaId, ed1));
688 if (!find_jobids_from_mediaid_list(jcr, &mid, "Volumes")) {
692 pm_strcat(jids.list, ",");
694 pm_strcat(jids.list, mid.list);
695 jids.count += mid.count;
697 /* Now get the count of bytes added */
699 /* Find count of bytes from Jobs */
700 Mmsg(query, sql_job_bytes, mid.list);
701 if (!db_sql_query(jcr->db, query.c_str(), db_int64_handler, (void *)&ctx)) {
702 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
705 pool_bytes -= ctx.value;
706 Dmsg1(dbglevel, "Job bytes=%d\n", (int)ctx.value);
707 Dmsg2(dbglevel, "lowbytes=%d pool=%d\n", (int)jcr->pool->MigrationLowBytes,
709 if (pool_bytes <= (int64_t)jcr->pool->MigrationLowBytes) {
710 Dmsg0(dbglevel, "We should be done.\n");
715 Dmsg2(dbglevel, "Pool Occupancy ids=%d JobIds=%s\n", jids.count, jids.list);
720 ttime = time(NULL) - (time_t)jcr->pool->MigrationTime;
721 (void)localtime_r(&ttime, &tm);
722 strftime(dt, sizeof(dt), "%Y-%m-%d %H:%M:%S", &tm);
725 Mmsg(query, sql_pool_time, jcr->pool->hdr.name, dt);
726 Dmsg1(dbglevel, "query=%s\n", query.c_str());
727 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)&ids)) {
728 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
731 if (ids.count == 0) {
732 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
735 Dmsg2(dbglevel, "PoolTime ids=%d JobIds=%s\n", ids.count, ids.list);
739 Jmsg(jcr, M_FATAL, 0, _("Unknown Migration Selection Type.\n"));
745 * Loop over all jobids except the last one, sending
746 * them to start_migration_job(), which will start a job
747 * for each of them. For the last JobId, we handle it below.
750 if (ids.count == 0) {
751 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
754 Jmsg(jcr, M_INFO, 0, _("The following %u JobId%s will be migrated: %s\n"),
755 ids.count, ids.count==0?"":"s", ids.list);
756 Dmsg2(dbglevel, "Before loop count=%d ids=%s\n", ids.count, ids.list);
757 for (int i=1; i < (int)ids.count; i++) {
759 stat = get_next_jobid_from_list(&p, &JobId);
760 Dmsg3(dbglevel, "get_jobid_no=%d stat=%d JobId=%u\n", i, stat, JobId);
761 jcr->MigrateJobId = JobId;
762 start_migration_job(jcr);
763 Dmsg0(dbglevel, "Back from start_migration_job\n");
765 Jmsg(jcr, M_FATAL, 0, _("Invalid JobId found.\n"));
767 } else if (stat == 0) {
768 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
773 /* Now get the last JobId and handle it in the current job */
775 stat = get_next_jobid_from_list(&p, &JobId);
776 Dmsg2(dbglevel, "Last get_next_jobid stat=%d JobId=%u\n", stat, (int)JobId);
778 Jmsg(jcr, M_FATAL, 0, _("Invalid JobId found.\n"));
780 } else if (stat == 0) {
781 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
785 jcr->previous_jr.JobId = JobId;
786 Dmsg1(dbglevel, "Previous jobid=%d\n", (int)jcr->previous_jr.JobId);
788 if (!db_get_job_record(jcr, jcr->db, &jcr->previous_jr)) {
789 Jmsg(jcr, M_FATAL, 0, _("Could not get job record for JobId %s to migrate. ERR=%s"),
790 edit_int64(jcr->previous_jr.JobId, ed1),
791 db_strerror(jcr->db));
794 Jmsg(jcr, M_INFO, 0, _("Migration using JobId=%s Job=%s\n"),
795 edit_int64(jcr->previous_jr.JobId, ed1), jcr->previous_jr.Job);
796 Dmsg3(dbglevel, "Migration JobId=%d using JobId=%s Job=%s\n",
798 edit_int64(jcr->previous_jr.JobId, ed1), jcr->previous_jr.Job);
808 free_pool_memory(ids.list);
809 free_pool_memory(mid.list);
810 free_pool_memory(jids.list);
814 static void start_migration_job(JCR *jcr)
816 UAContext *ua = new_ua_context(jcr);
819 Mmsg(ua->cmd, "run %s jobid=%s", jcr->job->hdr.name,
820 edit_uint64(jcr->MigrateJobId, ed1));
821 Dmsg1(dbglevel, "=============== Migration cmd=%s\n", ua->cmd);
822 parse_ua_args(ua); /* parse command */
823 int stat = run_cmd(ua, ua->cmd);
825 Jmsg(jcr, M_ERROR, 0, _("Could not start migration job.\n"));
827 Jmsg(jcr, M_INFO, 0, _("Migration JobId %d started.\n"), stat);
832 static bool find_mediaid_then_jobids(JCR *jcr, idpkt *ids, const char *query1,
836 POOL_MEM query(PM_MESSAGE);
839 /* Basic query for MediaId */
840 Mmsg(query, query1, jcr->pool->hdr.name);
841 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
842 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
845 if (ids->count == 0) {
846 Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type);
848 if (ids->count != 1) {
849 Jmsg(jcr, M_FATAL, 0, _("SQL logic error. Count should be 1 but is %d\n"),
853 Dmsg1(dbglevel, "Smallest Vol Jobids=%s\n", ids->list);
855 ok = find_jobids_from_mediaid_list(jcr, ids, type);
861 static bool find_jobids_from_mediaid_list(JCR *jcr, idpkt *ids, const char *type)
864 POOL_MEM query(PM_MESSAGE);
866 Mmsg(query, sql_jobids_from_mediaid, ids->list);
868 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
869 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
872 if (ids->count == 0) {
873 Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type);
880 static bool regex_find_jobids(JCR *jcr, idpkt *ids, const char *query1,
881 const char *query2, const char *type)
885 uitem *last_item = NULL;
890 POOL_MEM query(PM_MESSAGE);
892 item_chain = New(dlist(item, &item->link));
893 if (!jcr->job->selection_pattern) {
894 Jmsg(jcr, M_FATAL, 0, _("No Migration %s selection pattern specified.\n"),
898 Dmsg1(dbglevel, "regex=%s\n", jcr->job->selection_pattern);
899 /* Compile regex expression */
900 rc = regcomp(&preg, jcr->job->selection_pattern, REG_EXTENDED);
902 regerror(rc, &preg, prbuf, sizeof(prbuf));
903 Jmsg(jcr, M_FATAL, 0, _("Could not compile regex pattern \"%s\" ERR=%s\n"),
904 jcr->job->selection_pattern, prbuf);
907 /* Basic query for names */
908 Mmsg(query, query1, jcr->pool->hdr.name);
909 Dmsg1(dbglevel, "get name query1=%s\n", query.c_str());
910 if (!db_sql_query(jcr->db, query.c_str(), unique_name_handler,
911 (void *)item_chain)) {
912 Jmsg(jcr, M_FATAL, 0,
913 _("SQL to get %s failed. ERR=%s\n"), type, db_strerror(jcr->db));
916 /* Now apply the regex to the names and remove any item not matched */
917 foreach_dlist(item, item_chain) {
918 const int nmatch = 30;
919 regmatch_t pmatch[nmatch];
921 Dmsg1(dbglevel, "Remove item %s\n", last_item->item);
922 free(last_item->item);
923 item_chain->remove(last_item);
925 Dmsg1(dbglevel, "get name Item=%s\n", item->item);
926 rc = regexec(&preg, item->item, nmatch, pmatch, 0);
928 last_item = NULL; /* keep this one */
934 free(last_item->item);
935 Dmsg1(dbglevel, "Remove item %s\n", last_item->item);
936 item_chain->remove(last_item);
940 * At this point, we have a list of items in item_chain
941 * that have been matched by the regex, so now we need
942 * to look up their jobids.
945 foreach_dlist(item, item_chain) {
946 Dmsg2(dbglevel, "Got %s: %s\n", type, item->item);
947 Mmsg(query, query2, item->item, jcr->pool->hdr.name);
948 Dmsg1(dbglevel, "get id from name query2=%s\n", query.c_str());
949 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
950 Jmsg(jcr, M_FATAL, 0,
951 _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
955 if (ids->count == 0) {
956 Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type);
960 Dmsg2(dbglevel, "Count=%d Jobids=%s\n", ids->count, ids->list);
962 Dmsg0(dbglevel, "After delete item_chain\n");
968 * Release resources allocated during backup.
970 void migration_cleanup(JCR *jcr, int TermCode)
972 char sdt[MAX_TIME_LENGTH], edt[MAX_TIME_LENGTH];
973 char ec1[30], ec2[30], ec3[30], ec4[30], ec5[30], elapsed[50];
974 char ec6[50], ec7[50], ec8[50];
975 char term_code[100], sd_term_msg[100];
976 const char *term_msg;
977 int msg_type = M_INFO;
981 JCR *mig_jcr = jcr->mig_jcr;
982 POOL_MEM query(PM_MESSAGE);
984 Dmsg2(100, "Enter migrate_cleanup %d %c\n", TermCode, TermCode);
985 dequeue_messages(jcr); /* display any queued messages */
986 memset(&mr, 0, sizeof(mr));
987 set_jcr_job_status(jcr, TermCode);
988 update_job_end_record(jcr); /* update database */
991 * Check if we actually did something.
992 * mig_jcr is jcr of the newly migrated job.
995 mig_jcr->JobFiles = jcr->JobFiles = jcr->SDJobFiles;
996 mig_jcr->JobBytes = jcr->JobBytes = jcr->SDJobBytes;
997 mig_jcr->VolSessionId = jcr->VolSessionId;
998 mig_jcr->VolSessionTime = jcr->VolSessionTime;
999 mig_jcr->jr.RealEndTime = 0;
1000 mig_jcr->jr.PriorJobId = jcr->previous_jr.JobId;
1002 set_jcr_job_status(mig_jcr, TermCode);
1003 update_job_end_record(mig_jcr);
1005 /* Update final items to set them to the previous job's values */
1006 Mmsg(query, "UPDATE Job SET StartTime='%s',EndTime='%s',"
1007 "JobTDate=%s WHERE JobId=%s",
1008 jcr->previous_jr.cStartTime, jcr->previous_jr.cEndTime,
1009 edit_uint64(jcr->previous_jr.JobTDate, ec1),
1010 edit_uint64(mig_jcr->jr.JobId, ec2));
1011 db_sql_query(mig_jcr->db, query.c_str(), NULL, NULL);
1013 /* Now mark the previous job as migrated if it terminated normally */
1014 if (jcr->JobStatus == JS_Terminated) {
1015 Mmsg(query, "UPDATE Job SET Type='%c' WHERE JobId=%s",
1016 (char)JT_MIGRATED_JOB, edit_uint64(jcr->previous_jr.JobId, ec1));
1017 db_sql_query(mig_jcr->db, query.c_str(), NULL, NULL);
1020 if (!db_get_job_record(jcr, jcr->db, &jcr->jr)) {
1021 Jmsg(jcr, M_WARNING, 0, _("Error getting job record for stats: %s"),
1022 db_strerror(jcr->db));
1023 set_jcr_job_status(jcr, JS_ErrorTerminated);
1026 bstrncpy(mr.VolumeName, jcr->VolumeName, sizeof(mr.VolumeName));
1027 if (!db_get_media_record(jcr, jcr->db, &mr)) {
1028 Jmsg(jcr, M_WARNING, 0, _("Error getting Media record for Volume \"%s\": ERR=%s"),
1029 mr.VolumeName, db_strerror(jcr->db));
1030 set_jcr_job_status(jcr, JS_ErrorTerminated);
1033 update_bootstrap_file(mig_jcr);
1035 if (!db_get_job_volume_names(mig_jcr, mig_jcr->db, mig_jcr->jr.JobId, &mig_jcr->VolumeName)) {
1037 * Note, if the job has failed, most likely it did not write any
1038 * tape, so suppress this "error" message since in that case
1039 * it is normal. Or look at it the other way, only for a
1040 * normal exit should we complain about this error.
1042 if (jcr->JobStatus == JS_Terminated && jcr->jr.JobBytes) {
1043 Jmsg(jcr, M_ERROR, 0, "%s", db_strerror(mig_jcr->db));
1045 mig_jcr->VolumeName[0] = 0; /* none */
1047 switch (jcr->JobStatus) {
1049 if (jcr->Errors || jcr->SDErrors) {
1050 term_msg = _("%s OK -- with warnings");
1052 term_msg = _("%s OK");
1056 case JS_ErrorTerminated:
1057 term_msg = _("*** %s Error ***");
1058 msg_type = M_ERROR; /* Generate error message */
1059 if (jcr->store_bsock) {
1060 bnet_sig(jcr->store_bsock, BNET_TERMINATE);
1061 if (jcr->SD_msg_chan) {
1062 pthread_cancel(jcr->SD_msg_chan);
1067 term_msg = _("%s Canceled");
1068 if (jcr->store_bsock) {
1069 bnet_sig(jcr->store_bsock, BNET_TERMINATE);
1070 if (jcr->SD_msg_chan) {
1071 pthread_cancel(jcr->SD_msg_chan);
1076 term_msg = _("Inappropriate %s term code");
1080 term_msg = _("%s -- no files to migrate");
1083 bsnprintf(term_code, sizeof(term_code), term_msg, "Migration");
1084 bstrftimes(sdt, sizeof(sdt), jcr->jr.StartTime);
1085 bstrftimes(edt, sizeof(edt), jcr->jr.EndTime);
1086 RunTime = jcr->jr.EndTime - jcr->jr.StartTime;
1090 kbps = (double)jcr->SDJobBytes / (1000 * RunTime);
1094 jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg));
1096 Jmsg(jcr, msg_type, 0, _("Bacula %s (%s): %s\n"
1097 " Prev Backup JobId: %s\n"
1098 " New Backup JobId: %s\n"
1099 " Migration JobId: %s\n"
1100 " Migration Job: %s\n"
1101 " Backup Level: %s%s\n"
1103 " FileSet: \"%s\" %s\n"
1104 " Pool: \"%s\" (From %s)\n"
1105 " Read Storage: \"%s\" (From %s)\n"
1106 " Write Storage: \"%s\" (From %s)\n"
1109 " Elapsed time: %s\n"
1111 " SD Files Written: %s\n"
1112 " SD Bytes Written: %s (%sB)\n"
1113 " Rate: %.1f KB/s\n"
1114 " Volume name(s): %s\n"
1115 " Volume Session Id: %d\n"
1116 " Volume Session Time: %d\n"
1117 " Last Volume Bytes: %s (%sB)\n"
1119 " SD termination status: %s\n"
1120 " Termination: %s\n\n"),
1124 edit_uint64(jcr->previous_jr.JobId, ec6),
1125 mig_jcr ? edit_uint64(mig_jcr->jr.JobId, ec7) : "0",
1126 edit_uint64(jcr->jr.JobId, ec8),
1128 level_to_str(jcr->JobLevel), jcr->since,
1129 jcr->client->name(),
1130 jcr->fileset->name(), jcr->FSCreateTime,
1131 jcr->pool->name(), jcr->pool_source,
1132 jcr->rstore?jcr->rstore->name():"*None*",
1133 NPRT(jcr->rstore_source),
1134 jcr->wstore?jcr->wstore->name():"*None*",
1135 NPRT(jcr->wstore_source),
1138 edit_utime(RunTime, elapsed, sizeof(elapsed)),
1140 edit_uint64_with_commas(jcr->SDJobFiles, ec1),
1141 edit_uint64_with_commas(jcr->SDJobBytes, ec2),
1142 edit_uint64_with_suffix(jcr->SDJobBytes, ec3),
1144 mig_jcr ? mig_jcr->VolumeName : "",
1146 jcr->VolSessionTime,
1147 edit_uint64_with_commas(mr.VolBytes, ec4),
1148 edit_uint64_with_suffix(mr.VolBytes, ec5),
1153 Dmsg1(100, "migrate_cleanup() mig_jcr=0x%x\n", jcr->mig_jcr);
1155 free_jcr(jcr->mig_jcr);
1156 jcr->mig_jcr = NULL;
1158 Dmsg0(100, "Leave migrate_cleanup()\n");
1162 * Return next DBId from comma separated list
1165 * 1 if next DBId returned
1166 * 0 if no more DBIds are in list
1167 * -1 there is an error
1169 static int get_next_dbid_from_list(char **p, DBId_t *DBId)
1175 for (int i=0; i<(int)sizeof(id); i++) {
1178 } else if (*q == ',') {
1187 } else if (!is_a_number(id)) {
1188 return -1; /* error */
1191 *DBId = str_to_int64(id);