3 * Bacula Director -- migrate.c -- responsible for doing
6 * Kern Sibbald, September MMIV
8 * Basic tasks done here:
9 * Open DB and create records for this job.
10 * Open Message Channel with Storage daemon to tell him a job will be starting.
11 * Open connection with Storage daemon and pass him commands
13 * When the Storage daemon finishes the job, update the DB.
18 Bacula® - The Network Backup Solution
20 Copyright (C) 2004-2006 Free Software Foundation Europe e.V.
22 The main author of Bacula is Kern Sibbald, with contributions from
23 many others, a complete list can be found in the file AUTHORS.
24 This program is Free Software; you can redistribute it and/or
25 modify it under the terms of version two of the GNU General Public
26 License as published by the Free Software Foundation plus additions
27 that are listed in the file LICENSE.
29 This program is distributed in the hope that it will be useful, but
30 WITHOUT ANY WARRANTY; without even the implied warranty of
31 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
32 General Public License for more details.
34 You should have received a copy of the GNU General Public License
35 along with this program; if not, write to the Free Software
36 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
39 Bacula® is a registered trademark of John Walker.
40 The licensor of Bacula is the Free Software Foundation Europe
41 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
42 Switzerland, email:ftf@fsfeurope.org.
49 #include "lib/bregex.h"
54 static const int dbglevel = 10;
56 static char OKbootstrap[] = "3000 OK bootstrap\n";
57 static bool get_job_to_migrate(JCR *jcr);
59 static bool regex_find_jobids(JCR *jcr, idpkt *ids, const char *query1,
60 const char *query2, const char *type);
61 static bool find_mediaid_then_jobids(JCR *jcr, idpkt *ids, const char *query1,
63 static bool find_jobids_from_mediaid_list(JCR *jcr, idpkt *ids, const char *type);
64 static void start_migration_job(JCR *jcr);
65 static int get_next_dbid_from_list(char **p, DBId_t *DBId);
68 * Called here before the job is run to do the job
69 * specific setup. Note, one of the important things to
70 * complete in this init code is to make the definitive
71 * choice of input and output storage devices. This is
72 * because immediately after the init, the job is queued
73 * in the jobq.c code, and it checks that all the resources
74 * (storage resources in particular) are available, so these
75 * must all be properly defined.
77 * previous_jr refers to the job DB record of the Job that is
78 * going to be migrated.
79 * prev_job refers to the job resource of the Job that is
80 * going to be migrated.
81 * jcr is the jcr for the current "migration" job. It is a
82 * control job that is put in the DB as a migration job, which
83 * means that this job migrated a previous job to a new job.
84 * No Volume or File data is associated with this control
86 * mig_jcr refers to the newly migrated job that is run by
87 * the current jcr. It is a backup job that moves (migrates) the
88 * data written for the previous_jr into the new pool. This
89 * job (mig_jcr) becomes the new backup job that replaces
90 * the original backup job.
92 bool do_migration_init(JCR *jcr)
98 JCR *mig_jcr; /* newly migrated job */
100 /* If we find a job or jobs to migrate it is previous_jr.JobId */
101 if (!get_job_to_migrate(jcr)) {
104 Dmsg1(dbglevel, "Back from get_job_to_migrate JobId=%d\n", (int)jcr->JobId);
106 if (jcr->previous_jr.JobId == 0) {
107 Dmsg1(dbglevel, "JobId=%d no previous JobId\n", (int)jcr->JobId);
108 Jmsg(jcr, M_INFO, 0, _("No previous Job found to migrate.\n"));
109 return true; /* no work */
112 if (!get_or_create_fileset_record(jcr)) {
113 Dmsg1(dbglevel, "JobId=%d no FileSet\n", (int)jcr->JobId);
114 Jmsg(jcr, M_FATAL, 0, _("Could not get or create the FileSet record.\n"));
118 apply_pool_overrides(jcr);
120 jcr->jr.PoolId = get_or_create_pool_record(jcr, jcr->pool->hdr.name);
121 if (jcr->jr.PoolId == 0) {
122 Dmsg1(dbglevel, "JobId=%d no PoolId\n", (int)jcr->JobId);
123 Jmsg(jcr, M_FATAL, 0, _("Could not get or create a Pool record.\n"));
127 create_restore_bootstrap_file(jcr);
129 if (jcr->previous_jr.JobId == 0 || jcr->ExpectedFiles == 0) {
130 set_jcr_job_status(jcr, JS_Terminated);
131 Dmsg1(dbglevel, "JobId=%d expected files == 0\n", (int)jcr->JobId);
132 if (jcr->previous_jr.JobId == 0) {
133 Jmsg(jcr, M_INFO, 0, _("No previous Job found to migrate.\n"));
135 Jmsg(jcr, M_INFO, 0, _("Previous Job has no data to migrate.\n"));
137 return true; /* no work */
140 Dmsg5(dbglevel, "JobId=%d: Previous: Name=%s JobId=%d Type=%c Level=%c\n",
142 jcr->previous_jr.Name, (int)jcr->previous_jr.JobId,
143 jcr->previous_jr.JobType, jcr->previous_jr.JobLevel);
145 Dmsg5(dbglevel, "JobId=%d: Current: Name=%s JobId=%d Type=%c Level=%c\n",
147 jcr->jr.Name, (int)jcr->jr.JobId,
148 jcr->jr.JobType, jcr->jr.JobLevel);
151 job = (JOB *)GetResWithName(R_JOB, jcr->jr.Name);
152 prev_job = (JOB *)GetResWithName(R_JOB, jcr->previous_jr.Name);
155 Jmsg(jcr, M_FATAL, 0, _("Job resource not found for \"%s\".\n"), jcr->jr.Name);
159 Jmsg(jcr, M_FATAL, 0, _("Previous Job resource not found for \"%s\".\n"),
160 jcr->previous_jr.Name);
164 /* Create a migation jcr */
165 mig_jcr = jcr->mig_jcr = new_jcr(sizeof(JCR), dird_free_jcr);
166 memcpy(&mig_jcr->previous_jr, &jcr->previous_jr, sizeof(mig_jcr->previous_jr));
169 * Turn the mig_jcr into a "real" job that takes on the aspects of
170 * the previous backup job "prev_job".
172 set_jcr_defaults(mig_jcr, prev_job);
173 if (!setup_job(mig_jcr)) {
174 Jmsg(jcr, M_FATAL, 0, _("setup job failed.\n"));
178 /* Now reset the job record from the previous job */
179 memcpy(&mig_jcr->jr, &jcr->previous_jr, sizeof(mig_jcr->jr));
180 /* Update the jr to reflect the new values of PoolId, FileSetId, and JobId. */
181 mig_jcr->jr.PoolId = jcr->jr.PoolId;
182 mig_jcr->jr.FileSetId = jcr->jr.FileSetId;
183 mig_jcr->jr.JobId = mig_jcr->JobId;
185 Dmsg4(dbglevel, "mig_jcr: Name=%s JobId=%d Type=%c Level=%c\n",
186 mig_jcr->jr.Name, (int)mig_jcr->jr.JobId,
187 mig_jcr->jr.JobType, mig_jcr->jr.JobLevel);
190 * Get the PoolId used with the original job. Then
191 * find the pool name from the database record.
193 memset(&pr, 0, sizeof(pr));
194 pr.PoolId = mig_jcr->previous_jr.PoolId;
195 if (!db_get_pool_record(jcr, jcr->db, &pr)) {
196 Jmsg(jcr, M_FATAL, 0, _("Pool for JobId %s not in database. ERR=%s\n"),
197 edit_int64(pr.PoolId, ed1), db_strerror(jcr->db));
200 /* Get the pool resource corresponding to the original job */
201 pool = (POOL *)GetResWithName(R_POOL, pr.Name);
203 Jmsg(jcr, M_FATAL, 0, _("Pool resource \"%s\" not found.\n"), pr.Name);
207 /* If pool storage specified, use it for restore */
208 copy_rstorage(mig_jcr, pool->storage, _("Pool resource"));
209 copy_rstorage(jcr, pool->storage, _("Pool resource"));
212 * If the original backup pool has a NextPool, make sure a
213 * record exists in the database. Note, in this case, we
214 * will be migrating from pool to pool->NextPool.
216 if (pool->NextPool) {
217 jcr->jr.PoolId = get_or_create_pool_record(jcr, pool->NextPool->hdr.name);
218 if (jcr->jr.PoolId == 0) {
222 * put the "NextPool" resource pointer in our jcr so that we
223 * can pull the Storage reference from it.
225 mig_jcr->pool = jcr->pool = pool->NextPool;
226 mig_jcr->jr.PoolId = jcr->jr.PoolId;
227 pm_strcpy(jcr->pool_source, _("NextPool in Pool resource"));
229 Jmsg(jcr, M_FATAL, 0, _("No Next Pool specification found in Pool \"%s\".\n"),
234 if (!jcr->pool->storage || jcr->pool->storage->size() == 0) {
235 Jmsg(jcr, M_FATAL, 0, _("No Storage specification found in Next Pool \"%s\".\n"),
236 jcr->pool->hdr.name);
240 /* If pool storage specified, use it instead of job storage for backup */
241 copy_wstorage(jcr, jcr->pool->storage, _("NextPool in Pool resource"));
247 * Do a Migration of a previous job
249 * Returns: false on failure
252 bool do_migration(JCR *jcr)
256 JCR *mig_jcr = jcr->mig_jcr; /* newly migrated job */
262 /* Print Job Start message */
263 Jmsg(jcr, M_INFO, 0, _("Start Migration JobId %s, Job=%s\n"),
264 edit_uint64(jcr->JobId, ed1), jcr->Job);
266 set_jcr_job_status(jcr, JS_Running);
267 set_jcr_job_status(mig_jcr, JS_Running);
268 Dmsg2(dbglevel, "JobId=%d JobLevel=%c\n", (int)jcr->jr.JobId, jcr->jr.JobLevel);
270 /* Update job start record for this migration control job */
271 if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) {
272 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db));
276 Dmsg4(dbglevel, "mig_jcr: Name=%s JobId=%d Type=%c Level=%c\n",
277 mig_jcr->jr.Name, (int)mig_jcr->jr.JobId,
278 mig_jcr->jr.JobType, mig_jcr->jr.JobLevel);
280 /* Update job start record for the real migration backup job */
281 if (!db_update_job_start_record(mig_jcr, mig_jcr->db, &mig_jcr->jr)) {
282 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(mig_jcr->db));
288 * Open a message channel connection with the Storage
289 * daemon. This is to let him know that our client
290 * will be contacting him for a backup session.
293 Dmsg0(110, "Open connection with storage daemon\n");
294 set_jcr_job_status(jcr, JS_WaitSD);
295 set_jcr_job_status(mig_jcr, JS_WaitSD);
297 * Start conversation with Storage daemon
299 if (!connect_to_storage_daemon(jcr, 10, SDConnectTimeout, 1)) {
302 sd = jcr->store_bsock;
304 * Now start a job with the Storage daemon
306 Dmsg2(dbglevel, "Read store=%s, write store=%s\n",
307 ((STORE *)jcr->rstorage->first())->name(),
308 ((STORE *)jcr->wstorage->first())->name());
309 if (((STORE *)jcr->rstorage->first())->name() == ((STORE *)jcr->wstorage->first())->name()) {
310 Jmsg(jcr, M_FATAL, 0, _("Read storage \"%s\" same as write storage.\n"),
311 ((STORE *)jcr->rstorage->first())->name());
314 if (!start_storage_daemon_job(jcr, jcr->rstorage, jcr->wstorage)) {
317 Dmsg0(150, "Storage daemon connection OK\n");
319 if (!send_bootstrap_file(jcr, sd) ||
320 !response(jcr, sd, OKbootstrap, "Bootstrap", DISPLAY_ERROR)) {
324 if (!bnet_fsend(sd, "run")) {
329 * Now start a Storage daemon message thread
331 if (!start_storage_daemon_message_thread(jcr)) {
336 set_jcr_job_status(jcr, JS_Running);
337 set_jcr_job_status(mig_jcr, JS_Running);
339 /* Pickup Job termination data */
340 /* Note, the SD stores in jcr->JobFiles/ReadBytes/JobBytes/Errors */
341 wait_for_storage_daemon_termination(jcr);
343 set_jcr_job_status(jcr, jcr->SDJobStatus);
344 if (jcr->JobStatus != JS_Terminated) {
347 migration_cleanup(jcr, jcr->JobStatus);
349 UAContext *ua = new_ua_context(jcr);
350 purge_files_from_job(ua, jcr->previous_jr.JobId);
361 /* Add an item to the list if it is unique */
362 static void add_unique_id(idpkt *ids, char *item)
367 /* Walk through current list to see if each item is the same as item */
370 for (int i=0; i<(int)sizeof(id); i++) {
373 } else if (*q == ',') {
380 if (strcmp(item, id) == 0) {
384 /* Did not find item, so add it to list */
385 if (ids->count == 0) {
388 pm_strcat(ids->list, ",");
390 pm_strcat(ids->list, item);
392 // Dmsg3(0, "add_uniq count=%d Ids=%p %s\n", ids->count, ids->list, ids->list);
397 * Callback handler make list of DB Ids
399 static int unique_dbid_handler(void *ctx, int num_fields, char **row)
401 idpkt *ids = (idpkt *)ctx;
403 add_unique_id(ids, row[0]);
404 Dmsg3(dbglevel, "dbid_hdlr count=%d Ids=%p %s\n", ids->count, ids->list, ids->list);
414 static int item_compare(void *item1, void *item2)
416 uitem *i1 = (uitem *)item1;
417 uitem *i2 = (uitem *)item2;
418 return strcmp(i1->item, i2->item);
421 static int unique_name_handler(void *ctx, int num_fields, char **row)
423 dlist *list = (dlist *)ctx;
425 uitem *new_item = (uitem *)malloc(sizeof(uitem));
428 memset(new_item, 0, sizeof(uitem));
429 new_item->item = bstrdup(row[0]);
430 Dmsg1(dbglevel, "Unique_name_hdlr Item=%s\n", row[0]);
431 item = (uitem *)list->binary_insert((void *)new_item, item_compare);
432 if (item != new_item) { /* already in list */
433 free(new_item->item);
434 free((char *)new_item);
440 /* Get Job names in Pool */
441 const char *sql_job =
442 "SELECT DISTINCT Job.Name from Job,Pool"
443 " WHERE Pool.Name='%s' AND Job.PoolId=Pool.PoolId";
445 /* Get JobIds from regex'ed Job names */
446 const char *sql_jobids_from_job =
447 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Job,Pool"
448 " WHERE Job.Name='%s' AND Pool.Name='%s' AND Job.PoolId=Pool.PoolId"
449 " ORDER by Job.StartTime";
451 /* Get Client names in Pool */
452 const char *sql_client =
453 "SELECT DISTINCT Client.Name from Client,Pool,Job"
454 " WHERE Pool.Name='%s' AND Job.ClientId=Client.ClientId AND"
455 " Job.PoolId=Pool.PoolId";
457 /* Get JobIds from regex'ed Client names */
458 const char *sql_jobids_from_client =
459 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Job,Pool,Client"
460 " WHERE Client.Name='%s' AND Pool.Name='%s' AND Job.PoolId=Pool.PoolId"
461 " AND Job.ClientId=Client.ClientId "
462 " ORDER by Job.StartTime";
464 /* Get Volume names in Pool */
465 const char *sql_vol =
466 "SELECT DISTINCT VolumeName FROM Media,Pool WHERE"
467 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
468 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'";
470 /* Get JobIds from regex'ed Volume names */
471 const char *sql_jobids_from_vol =
472 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Media,JobMedia,Job"
473 " WHERE Media.VolumeName='%s' AND Media.MediaId=JobMedia.MediaId"
474 " AND JobMedia.JobId=Job.JobId"
475 " ORDER by Job.StartTime";
478 const char *sql_smallest_vol =
479 "SELECT MediaId FROM Media,Pool WHERE"
480 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
481 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'"
482 " ORDER BY VolBytes ASC LIMIT 1";
484 const char *sql_oldest_vol =
485 "SELECT MediaId FROM Media,Pool WHERE"
486 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
487 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'"
488 " ORDER BY LastWritten ASC LIMIT 1";
490 /* Get JobIds when we have selected MediaId */
491 const char *sql_jobids_from_mediaid =
492 "SELECT DISTINCT Job.JobId,Job.StartTime FROM JobMedia,Job"
493 " WHERE JobMedia.JobId=Job.JobId AND JobMedia.MediaId=%s"
494 " ORDER by Job.StartTime";
496 /* Get tne number of bytes in the pool */
497 const char *sql_pool_bytes =
498 "SELECT SUM(VolBytes) FROM Media,Pool WHERE"
499 " VolStatus in ('Full','Used','Error','Append') AND Media.Enabled=1 AND"
500 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'";
502 /* Get tne number of bytes in the Jobs */
503 const char *sql_job_bytes =
504 "SELECT SUM(JobBytes) FROM Job WHERE JobId IN (%s)";
507 /* Get Media Ids in Pool */
508 const char *sql_mediaids =
509 "SELECT MediaId FROM Media,Pool WHERE"
510 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
511 " Media.PoolId=Pool.PoolId AND Pool.Name='%s' ORDER BY LastWritten ASC";
513 /* Get JobIds in Pool longer than specified time */
514 const char *sql_pool_time =
515 "SELECT DISTINCT Job.JobId from Pool,Job,Media,JobMedia WHERE"
516 " Pool.Name='%s' AND Media.PoolId=Pool.PoolId AND"
517 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
518 " JobMedia.JobId=Job.JobId AND Job.PoolId=Media.PoolId"
519 " AND Job.RealEndTime<='%s'";
522 * const char *sql_ujobid =
523 * "SELECT DISTINCT Job.Job from Client,Pool,Media,Job,JobMedia "
524 * " WHERE Media.PoolId=Pool.PoolId AND Pool.Name='%s' AND"
525 * " JobMedia.JobId=Job.JobId AND Job.PoolId=Media.PoolId";
532 * This is the central piece of code that finds a job or jobs
533 * actually JobIds to migrate. It first looks to see if one
534 * has been "manually" specified in jcr->MigrateJobId, and if
535 * so, it returns that JobId to be run. Otherwise, it
536 * examines the Selection Type to see what kind of migration
537 * we are doing (Volume, Job, Client, ...) and applies any
538 * Selection Pattern if appropriate to obtain a list of JobIds.
539 * Finally, it will loop over all the JobIds found, except the last
540 * one starting a new job with MigrationJobId set to that JobId, and
541 * finally, it returns the last JobId to the caller.
543 * Returns: false on error
544 * true if OK and jcr->previous_jr filled in
546 static bool get_job_to_migrate(JCR *jcr)
549 POOL_MEM query(PM_MESSAGE);
554 idpkt ids, mid, jids;
560 char dt[MAX_TIME_LENGTH];
562 ids.list = get_pool_memory(PM_MESSAGE);
565 mid.list = get_pool_memory(PM_MESSAGE);
568 jids.list = get_pool_memory(PM_MESSAGE);
574 * If MigrateJobId is set, then we migrate only that Job,
575 * otherwise, we go through the full selection of jobs to
578 if (jcr->MigrateJobId != 0) {
579 Dmsg1(dbglevel, "At Job start previous jobid=%u\n", jcr->MigrateJobId);
580 edit_uint64(jcr->MigrateJobId, ids.list);
583 switch (jcr->job->selection_type) {
585 if (!regex_find_jobids(jcr, &ids, sql_job, sql_jobids_from_job, "Job")) {
590 if (!regex_find_jobids(jcr, &ids, sql_client, sql_jobids_from_client, "Client")) {
595 if (!regex_find_jobids(jcr, &ids, sql_vol, sql_jobids_from_vol, "Volume")) {
600 if (!jcr->job->selection_pattern) {
601 Jmsg(jcr, M_FATAL, 0, _("No Migration SQL selection pattern specified.\n"));
604 Dmsg1(dbglevel, "SQL=%s\n", jcr->job->selection_pattern);
605 if (!db_sql_query(jcr->db, jcr->job->selection_pattern,
606 unique_dbid_handler, (void *)&ids)) {
607 Jmsg(jcr, M_FATAL, 0,
608 _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
612 case MT_SMALLEST_VOL:
613 if (!find_mediaid_then_jobids(jcr, &ids, sql_smallest_vol, "Smallest Volume")) {
618 if (!find_mediaid_then_jobids(jcr, &ids, sql_oldest_vol, "Oldest Volume")) {
623 case MT_POOL_OCCUPANCY:
625 /* Find count of bytes in pool */
626 Mmsg(query, sql_pool_bytes, jcr->pool->hdr.name);
627 if (!db_sql_query(jcr->db, query.c_str(), db_int64_handler, (void *)&ctx)) {
628 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
631 if (ctx.count == 0) {
632 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
635 pool_bytes = ctx.value;
636 Dmsg2(dbglevel, "highbytes=%d pool=%d\n", (int)jcr->pool->MigrationHighBytes,
638 if (pool_bytes < (int64_t)jcr->pool->MigrationHighBytes) {
639 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
642 Dmsg0(dbglevel, "We should do Occupation migration.\n");
645 /* Find a list of MediaIds that could be migrated */
646 Mmsg(query, sql_mediaids, jcr->pool->hdr.name);
647 Dmsg1(dbglevel, "query=%s\n", query.c_str());
648 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)&ids)) {
649 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
652 if (ids.count == 0) {
653 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
656 Dmsg2(dbglevel, "Pool Occupancy ids=%d MediaIds=%s\n", ids.count, ids.list);
659 * Now loop over MediaIds getting more JobIds to migrate until
660 * we reduce the pool occupancy below the low water mark.
663 for (int i=0; i < (int)ids.count; i++) {
664 stat = get_next_dbid_from_list(&p, &MediaId);
665 Dmsg2(dbglevel, "get_next_dbid stat=%d MediaId=%u\n", stat, MediaId);
667 Jmsg(jcr, M_FATAL, 0, _("Invalid MediaId found.\n"));
669 } else if (stat == 0) {
673 Mmsg(mid.list, "%s", edit_int64(MediaId, ed1));
674 ok = find_jobids_from_mediaid_list(jcr, &mid, "Volumes");
679 pm_strcat(jids.list, ",");
681 pm_strcat(jids.list, mid.list);
682 jids.count += mid.count;
684 /* Now get the count of bytes added */
686 /* Find count of bytes from Jobs */
687 Mmsg(query, sql_job_bytes, mid.list);
688 if (!db_sql_query(jcr->db, query.c_str(), db_int64_handler, (void *)&ctx)) {
689 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
692 pool_bytes -= ctx.value;
693 Dmsg1(dbglevel, "Job bytes=%d\n", (int)ctx.value);
694 Dmsg2(dbglevel, "lowbytes=%d pool=%d\n", (int)jcr->pool->MigrationLowBytes,
696 if (pool_bytes <= (int64_t)jcr->pool->MigrationLowBytes) {
697 Dmsg0(dbglevel, "We should be done.\n");
702 Dmsg2(dbglevel, "Pool Occupancy ids=%d JobIds=%s\n", jids.count, jids.list);
707 ttime = time(NULL) - (time_t)jcr->pool->MigrationTime;
708 (void)localtime_r(&ttime, &tm);
709 strftime(dt, sizeof(dt), "%Y-%m-%d %H:%M:%S", &tm);
712 Mmsg(query, sql_pool_time, jcr->pool->hdr.name, dt);
713 Dmsg1(dbglevel, "query=%s\n", query.c_str());
714 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)&ids)) {
715 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
718 if (ids.count == 0) {
719 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
722 Dmsg2(dbglevel, "PoolTime ids=%d JobIds=%s\n", ids.count, ids.list);
726 Jmsg(jcr, M_FATAL, 0, _("Unknown Migration Selection Type.\n"));
732 * Loop over all jobids except the last one, sending
733 * them to start_migration_job(), which will start a job
734 * for each of them. For the last JobId, we handle it below.
737 if (ids.count == 0) {
738 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
741 Jmsg(jcr, M_INFO, 0, _("The following %u JobId%s will be migrated: %s\n"),
742 ids.count, ids.count==0?"":"s", ids.list);
743 Dmsg2(dbglevel, "Before loop count=%d ids=%s\n", ids.count, ids.list);
744 for (int i=1; i < (int)ids.count; i++) {
746 stat = get_next_jobid_from_list(&p, &JobId);
747 Dmsg3(dbglevel, "get_jobid_no=%d stat=%d JobId=%u\n", i, stat, JobId);
748 jcr->MigrateJobId = JobId;
749 start_migration_job(jcr);
750 Dmsg0(dbglevel, "Back from start_migration_job\n");
752 Jmsg(jcr, M_FATAL, 0, _("Invalid JobId found.\n"));
754 } else if (stat == 0) {
755 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
760 /* Now get the last JobId and handle it in the current job */
762 stat = get_next_jobid_from_list(&p, &JobId);
763 Dmsg2(dbglevel, "Last get_next_jobid stat=%d JobId=%u\n", stat, (int)JobId);
765 Jmsg(jcr, M_FATAL, 0, _("Invalid JobId found.\n"));
767 } else if (stat == 0) {
768 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
772 jcr->previous_jr.JobId = JobId;
773 Dmsg1(dbglevel, "Previous jobid=%d\n", (int)jcr->previous_jr.JobId);
775 if (!db_get_job_record(jcr, jcr->db, &jcr->previous_jr)) {
776 Jmsg(jcr, M_FATAL, 0, _("Could not get job record for JobId %s to migrate. ERR=%s"),
777 edit_int64(jcr->previous_jr.JobId, ed1),
778 db_strerror(jcr->db));
781 Jmsg(jcr, M_INFO, 0, _("Migration using JobId=%s Job=%s\n"),
782 edit_int64(jcr->previous_jr.JobId, ed1), jcr->previous_jr.Job);
783 Dmsg3(dbglevel, "Migration JobId=%d using JobId=%s Job=%s\n",
785 edit_int64(jcr->previous_jr.JobId, ed1), jcr->previous_jr.Job);
795 free_pool_memory(ids.list);
796 free_pool_memory(mid.list);
797 free_pool_memory(jids.list);
801 static void start_migration_job(JCR *jcr)
803 UAContext *ua = new_ua_context(jcr);
806 Mmsg(ua->cmd, "run %s jobid=%s", jcr->job->hdr.name,
807 edit_uint64(jcr->MigrateJobId, ed1));
808 Dmsg1(dbglevel, "=============== Migration cmd=%s\n", ua->cmd);
809 parse_ua_args(ua); /* parse command */
810 int stat = run_cmd(ua, ua->cmd);
812 Jmsg(jcr, M_ERROR, 0, _("Could not start migration job.\n"));
814 Jmsg(jcr, M_INFO, 0, _("Migration JobId %d started.\n"), stat);
819 static bool find_mediaid_then_jobids(JCR *jcr, idpkt *ids, const char *query1,
823 POOL_MEM query(PM_MESSAGE);
826 /* Basic query for MediaId */
827 Mmsg(query, query1, jcr->pool->hdr.name);
828 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
829 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
832 if (ids->count == 0) {
833 Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type);
835 if (ids->count != 1) {
836 Jmsg(jcr, M_FATAL, 0, _("SQL logic error. Count should be 1 but is %d\n"),
840 Dmsg1(dbglevel, "Smallest Vol Jobids=%s\n", ids->list);
842 ok = find_jobids_from_mediaid_list(jcr, ids, type);
848 static bool find_jobids_from_mediaid_list(JCR *jcr, idpkt *ids, const char *type)
851 POOL_MEM query(PM_MESSAGE);
853 Mmsg(query, sql_jobids_from_mediaid, ids->list);
855 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
856 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
859 if (ids->count == 0) {
860 Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type);
867 static bool regex_find_jobids(JCR *jcr, idpkt *ids, const char *query1,
868 const char *query2, const char *type)
872 uitem *last_item = NULL;
877 POOL_MEM query(PM_MESSAGE);
879 item_chain = New(dlist(item, &item->link));
880 if (!jcr->job->selection_pattern) {
881 Jmsg(jcr, M_FATAL, 0, _("No Migration %s selection pattern specified.\n"),
885 Dmsg1(dbglevel, "regex=%s\n", jcr->job->selection_pattern);
886 /* Compile regex expression */
887 rc = regcomp(&preg, jcr->job->selection_pattern, REG_EXTENDED);
889 regerror(rc, &preg, prbuf, sizeof(prbuf));
890 Jmsg(jcr, M_FATAL, 0, _("Could not compile regex pattern \"%s\" ERR=%s\n"),
891 jcr->job->selection_pattern, prbuf);
894 /* Basic query for names */
895 Mmsg(query, query1, jcr->pool->hdr.name);
896 Dmsg1(dbglevel, "get name query1=%s\n", query.c_str());
897 if (!db_sql_query(jcr->db, query.c_str(), unique_name_handler,
898 (void *)item_chain)) {
899 Jmsg(jcr, M_FATAL, 0,
900 _("SQL to get %s failed. ERR=%s\n"), type, db_strerror(jcr->db));
903 /* Now apply the regex to the names and remove any item not matched */
904 foreach_dlist(item, item_chain) {
905 const int nmatch = 30;
906 regmatch_t pmatch[nmatch];
908 Dmsg1(dbglevel, "Remove item %s\n", last_item->item);
909 free(last_item->item);
910 item_chain->remove(last_item);
912 Dmsg1(dbglevel, "get name Item=%s\n", item->item);
913 rc = regexec(&preg, item->item, nmatch, pmatch, 0);
915 last_item = NULL; /* keep this one */
921 free(last_item->item);
922 Dmsg1(dbglevel, "Remove item %s\n", last_item->item);
923 item_chain->remove(last_item);
927 * At this point, we have a list of items in item_chain
928 * that have been matched by the regex, so now we need
929 * to look up their jobids.
932 foreach_dlist(item, item_chain) {
933 Dmsg2(dbglevel, "Got %s: %s\n", type, item->item);
934 Mmsg(query, query2, item->item, jcr->pool->hdr.name);
935 Dmsg1(dbglevel, "get id from name query2=%s\n", query.c_str());
936 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
937 Jmsg(jcr, M_FATAL, 0,
938 _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
942 if (ids->count == 0) {
943 Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type);
947 Dmsg2(dbglevel, "Count=%d Jobids=%s\n", ids->count, ids->list);
949 Dmsg0(dbglevel, "After delete item_chain\n");
955 * Release resources allocated during backup.
957 void migration_cleanup(JCR *jcr, int TermCode)
959 char sdt[MAX_TIME_LENGTH], edt[MAX_TIME_LENGTH];
960 char ec1[30], ec2[30], ec3[30], ec4[30], ec5[30], elapsed[50];
961 char ec6[50], ec7[50], ec8[50];
962 char term_code[100], sd_term_msg[100];
963 const char *term_msg;
968 JCR *mig_jcr = jcr->mig_jcr;
969 POOL_MEM query(PM_MESSAGE);
971 Dmsg2(100, "Enter migrate_cleanup %d %c\n", TermCode, TermCode);
972 dequeue_messages(jcr); /* display any queued messages */
973 memset(&mr, 0, sizeof(mr));
974 set_jcr_job_status(jcr, TermCode);
975 update_job_end_record(jcr); /* update database */
978 * Check if we actually did something.
979 * mig_jcr is jcr of the newly migrated job.
982 mig_jcr->JobFiles = jcr->JobFiles = jcr->SDJobFiles;
983 mig_jcr->JobBytes = jcr->JobBytes = jcr->SDJobBytes;
984 mig_jcr->VolSessionId = jcr->VolSessionId;
985 mig_jcr->VolSessionTime = jcr->VolSessionTime;
986 mig_jcr->jr.RealEndTime = 0;
987 mig_jcr->jr.PriorJobId = jcr->previous_jr.JobId;
989 set_jcr_job_status(mig_jcr, TermCode);
992 update_job_end_record(mig_jcr);
994 /* Update final items to set them to the previous job's values */
995 Mmsg(query, "UPDATE Job SET StartTime='%s',EndTime='%s',"
996 "JobTDate=%s WHERE JobId=%s",
997 jcr->previous_jr.cStartTime, jcr->previous_jr.cEndTime,
998 edit_uint64(jcr->previous_jr.JobTDate, ec1),
999 edit_uint64(mig_jcr->jr.JobId, ec2));
1000 db_sql_query(mig_jcr->db, query.c_str(), NULL, NULL);
1002 /* Now marke the previous job as migrated */
1003 Mmsg(query, "UPDATE Job SET Type='%c' WHERE JobId=%s",
1004 (char)JT_MIGRATED_JOB, edit_uint64(jcr->previous_jr.JobId, ec1));
1005 db_sql_query(mig_jcr->db, query.c_str(), NULL, NULL);
1007 if (!db_get_job_record(jcr, jcr->db, &jcr->jr)) {
1008 Jmsg(jcr, M_WARNING, 0, _("Error getting job record for stats: %s"),
1009 db_strerror(jcr->db));
1010 set_jcr_job_status(jcr, JS_ErrorTerminated);
1013 bstrncpy(mr.VolumeName, jcr->VolumeName, sizeof(mr.VolumeName));
1014 if (!db_get_media_record(jcr, jcr->db, &mr)) {
1015 Jmsg(jcr, M_WARNING, 0, _("Error getting Media record for Volume \"%s\": ERR=%s"),
1016 mr.VolumeName, db_strerror(jcr->db));
1017 set_jcr_job_status(jcr, JS_ErrorTerminated);
1020 update_bootstrap_file(mig_jcr);
1022 if (!db_get_job_volume_names(mig_jcr, mig_jcr->db, mig_jcr->jr.JobId, &mig_jcr->VolumeName)) {
1024 * Note, if the job has erred, most likely it did not write any
1025 * tape, so suppress this "error" message since in that case
1026 * it is normal. Or look at it the other way, only for a
1027 * normal exit should we complain about this error.
1029 if (jcr->JobStatus == JS_Terminated && jcr->jr.JobBytes) {
1030 Jmsg(jcr, M_ERROR, 0, "%s", db_strerror(mig_jcr->db));
1032 mig_jcr->VolumeName[0] = 0; /* none */
1034 switch (jcr->JobStatus) {
1036 if (jcr->Errors || jcr->SDErrors) {
1037 term_msg = _("%s OK -- with warnings");
1039 term_msg = _("%s OK");
1043 case JS_ErrorTerminated:
1044 term_msg = _("*** %s Error ***");
1045 msg_type = M_ERROR; /* Generate error message */
1046 if (jcr->store_bsock) {
1047 bnet_sig(jcr->store_bsock, BNET_TERMINATE);
1048 if (jcr->SD_msg_chan) {
1049 pthread_cancel(jcr->SD_msg_chan);
1054 term_msg = _("%s Canceled");
1055 if (jcr->store_bsock) {
1056 bnet_sig(jcr->store_bsock, BNET_TERMINATE);
1057 if (jcr->SD_msg_chan) {
1058 pthread_cancel(jcr->SD_msg_chan);
1063 term_msg = _("Inappropriate %s term code");
1067 term_msg = _("%s -- no files to migrate");
1070 bsnprintf(term_code, sizeof(term_code), term_msg, "Migration");
1071 bstrftimes(sdt, sizeof(sdt), jcr->jr.StartTime);
1072 bstrftimes(edt, sizeof(edt), jcr->jr.EndTime);
1073 RunTime = jcr->jr.EndTime - jcr->jr.StartTime;
1077 kbps = (double)jcr->SDJobBytes / (1000 * RunTime);
1081 jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg));
1083 Jmsg(jcr, msg_type, 0, _("Bacula %s (%s): %s\n"
1084 " Prev Backup JobId: %s\n"
1085 " New Backup JobId: %s\n"
1086 " Migration JobId: %s\n"
1087 " Migration Job: %s\n"
1088 " Backup Level: %s%s\n"
1090 " FileSet: \"%s\" %s\n"
1091 " Pool: \"%s\" (From %s)\n"
1092 " Read Storage: \"%s\" (From %s)\n"
1093 " Write Storage: \"%s\" (From %s)\n"
1096 " Elapsed time: %s\n"
1098 " SD Files Written: %s\n"
1099 " SD Bytes Written: %s (%sB)\n"
1100 " Rate: %.1f KB/s\n"
1101 " Volume name(s): %s\n"
1102 " Volume Session Id: %d\n"
1103 " Volume Session Time: %d\n"
1104 " Last Volume Bytes: %s (%sB)\n"
1106 " SD termination status: %s\n"
1107 " Termination: %s\n\n"),
1111 edit_uint64(jcr->previous_jr.JobId, ec6),
1112 mig_jcr ? edit_uint64(mig_jcr->jr.JobId, ec7) : "0",
1113 edit_uint64(jcr->jr.JobId, ec8),
1115 level_to_str(jcr->JobLevel), jcr->since,
1116 jcr->client->name(),
1117 jcr->fileset->name(), jcr->FSCreateTime,
1118 jcr->pool->name(), jcr->pool_source,
1119 jcr->rstore?jcr->rstore->name():"*None*",
1120 NPRT(jcr->rstore_source),
1121 jcr->wstore?jcr->wstore->name():"*None*",
1122 NPRT(jcr->wstore_source),
1125 edit_utime(RunTime, elapsed, sizeof(elapsed)),
1127 edit_uint64_with_commas(jcr->SDJobFiles, ec1),
1128 edit_uint64_with_commas(jcr->SDJobBytes, ec2),
1129 edit_uint64_with_suffix(jcr->SDJobBytes, ec3),
1131 mig_jcr ? mig_jcr->VolumeName : "",
1133 jcr->VolSessionTime,
1134 edit_uint64_with_commas(mr.VolBytes, ec4),
1135 edit_uint64_with_suffix(mr.VolBytes, ec5),
1140 Dmsg1(100, "migrate_cleanup() mig_jcr=0x%x\n", jcr->mig_jcr);
1142 free_jcr(jcr->mig_jcr);
1143 jcr->mig_jcr = NULL;
1145 Dmsg0(100, "Leave migrate_cleanup()\n");
1149 * Return next DBId from comma separated list
1152 * 1 if next DBId returned
1153 * 0 if no more DBIds are in list
1154 * -1 there is an error
1156 static int get_next_dbid_from_list(char **p, DBId_t *DBId)
1162 for (int i=0; i<(int)sizeof(id); i++) {
1165 } else if (*q == ',') {
1174 } else if (!is_a_number(id)) {
1175 return -1; /* error */
1178 *DBId = str_to_int64(id);