2 Bacula® - The Network Backup Solution
4 Copyright (C) 2004-2007 Free Software Foundation Europe e.V.
6 The main author of Bacula is Kern Sibbald, with contributions from
7 many others, a complete list can be found in the file AUTHORS.
8 This program is Free Software; you can redistribute it and/or
9 modify it under the terms of version two of the GNU General Public
10 License as published by the Free Software Foundation plus additions
11 that are listed in the file LICENSE.
13 This program is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; if not, write to the Free Software
20 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
23 Bacula® is a registered trademark of John Walker.
24 The licensor of Bacula is the Free Software Foundation Europe
25 (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich,
26 Switzerland, email:ftf@fsfeurope.org.
30 * Bacula Director -- migrate.c -- responsible for doing
33 * Kern Sibbald, September MMIV
35 * Basic tasks done here:
36 * Open DB and create records for this job.
37 * Open Message Channel with Storage daemon to tell him a job will be starting.
38 * Open connection with Storage daemon and pass him commands
40 * When the Storage daemon finishes the job, update the DB.
49 #include "lib/bregex.h"
54 static const int dbglevel = 10;
56 static char OKbootstrap[] = "3000 OK bootstrap\n";
57 static int get_job_to_migrate(JCR *jcr);
59 static bool regex_find_jobids(JCR *jcr, idpkt *ids, const char *query1,
60 const char *query2, const char *type);
61 static bool find_mediaid_then_jobids(JCR *jcr, idpkt *ids, const char *query1,
63 static bool find_jobids_from_mediaid_list(JCR *jcr, idpkt *ids, const char *type);
64 static void start_migration_job(JCR *jcr);
65 static int get_next_dbid_from_list(char **p, DBId_t *DBId);
68 * Called here before the job is run to do the job
69 * specific setup. Note, one of the important things to
70 * complete in this init code is to make the definitive
71 * choice of input and output storage devices. This is
72 * because immediately after the init, the job is queued
73 * in the jobq.c code, and it checks that all the resources
74 * (storage resources in particular) are available, so these
75 * must all be properly defined.
77 * previous_jr refers to the job DB record of the Job that is
78 * going to be migrated.
79 * prev_job refers to the job resource of the Job that is
80 * going to be migrated.
81 * jcr is the jcr for the current "migration" job. It is a
82 * control job that is put in the DB as a migration job, which
83 * means that this job migrated a previous job to a new job.
84 * No Volume or File data is associated with this control
86 * mig_jcr refers to the newly migrated job that is run by
87 * the current jcr. It is a backup job that moves (migrates) the
88 * data written for the previous_jr into the new pool. This
89 * job (mig_jcr) becomes the new backup job that replaces
90 * the original backup job. Note, this jcr is not really run. It
91 * is simply attached to the current jcr. It will show up in
92 * the Director's status output, but not in the SD or FD, both of
93 * which deal only with the current migration job (i.e. jcr).
95 bool do_migration_init(JCR *jcr)
101 JCR *mig_jcr; /* newly migrated job */
105 apply_pool_overrides(jcr);
107 jcr->jr.PoolId = get_or_create_pool_record(jcr, jcr->pool->name());
108 if (jcr->jr.PoolId == 0) {
109 Dmsg1(dbglevel, "JobId=%d no PoolId\n", (int)jcr->JobId);
110 Jmsg(jcr, M_FATAL, 0, _("Could not get or create a Pool record.\n"));
114 * Note, at this point, pool is the pool for this job. We
115 * transfer it to rpool (read pool), and a bit later,
116 * pool will be changed to point to the write pool,
117 * which comes from pool->NextPool.
119 jcr->rpool = jcr->pool; /* save read pool */
120 pm_strcpy(jcr->rpool_source, jcr->pool_source);
123 Dmsg2(dbglevel, "Read pool=%s (From %s)\n", jcr->rpool->name(), jcr->rpool_source);
125 /* If we find a job or jobs to migrate it is previous_jr.JobId */
126 count = get_job_to_migrate(jcr);
134 Dmsg1(dbglevel, "Back from get_job_to_migrate JobId=%d\n", (int)jcr->JobId);
136 if (jcr->previous_jr.JobId == 0) {
137 Dmsg1(dbglevel, "JobId=%d no previous JobId\n", (int)jcr->JobId);
138 Jmsg(jcr, M_INFO, 0, _("No previous Job found to migrate.\n"));
139 return true; /* no work */
142 if (!get_or_create_fileset_record(jcr)) {
143 Dmsg1(dbglevel, "JobId=%d no FileSet\n", (int)jcr->JobId);
144 Jmsg(jcr, M_FATAL, 0, _("Could not get or create the FileSet record.\n"));
148 create_restore_bootstrap_file(jcr);
150 if (jcr->previous_jr.JobId == 0 || jcr->ExpectedFiles == 0) {
151 set_jcr_job_status(jcr, JS_Terminated);
152 Dmsg1(dbglevel, "JobId=%d expected files == 0\n", (int)jcr->JobId);
153 if (jcr->previous_jr.JobId == 0) {
154 Jmsg(jcr, M_INFO, 0, _("No previous Job found to migrate.\n"));
156 Jmsg(jcr, M_INFO, 0, _("Previous Job has no data to migrate.\n"));
158 return true; /* no work */
161 Dmsg5(dbglevel, "JobId=%d: Previous: Name=%s JobId=%d Type=%c Level=%c\n",
163 jcr->previous_jr.Name, (int)jcr->previous_jr.JobId,
164 jcr->previous_jr.JobType, jcr->previous_jr.JobLevel);
166 Dmsg5(dbglevel, "JobId=%d: Current: Name=%s JobId=%d Type=%c Level=%c\n",
168 jcr->jr.Name, (int)jcr->jr.JobId,
169 jcr->jr.JobType, jcr->jr.JobLevel);
172 job = (JOB *)GetResWithName(R_JOB, jcr->jr.Name);
173 prev_job = (JOB *)GetResWithName(R_JOB, jcr->previous_jr.Name);
176 Jmsg(jcr, M_FATAL, 0, _("Job resource not found for \"%s\".\n"), jcr->jr.Name);
180 Jmsg(jcr, M_FATAL, 0, _("Previous Job resource not found for \"%s\".\n"),
181 jcr->previous_jr.Name);
185 /* Create a migation jcr */
186 mig_jcr = jcr->mig_jcr = new_jcr(sizeof(JCR), dird_free_jcr);
187 memcpy(&mig_jcr->previous_jr, &jcr->previous_jr, sizeof(mig_jcr->previous_jr));
190 * Turn the mig_jcr into a "real" job that takes on the aspects of
191 * the previous backup job "prev_job".
193 set_jcr_defaults(mig_jcr, prev_job);
194 if (!setup_job(mig_jcr)) {
195 Jmsg(jcr, M_FATAL, 0, _("setup job failed.\n"));
199 /* Now reset the job record from the previous job */
200 memcpy(&mig_jcr->jr, &jcr->previous_jr, sizeof(mig_jcr->jr));
201 /* Update the jr to reflect the new values of PoolId, FileSetId, and JobId. */
202 mig_jcr->jr.PoolId = jcr->jr.PoolId;
203 mig_jcr->jr.FileSetId = jcr->jr.FileSetId;
204 mig_jcr->jr.JobId = mig_jcr->JobId;
206 Dmsg4(dbglevel, "mig_jcr: Name=%s JobId=%d Type=%c Level=%c\n",
207 mig_jcr->jr.Name, (int)mig_jcr->jr.JobId,
208 mig_jcr->jr.JobType, mig_jcr->jr.JobLevel);
211 * Get the PoolId used with the original job. Then
212 * find the pool name from the database record.
214 memset(&pr, 0, sizeof(pr));
215 pr.PoolId = mig_jcr->previous_jr.PoolId;
216 if (!db_get_pool_record(jcr, jcr->db, &pr)) {
217 Jmsg(jcr, M_FATAL, 0, _("Pool for JobId %s not in database. ERR=%s\n"),
218 edit_int64(pr.PoolId, ed1), db_strerror(jcr->db));
221 /* Get the pool resource corresponding to the original job */
222 pool = (POOL *)GetResWithName(R_POOL, pr.Name);
224 Jmsg(jcr, M_FATAL, 0, _("Pool resource \"%s\" not found.\n"), pr.Name);
228 /* If pool storage specified, use it for restore */
229 copy_rstorage(mig_jcr, pool->storage, _("Pool resource"));
230 copy_rstorage(jcr, pool->storage, _("Pool resource"));
233 * If the original backup pool has a NextPool, make sure a
234 * record exists in the database. Note, in this case, we
235 * will be migrating from pool to pool->NextPool.
237 if (pool->NextPool) {
238 jcr->jr.PoolId = get_or_create_pool_record(jcr, pool->NextPool->name());
239 if (jcr->jr.PoolId == 0) {
243 if (!set_migration_wstorage(jcr, pool)) {
246 mig_jcr->pool = jcr->pool = pool->NextPool;
247 pm_strcpy(jcr->pool_source, _("Job Pool's NextPool resource"));
248 mig_jcr->jr.PoolId = jcr->jr.PoolId;
250 Dmsg2(dbglevel, "Write pool=%s read rpool=%s\n", jcr->pool->name(), jcr->rpool->name());
255 * Do a Migration of a previous job
257 * Returns: false on failure
260 bool do_migration(JCR *jcr)
264 JCR *mig_jcr = jcr->mig_jcr; /* newly migrated job */
267 * If mig_jcr is NULL, there is nothing to do for this job,
268 * so set a normal status, cleanup and return OK.
271 set_jcr_job_status(jcr, JS_Terminated);
272 migration_cleanup(jcr, jcr->JobStatus);
276 /* Print Job Start message */
277 Jmsg(jcr, M_INFO, 0, _("Start Migration JobId %s, Job=%s\n"),
278 edit_uint64(jcr->JobId, ed1), jcr->Job);
283 * Open a message channel connection with the Storage
284 * daemon. This is to let him know that our client
285 * will be contacting him for a backup session.
288 Dmsg0(110, "Open connection with storage daemon\n");
289 set_jcr_job_status(jcr, JS_WaitSD);
290 set_jcr_job_status(mig_jcr, JS_WaitSD);
292 * Start conversation with Storage daemon
294 if (!connect_to_storage_daemon(jcr, 10, SDConnectTimeout, 1)) {
297 sd = jcr->store_bsock;
299 * Now start a job with the Storage daemon
301 Dmsg2(dbglevel, "Read store=%s, write store=%s\n",
302 ((STORE *)jcr->rstorage->first())->name(),
303 ((STORE *)jcr->wstorage->first())->name());
304 if (((STORE *)jcr->rstorage->first())->name() == ((STORE *)jcr->wstorage->first())->name()) {
305 Jmsg(jcr, M_FATAL, 0, _("Read storage \"%s\" same as write storage.\n"),
306 ((STORE *)jcr->rstorage->first())->name());
309 if (!start_storage_daemon_job(jcr, jcr->rstorage, jcr->wstorage)) {
312 Dmsg0(150, "Storage daemon connection OK\n");
314 if (!send_bootstrap_file(jcr, sd) ||
315 !response(jcr, sd, OKbootstrap, "Bootstrap", DISPLAY_ERROR)) {
320 * We re-update the job start record so that the start
321 * time is set after the run before job. This avoids
322 * that any files created by the run before job will
323 * be saved twice. They will be backed up in the current
324 * job, but not in the next one unless they are changed.
325 * Without this, they will be backed up in this job and
326 * in the next job run because in that case, their date
327 * is after the start of this run.
329 jcr->start_time = time(NULL);
330 jcr->jr.StartTime = jcr->start_time;
331 jcr->jr.JobTDate = jcr->start_time;
332 set_jcr_job_status(jcr, JS_Running);
334 /* Update job start record for this migration control job */
335 if (!db_update_job_start_record(jcr, jcr->db, &jcr->jr)) {
336 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(jcr->db));
341 mig_jcr->start_time = time(NULL);
342 mig_jcr->jr.StartTime = mig_jcr->start_time;
343 mig_jcr->jr.JobTDate = mig_jcr->start_time;
344 set_jcr_job_status(mig_jcr, JS_Running);
346 /* Update job start record for the real migration backup job */
347 if (!db_update_job_start_record(mig_jcr, mig_jcr->db, &mig_jcr->jr)) {
348 Jmsg(jcr, M_FATAL, 0, "%s", db_strerror(mig_jcr->db));
352 Dmsg4(dbglevel, "mig_jcr: Name=%s JobId=%d Type=%c Level=%c\n",
353 mig_jcr->jr.Name, (int)mig_jcr->jr.JobId,
354 mig_jcr->jr.JobType, mig_jcr->jr.JobLevel);
358 * Start the job prior to starting the message thread below
359 * to avoid two threads from using the BSOCK structure at
362 if (!bnet_fsend(sd, "run")) {
367 * Now start a Storage daemon message thread
369 if (!start_storage_daemon_message_thread(jcr)) {
374 set_jcr_job_status(jcr, JS_Running);
375 set_jcr_job_status(mig_jcr, JS_Running);
377 /* Pickup Job termination data */
378 /* Note, the SD stores in jcr->JobFiles/ReadBytes/JobBytes/Errors */
379 wait_for_storage_daemon_termination(jcr);
380 set_jcr_job_status(jcr, jcr->SDJobStatus);
381 db_write_batch_file_records(jcr); /* used by bulk batch file insert */
382 if (jcr->JobStatus != JS_Terminated) {
386 migration_cleanup(jcr, jcr->JobStatus);
389 UAContext *ua = new_ua_context(jcr);
390 edit_uint64(jcr->previous_jr.JobId, jobid);
391 /* Purge all old file records, but leave Job record */
392 purge_files_from_jobs(ua, jobid);
403 /* Add an item to the list if it is unique */
404 static void add_unique_id(idpkt *ids, char *item)
409 /* Walk through current list to see if each item is the same as item */
412 for (int i=0; i<(int)sizeof(id); i++) {
415 } else if (*q == ',') {
422 if (strcmp(item, id) == 0) {
426 /* Did not find item, so add it to list */
427 if (ids->count == 0) {
430 pm_strcat(ids->list, ",");
432 pm_strcat(ids->list, item);
434 // Dmsg3(0, "add_uniq count=%d Ids=%p %s\n", ids->count, ids->list, ids->list);
439 * Callback handler make list of DB Ids
441 static int unique_dbid_handler(void *ctx, int num_fields, char **row)
443 idpkt *ids = (idpkt *)ctx;
445 add_unique_id(ids, row[0]);
446 Dmsg3(dbglevel, "dbid_hdlr count=%d Ids=%p %s\n", ids->count, ids->list, ids->list);
456 static int item_compare(void *item1, void *item2)
458 uitem *i1 = (uitem *)item1;
459 uitem *i2 = (uitem *)item2;
460 return strcmp(i1->item, i2->item);
463 static int unique_name_handler(void *ctx, int num_fields, char **row)
465 dlist *list = (dlist *)ctx;
467 uitem *new_item = (uitem *)malloc(sizeof(uitem));
470 memset(new_item, 0, sizeof(uitem));
471 new_item->item = bstrdup(row[0]);
472 Dmsg1(dbglevel, "Unique_name_hdlr Item=%s\n", row[0]);
473 item = (uitem *)list->binary_insert((void *)new_item, item_compare);
474 if (item != new_item) { /* already in list */
475 free(new_item->item);
476 free((char *)new_item);
482 /* Get Job names in Pool */
483 const char *sql_job =
484 "SELECT DISTINCT Job.Name from Job,Pool"
485 " WHERE Pool.Name='%s' AND Job.PoolId=Pool.PoolId";
487 /* Get JobIds from regex'ed Job names */
488 const char *sql_jobids_from_job =
489 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Job,Pool"
490 " WHERE Job.Name='%s' AND Pool.Name='%s' AND Job.PoolId=Pool.PoolId"
491 " ORDER by Job.StartTime";
493 /* Get Client names in Pool */
494 const char *sql_client =
495 "SELECT DISTINCT Client.Name from Client,Pool,Job"
496 " WHERE Pool.Name='%s' AND Job.ClientId=Client.ClientId AND"
497 " Job.PoolId=Pool.PoolId";
499 /* Get JobIds from regex'ed Client names */
500 const char *sql_jobids_from_client =
501 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Job,Pool,Client"
502 " WHERE Client.Name='%s' AND Pool.Name='%s' AND Job.PoolId=Pool.PoolId"
503 " AND Job.ClientId=Client.ClientId AND Job.Type='B'"
504 " ORDER by Job.StartTime";
506 /* Get Volume names in Pool */
507 const char *sql_vol =
508 "SELECT DISTINCT VolumeName FROM Media,Pool WHERE"
509 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
510 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'";
512 /* Get JobIds from regex'ed Volume names */
513 const char *sql_jobids_from_vol =
514 "SELECT DISTINCT Job.JobId,Job.StartTime FROM Media,JobMedia,Job"
515 " WHERE Media.VolumeName='%s' AND Media.MediaId=JobMedia.MediaId"
516 " AND JobMedia.JobId=Job.JobId AND Job.Type='B'"
517 " ORDER by Job.StartTime";
520 const char *sql_smallest_vol =
521 "SELECT MediaId FROM Media,Pool WHERE"
522 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
523 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'"
524 " ORDER BY VolBytes ASC LIMIT 1";
526 const char *sql_oldest_vol =
527 "SELECT MediaId FROM Media,Pool WHERE"
528 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
529 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'"
530 " ORDER BY LastWritten ASC LIMIT 1";
532 /* Get JobIds when we have selected MediaId */
533 const char *sql_jobids_from_mediaid =
534 "SELECT DISTINCT Job.JobId,Job.StartTime FROM JobMedia,Job"
535 " WHERE JobMedia.JobId=Job.JobId AND JobMedia.MediaId=%s"
537 " ORDER by Job.StartTime";
539 /* Get tne number of bytes in the pool */
540 const char *sql_pool_bytes =
541 "SELECT SUM(VolBytes) FROM Media,Pool WHERE"
542 " VolStatus in ('Full','Used','Error','Append') AND Media.Enabled=1 AND"
543 " Media.PoolId=Pool.PoolId AND Pool.Name='%s'";
545 /* Get tne number of bytes in the Jobs */
546 const char *sql_job_bytes =
547 "SELECT SUM(JobBytes) FROM Job WHERE JobId IN (%s)";
550 /* Get Media Ids in Pool */
551 const char *sql_mediaids =
552 "SELECT MediaId FROM Media,Pool WHERE"
553 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
554 " Media.PoolId=Pool.PoolId AND Pool.Name='%s' ORDER BY LastWritten ASC";
556 /* Get JobIds in Pool longer than specified time */
557 const char *sql_pool_time =
558 "SELECT DISTINCT Job.JobId from Pool,Job,Media,JobMedia WHERE"
559 " Pool.Name='%s' AND Media.PoolId=Pool.PoolId AND"
560 " VolStatus in ('Full','Used','Error') AND Media.Enabled=1 AND"
562 " JobMedia.JobId=Job.JobId AND Job.PoolId=Media.PoolId"
563 " AND Job.RealEndTime<='%s'";
566 * const char *sql_ujobid =
567 * "SELECT DISTINCT Job.Job from Client,Pool,Media,Job,JobMedia "
568 * " WHERE Media.PoolId=Pool.PoolId AND Pool.Name='%s' AND"
569 * " JobMedia.JobId=Job.JobId AND Job.PoolId=Media.PoolId";
576 * This is the central piece of code that finds a job or jobs
577 * actually JobIds to migrate. It first looks to see if one
578 * has been "manually" specified in jcr->MigrateJobId, and if
579 * so, it returns that JobId to be run. Otherwise, it
580 * examines the Selection Type to see what kind of migration
581 * we are doing (Volume, Job, Client, ...) and applies any
582 * Selection Pattern if appropriate to obtain a list of JobIds.
583 * Finally, it will loop over all the JobIds found, except the last
584 * one starting a new job with MigrationJobId set to that JobId, and
585 * finally, it returns the last JobId to the caller.
587 * Returns: -1 on error
588 * 0 if no jobs to migrate
589 * 1 if OK and jcr->previous_jr filled in
591 static int get_job_to_migrate(JCR *jcr)
593 char ed1[30], ed2[30];
594 POOL_MEM query(PM_MESSAGE);
599 idpkt ids, mid, jids;
604 char dt[MAX_TIME_LENGTH];
607 ids.list = get_pool_memory(PM_MESSAGE);
610 mid.list = get_pool_memory(PM_MESSAGE);
613 jids.list = get_pool_memory(PM_MESSAGE);
619 * If MigrateJobId is set, then we migrate only that Job,
620 * otherwise, we go through the full selection of jobs to
623 if (jcr->MigrateJobId != 0) {
624 Dmsg1(dbglevel, "At Job start previous jobid=%u\n", jcr->MigrateJobId);
625 edit_uint64(jcr->MigrateJobId, ids.list);
628 switch (jcr->job->selection_type) {
630 if (!regex_find_jobids(jcr, &ids, sql_job, sql_jobids_from_job, "Job")) {
635 if (!regex_find_jobids(jcr, &ids, sql_client, sql_jobids_from_client, "Client")) {
640 if (!regex_find_jobids(jcr, &ids, sql_vol, sql_jobids_from_vol, "Volume")) {
645 if (!jcr->job->selection_pattern) {
646 Jmsg(jcr, M_FATAL, 0, _("No Migration SQL selection pattern specified.\n"));
649 Dmsg1(dbglevel, "SQL=%s\n", jcr->job->selection_pattern);
650 if (!db_sql_query(jcr->db, jcr->job->selection_pattern,
651 unique_dbid_handler, (void *)&ids)) {
652 Jmsg(jcr, M_FATAL, 0,
653 _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
657 case MT_SMALLEST_VOL:
658 if (!find_mediaid_then_jobids(jcr, &ids, sql_smallest_vol, "Smallest Volume")) {
663 if (!find_mediaid_then_jobids(jcr, &ids, sql_oldest_vol, "Oldest Volume")) {
668 case MT_POOL_OCCUPANCY:
670 /* Find count of bytes in pool */
671 Mmsg(query, sql_pool_bytes, jcr->rpool->name());
672 if (!db_sql_query(jcr->db, query.c_str(), db_int64_handler, (void *)&ctx)) {
673 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
676 if (ctx.count == 0) {
677 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
680 pool_bytes = ctx.value;
681 Dmsg2(dbglevel, "highbytes=%d pool=%d\n", (int)jcr->rpool->MigrationHighBytes,
683 if (pool_bytes < (int64_t)jcr->rpool->MigrationHighBytes) {
684 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
687 Dmsg0(dbglevel, "We should do Occupation migration.\n");
690 /* Find a list of MediaIds that could be migrated */
691 Mmsg(query, sql_mediaids, jcr->rpool->name());
692 Dmsg1(dbglevel, "query=%s\n", query.c_str());
693 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)&ids)) {
694 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
697 if (ids.count == 0) {
698 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
701 Dmsg2(dbglevel, "Pool Occupancy ids=%d MediaIds=%s\n", ids.count, ids.list);
704 * Now loop over MediaIds getting more JobIds to migrate until
705 * we reduce the pool occupancy below the low water mark.
708 for (int i=0; i < (int)ids.count; i++) {
709 stat = get_next_dbid_from_list(&p, &MediaId);
710 Dmsg2(dbglevel, "get_next_dbid stat=%d MediaId=%u\n", stat, MediaId);
712 Jmsg(jcr, M_FATAL, 0, _("Invalid MediaId found.\n"));
714 } else if (stat == 0) {
718 Mmsg(mid.list, "%s", edit_int64(MediaId, ed1));
719 if (!find_jobids_from_mediaid_list(jcr, &mid, "Volumes")) {
723 pm_strcat(jids.list, ",");
725 pm_strcat(jids.list, mid.list);
726 jids.count += mid.count;
728 /* Now get the count of bytes added */
730 /* Find count of bytes from Jobs */
731 Mmsg(query, sql_job_bytes, mid.list);
732 Dmsg1(dbglevel, "Jobbytes query: %s\n", query.c_str());
733 if (!db_sql_query(jcr->db, query.c_str(), db_int64_handler, (void *)&ctx)) {
734 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
737 pool_bytes -= ctx.value;
738 Dmsg1(dbglevel, "Total migrate Job bytes=%s\n", edit_int64(ctx.value, ed1));
739 Dmsg2(dbglevel, "lowbytes=%s poolafter=%s\n",
740 edit_int64(jcr->rpool->MigrationLowBytes, ed1),
741 edit_int64(pool_bytes, ed2));
742 if (pool_bytes <= (int64_t)jcr->rpool->MigrationLowBytes) {
743 Dmsg0(dbglevel, "We should be done.\n");
748 /* Transfer jids to ids, where the jobs list is expected */
749 ids.count = jids.count;
750 pm_strcpy(ids.list, jids.list);
751 Dmsg2(dbglevel, "Pool Occupancy ids=%d JobIds=%s\n", ids.count, ids.list);
755 ttime = time(NULL) - (time_t)jcr->rpool->MigrationTime;
756 (void)localtime_r(&ttime, &tm);
757 strftime(dt, sizeof(dt), "%Y-%m-%d %H:%M:%S", &tm);
760 Mmsg(query, sql_pool_time, jcr->rpool->name(), dt);
761 Dmsg1(dbglevel, "query=%s\n", query.c_str());
762 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)&ids)) {
763 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
766 if (ids.count == 0) {
767 Jmsg(jcr, M_INFO, 0, _("No Volumes found to migrate.\n"));
770 Dmsg2(dbglevel, "PoolTime ids=%d JobIds=%s\n", ids.count, ids.list);
774 Jmsg(jcr, M_FATAL, 0, _("Unknown Migration Selection Type.\n"));
780 * Loop over all jobids except the last one, sending
781 * them to start_migration_job(), which will start a job
782 * for each of them. For the last JobId, we handle it below.
785 if (ids.count == 0) {
786 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
790 Jmsg(jcr, M_INFO, 0, _("The following %u JobId%s were chosen to be migrated: %s\n"),
791 ids.count, ids.count==0?"":"s", ids.list);
793 Dmsg2(dbglevel, "Before loop count=%d ids=%s\n", ids.count, ids.list);
794 for (int i=1; i < (int)ids.count; i++) {
796 stat = get_next_jobid_from_list(&p, &JobId);
797 Dmsg3(dbglevel, "get_jobid_no=%d stat=%d JobId=%u\n", i, stat, JobId);
798 jcr->MigrateJobId = JobId;
799 start_migration_job(jcr);
800 Dmsg0(dbglevel, "Back from start_migration_job\n");
802 Jmsg(jcr, M_FATAL, 0, _("Invalid JobId found.\n"));
804 } else if (stat == 0) {
805 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
810 /* Now get the last JobId and handle it in the current job */
812 stat = get_next_jobid_from_list(&p, &JobId);
813 Dmsg2(dbglevel, "Last get_next_jobid stat=%d JobId=%u\n", stat, (int)JobId);
815 Jmsg(jcr, M_FATAL, 0, _("Invalid JobId found.\n"));
817 } else if (stat == 0) {
818 Jmsg(jcr, M_INFO, 0, _("No JobIds found to migrate.\n"));
822 jcr->previous_jr.JobId = JobId;
823 Dmsg1(dbglevel, "Previous jobid=%d\n", (int)jcr->previous_jr.JobId);
825 if (!db_get_job_record(jcr, jcr->db, &jcr->previous_jr)) {
826 Jmsg(jcr, M_FATAL, 0, _("Could not get job record for JobId %s to migrate. ERR=%s"),
827 edit_int64(jcr->previous_jr.JobId, ed1),
828 db_strerror(jcr->db));
831 Jmsg(jcr, M_INFO, 0, _("Migration using JobId=%s Job=%s\n"),
832 edit_int64(jcr->previous_jr.JobId, ed1), jcr->previous_jr.Job);
833 Dmsg3(dbglevel, "Migration JobId=%d using JobId=%s Job=%s\n",
835 edit_int64(jcr->previous_jr.JobId, ed1), jcr->previous_jr.Job);
845 free_pool_memory(ids.list);
846 free_pool_memory(mid.list);
847 free_pool_memory(jids.list);
851 static void start_migration_job(JCR *jcr)
853 UAContext *ua = new_ua_context(jcr);
856 Mmsg(ua->cmd, "run %s jobid=%s", jcr->job->hdr.name,
857 edit_uint64(jcr->MigrateJobId, ed1));
858 Dmsg1(dbglevel, "=============== Migration cmd=%s\n", ua->cmd);
859 parse_ua_args(ua); /* parse command */
860 int stat = run_cmd(ua, ua->cmd);
862 Jmsg(jcr, M_ERROR, 0, _("Could not start migration job.\n"));
864 Jmsg(jcr, M_INFO, 0, _("Migration JobId %d started.\n"), stat);
869 static bool find_mediaid_then_jobids(JCR *jcr, idpkt *ids, const char *query1,
873 POOL_MEM query(PM_MESSAGE);
876 /* Basic query for MediaId */
877 Mmsg(query, query1, jcr->rpool->name());
878 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
879 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
882 if (ids->count == 0) {
883 Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type);
884 ok = true; /* Not an error */
886 } else if (ids->count != 1) {
887 Jmsg(jcr, M_FATAL, 0, _("SQL error. Expected 1 MediaId got %d\n"),
891 Dmsg1(dbglevel, "Smallest Vol Jobids=%s\n", ids->list);
893 ok = find_jobids_from_mediaid_list(jcr, ids, type);
899 static bool find_jobids_from_mediaid_list(JCR *jcr, idpkt *ids, const char *type)
902 POOL_MEM query(PM_MESSAGE);
904 Mmsg(query, sql_jobids_from_mediaid, ids->list);
906 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
907 Jmsg(jcr, M_FATAL, 0, _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
910 if (ids->count == 0) {
911 Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type);
918 static bool regex_find_jobids(JCR *jcr, idpkt *ids, const char *query1,
919 const char *query2, const char *type)
923 uitem *last_item = NULL;
928 POOL_MEM query(PM_MESSAGE);
930 item_chain = New(dlist(item, &item->link));
931 if (!jcr->job->selection_pattern) {
932 Jmsg(jcr, M_FATAL, 0, _("No Migration %s selection pattern specified.\n"),
936 Dmsg1(dbglevel, "regex-sel-pattern=%s\n", jcr->job->selection_pattern);
937 /* Basic query for names */
938 Mmsg(query, query1, jcr->rpool->name());
939 Dmsg1(dbglevel, "get name query1=%s\n", query.c_str());
940 if (!db_sql_query(jcr->db, query.c_str(), unique_name_handler,
941 (void *)item_chain)) {
942 Jmsg(jcr, M_FATAL, 0,
943 _("SQL to get %s failed. ERR=%s\n"), type, db_strerror(jcr->db));
946 Dmsg1(dbglevel, "query1 returned %d names\n", item_chain->size());
947 if (item_chain->size() == 0) {
948 Jmsg(jcr, M_INFO, 0, _("Query of Pool \"%s\" returned no Jobs to migrate.\n"),
951 goto bail_out; /* skip regex match */
953 /* Compile regex expression */
954 rc = regcomp(&preg, jcr->job->selection_pattern, REG_EXTENDED);
956 regerror(rc, &preg, prbuf, sizeof(prbuf));
957 Jmsg(jcr, M_FATAL, 0, _("Could not compile regex pattern \"%s\" ERR=%s\n"),
958 jcr->job->selection_pattern, prbuf);
961 /* Now apply the regex to the names and remove any item not matched */
962 foreach_dlist(item, item_chain) {
963 const int nmatch = 30;
964 regmatch_t pmatch[nmatch];
966 Dmsg1(dbglevel, "Remove item %s\n", last_item->item);
967 free(last_item->item);
968 item_chain->remove(last_item);
970 Dmsg1(dbglevel, "get name Item=%s\n", item->item);
971 rc = regexec(&preg, item->item, nmatch, pmatch, 0);
973 last_item = NULL; /* keep this one */
979 free(last_item->item);
980 Dmsg1(dbglevel, "Remove item %s\n", last_item->item);
981 item_chain->remove(last_item);
985 if (item_chain->size() == 0) {
986 Jmsg(jcr, M_INFO, 0, _("Regex pattern matched no Jobs to migrate.\n"));
988 goto bail_out; /* skip regex match */
992 * At this point, we have a list of items in item_chain
993 * that have been matched by the regex, so now we need
994 * to look up their jobids.
997 foreach_dlist(item, item_chain) {
998 Dmsg2(dbglevel, "Got %s: %s\n", type, item->item);
999 Mmsg(query, query2, item->item, jcr->rpool->name());
1000 Dmsg1(dbglevel, "get id from name query2=%s\n", query.c_str());
1001 if (!db_sql_query(jcr->db, query.c_str(), unique_dbid_handler, (void *)ids)) {
1002 Jmsg(jcr, M_FATAL, 0,
1003 _("SQL failed. ERR=%s\n"), db_strerror(jcr->db));
1007 if (ids->count == 0) {
1008 Jmsg(jcr, M_INFO, 0, _("No %ss found to migrate.\n"), type);
1013 Dmsg2(dbglevel, "Count=%d Jobids=%s\n", ids->count, ids->list);
1014 foreach_dlist(item, item_chain) {
1023 * Release resources allocated during backup.
1025 void migration_cleanup(JCR *jcr, int TermCode)
1027 char sdt[MAX_TIME_LENGTH], edt[MAX_TIME_LENGTH];
1028 char ec1[30], ec2[30], ec3[30], ec4[30], ec5[30], elapsed[50];
1029 char ec6[50], ec7[50], ec8[50];
1030 char term_code[100], sd_term_msg[100];
1031 const char *term_msg;
1032 int msg_type = M_INFO;
1036 JCR *mig_jcr = jcr->mig_jcr;
1037 POOL_MEM query(PM_MESSAGE);
1039 Dmsg2(100, "Enter migrate_cleanup %d %c\n", TermCode, TermCode);
1040 update_job_end(jcr, TermCode);
1041 memset(&mr, 0, sizeof(mr));
1044 * Check if we actually did something.
1045 * mig_jcr is jcr of the newly migrated job.
1048 mig_jcr->JobFiles = jcr->JobFiles = jcr->SDJobFiles;
1049 mig_jcr->JobBytes = jcr->JobBytes = jcr->SDJobBytes;
1050 mig_jcr->VolSessionId = jcr->VolSessionId;
1051 mig_jcr->VolSessionTime = jcr->VolSessionTime;
1052 mig_jcr->jr.RealEndTime = 0;
1053 mig_jcr->jr.PriorJobId = jcr->previous_jr.JobId;
1055 update_job_end(mig_jcr, TermCode);
1057 /* Update final items to set them to the previous job's values */
1058 Mmsg(query, "UPDATE Job SET StartTime='%s',EndTime='%s',"
1059 "JobTDate=%s WHERE JobId=%s",
1060 jcr->previous_jr.cStartTime, jcr->previous_jr.cEndTime,
1061 edit_uint64(jcr->previous_jr.JobTDate, ec1),
1062 edit_uint64(mig_jcr->jr.JobId, ec2));
1063 db_sql_query(mig_jcr->db, query.c_str(), NULL, NULL);
1065 /* Now mark the previous job as migrated if it terminated normally */
1066 if (jcr->JobStatus == JS_Terminated) {
1067 Mmsg(query, "UPDATE Job SET Type='%c' WHERE JobId=%s",
1068 (char)JT_MIGRATED_JOB, edit_uint64(jcr->previous_jr.JobId, ec1));
1069 db_sql_query(mig_jcr->db, query.c_str(), NULL, NULL);
1072 if (!db_get_job_record(jcr, jcr->db, &jcr->jr)) {
1073 Jmsg(jcr, M_WARNING, 0, _("Error getting job record for stats: %s"),
1074 db_strerror(jcr->db));
1075 set_jcr_job_status(jcr, JS_ErrorTerminated);
1078 bstrncpy(mr.VolumeName, jcr->VolumeName, sizeof(mr.VolumeName));
1079 if (!db_get_media_record(jcr, jcr->db, &mr)) {
1080 Jmsg(jcr, M_WARNING, 0, _("Error getting Media record for Volume \"%s\": ERR=%s"),
1081 mr.VolumeName, db_strerror(jcr->db));
1082 set_jcr_job_status(jcr, JS_ErrorTerminated);
1085 update_bootstrap_file(mig_jcr);
1087 if (!db_get_job_volume_names(mig_jcr, mig_jcr->db, mig_jcr->jr.JobId, &mig_jcr->VolumeName)) {
1089 * Note, if the job has failed, most likely it did not write any
1090 * tape, so suppress this "error" message since in that case
1091 * it is normal. Or look at it the other way, only for a
1092 * normal exit should we complain about this error.
1094 if (jcr->JobStatus == JS_Terminated && jcr->jr.JobBytes) {
1095 Jmsg(jcr, M_ERROR, 0, "%s", db_strerror(mig_jcr->db));
1097 mig_jcr->VolumeName[0] = 0; /* none */
1099 switch (jcr->JobStatus) {
1101 if (jcr->Errors || jcr->SDErrors) {
1102 term_msg = _("%s OK -- with warnings");
1104 term_msg = _("%s OK");
1108 case JS_ErrorTerminated:
1109 term_msg = _("*** %s Error ***");
1110 msg_type = M_ERROR; /* Generate error message */
1111 if (jcr->store_bsock) {
1112 bnet_sig(jcr->store_bsock, BNET_TERMINATE);
1113 if (jcr->SD_msg_chan) {
1114 pthread_cancel(jcr->SD_msg_chan);
1119 term_msg = _("%s Canceled");
1120 if (jcr->store_bsock) {
1121 bnet_sig(jcr->store_bsock, BNET_TERMINATE);
1122 if (jcr->SD_msg_chan) {
1123 pthread_cancel(jcr->SD_msg_chan);
1128 term_msg = _("Inappropriate %s term code");
1132 if (jcr->previous_jr.JobId != 0) {
1133 /* Mark previous job as migrated */
1134 Mmsg(query, "UPDATE Job SET Type='%c' WHERE JobId=%s",
1135 (char)JT_MIGRATED_JOB, edit_uint64(jcr->previous_jr.JobId, ec1));
1136 Dmsg1(000, "Mark: %s\n", query.c_str());
1137 db_sql_query(jcr->db, query.c_str(), NULL, NULL);
1139 term_msg = _("%s -- no files to migrate");
1142 bsnprintf(term_code, sizeof(term_code), term_msg, "Migration");
1143 bstrftimes(sdt, sizeof(sdt), jcr->jr.StartTime);
1144 bstrftimes(edt, sizeof(edt), jcr->jr.EndTime);
1145 RunTime = jcr->jr.EndTime - jcr->jr.StartTime;
1149 kbps = (double)jcr->SDJobBytes / (1000 * RunTime);
1153 jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg));
1155 Jmsg(jcr, msg_type, 0, _("Bacula %s Version: %s (%s) %s %s %s at %s\n"
1156 " Prev Backup JobId: %s\n"
1157 " New Backup JobId: %s\n"
1158 " Migration JobId: %s\n"
1159 " Migration Job: %s\n"
1160 " Backup Level: %s%s\n"
1162 " FileSet: \"%s\" %s\n"
1163 " Read Pool: \"%s\" (From %s)\n"
1164 " Read Storage: \"%s\" (From %s)\n"
1165 " Write Pool: \"%s\" (From %s)\n"
1166 " Write Storage: \"%s\" (From %s)\n"
1169 " Elapsed time: %s\n"
1171 " SD Files Written: %s\n"
1172 " SD Bytes Written: %s (%sB)\n"
1173 " Rate: %.1f KB/s\n"
1174 " Volume name(s): %s\n"
1175 " Volume Session Id: %d\n"
1176 " Volume Session Time: %d\n"
1177 " Last Volume Bytes: %s (%sB)\n"
1179 " SD termination status: %s\n"
1180 " Termination: %s\n\n"),
1181 my_name, VERSION, BDATE, HOST_OS, DISTNAME, DISTVER,
1183 edit_uint64(jcr->previous_jr.JobId, ec6),
1184 mig_jcr ? edit_uint64(mig_jcr->jr.JobId, ec7) : "0",
1185 edit_uint64(jcr->jr.JobId, ec8),
1187 level_to_str(jcr->JobLevel), jcr->since,
1188 jcr->client->name(),
1189 jcr->fileset->name(), jcr->FSCreateTime,
1190 jcr->rpool->name(), jcr->rpool_source,
1191 jcr->rstore?jcr->rstore->name():"*None*",
1192 NPRT(jcr->rstore_source),
1193 jcr->pool->name(), jcr->pool_source,
1194 jcr->wstore?jcr->wstore->name():"*None*",
1195 NPRT(jcr->wstore_source),
1198 edit_utime(RunTime, elapsed, sizeof(elapsed)),
1200 edit_uint64_with_commas(jcr->SDJobFiles, ec1),
1201 edit_uint64_with_commas(jcr->SDJobBytes, ec2),
1202 edit_uint64_with_suffix(jcr->SDJobBytes, ec3),
1204 mig_jcr ? mig_jcr->VolumeName : "",
1206 jcr->VolSessionTime,
1207 edit_uint64_with_commas(mr.VolBytes, ec4),
1208 edit_uint64_with_suffix(mr.VolBytes, ec5),
1213 Dmsg1(100, "migrate_cleanup() mig_jcr=0x%x\n", jcr->mig_jcr);
1215 free_jcr(jcr->mig_jcr);
1216 jcr->mig_jcr = NULL;
1218 Dmsg0(100, "Leave migrate_cleanup()\n");
1222 * Return next DBId from comma separated list
1225 * 1 if next DBId returned
1226 * 0 if no more DBIds are in list
1227 * -1 there is an error
1229 static int get_next_dbid_from_list(char **p, DBId_t *DBId)
1235 for (int i=0; i<(int)sizeof(id); i++) {
1238 } else if (*q == ',') {
1247 } else if (!is_a_number(id)) {
1248 return -1; /* error */
1251 *DBId = str_to_int64(id);
1255 bool set_migration_wstorage(JCR *jcr, POOL *pool)
1257 POOL *wpool = pool->NextPool;
1260 Jmsg(jcr, M_FATAL, 0, _("No Next Pool specification found in Pool \"%s\".\n"),
1265 if (!wpool->storage || wpool->storage->size() == 0) {
1266 Jmsg(jcr, M_FATAL, 0, _("No Storage specification found in Next Pool \"%s\".\n"),
1271 /* If pool storage specified, use it instead of job storage for backup */
1272 copy_wstorage(jcr, wpool->storage, _("Storage from Pool's NextPool resource"));