From 508a6c5ed177f5a0f89d3aeca2f9acd680223d89 Mon Sep 17 00:00:00 2001 From: Kern Sibbald Date: Sat, 24 Mar 2007 21:55:54 +0000 Subject: [PATCH] kes Write new subroutine is_volume_purged() that explicitly checks if the Volume is purged, and if so marks it as such. This should resolve problems reported about needing to mount twice to recycle volumes. kes Rewrite pruning algorithm to do more work in the SQL engine, and to pass a list of JobIds to be deleted to SQL. Also, minimize the amount of duplicated code. kes Do volume pruning only for the Media Type desired (reduces pruning time if multiple Media Types are in the same pool). kes Implement more detailed info in the Job report for the Bacula version and architecture. kes Switch from POOLMEM to POOL_MEM (a real class) in ua_prune.c and ua_purge.c. git-svn-id: https://bacula.svn.sourceforge.net/svnroot/bacula/trunk@4408 91ce42f0-d328-0410-95d8-f526ca767f89 --- bacula/src/cats/mysql.c | 12 +- bacula/src/cats/protos.h | 2 +- bacula/src/cats/sql_cmds.c | 8 +- bacula/src/cats/sql_find.c | 5 +- bacula/src/cats/sql_get.c | 9 +- bacula/src/dird/autoprune.c | 57 +++-- bacula/src/dird/backup.c | 35 ++- bacula/src/dird/catreq.c | 2 +- bacula/src/dird/dird.h | 23 +- bacula/src/dird/job.c | 8 +- bacula/src/dird/migrate.c | 10 +- bacula/src/dird/next_vol.c | 38 +-- bacula/src/dird/protos.h | 17 +- bacula/src/dird/restore.c | 45 ++-- bacula/src/dird/scheduler.c | 24 +- bacula/src/dird/ua.h | 1 - bacula/src/dird/ua_cmds.c | 11 +- bacula/src/dird/ua_output.c | 8 +- bacula/src/dird/ua_prune.c | 233 +++++++++---------- bacula/src/dird/ua_purge.c | 437 ++++++++++++----------------------- bacula/src/dird/ua_restore.c | 6 +- bacula/src/dird/ua_status.c | 3 + bacula/src/dird/verify.c | 10 +- bacula/src/lib/bsnprintf.c | 1 + bacula/src/stored/status.c | 4 +- bacula/technotes-2.1 | 13 ++ 26 files changed, 476 insertions(+), 546 deletions(-) diff --git a/bacula/src/cats/mysql.c b/bacula/src/cats/mysql.c index 982057ba2a..9979157cf2 100644 --- a/bacula/src/cats/mysql.c +++ b/bacula/src/cats/mysql.c @@ -97,7 +97,7 @@ db_init_database(JCR *jcr, const char *db_name, const char *db_user, const char } } Dmsg0(100, "db_open first time\n"); - mdb = (B_DB *) malloc(sizeof(B_DB)); + mdb = (B_DB *)malloc(sizeof(B_DB)); memset(mdb, 0, sizeof(B_DB)); mdb->db_name = bstrdup(db_name); mdb->db_user = bstrdup(db_user); @@ -111,7 +111,7 @@ db_init_database(JCR *jcr, const char *db_name, const char *db_user, const char mdb->db_socket = bstrdup(db_socket); } mdb->db_port = db_port; - mdb->have_insert_id = TRUE; + mdb->have_insert_id = true; mdb->errmsg = get_pool_memory(PM_EMSG); /* get error message buffer */ *mdb->errmsg = 0; mdb->cmd = get_pool_memory(PM_EMSG); /* get command buffer */ @@ -143,7 +143,7 @@ db_open_database(JCR *jcr, B_DB *mdb) V(mutex); return 1; } - mdb->connected = FALSE; + mdb->connected = false; if ((errstat=rwl_init(&mdb->lock)) != 0) { Mmsg1(&mdb->errmsg, _("Unable to initialize DB lock. ERR=%s\n"), @@ -183,9 +183,9 @@ db_open_database(JCR *jcr, B_DB *mdb) mdb->db_password==NULL?"(NULL)":mdb->db_password); if (mdb->db == NULL) { - Mmsg2(&mdb->errmsg, _("Unable to connect to MySQL server. \n" + Mmsg2(&mdb->errmsg, _("Unable to connect to MySQL server.\n" "Database=%s User=%s\n" -"It is probably not running or your password is incorrect.\n"), +"It MySQL server not running or your password is incorrect.\n"), mdb->db_name, mdb->db_user); V(mutex); return 0; @@ -200,7 +200,7 @@ db_open_database(JCR *jcr, B_DB *mdb) my_thread_init(); #endif - mdb->connected = TRUE; + mdb->connected = true; V(mutex); return 1; } diff --git a/bacula/src/cats/protos.h b/bacula/src/cats/protos.h index 4ea426458b..342fa51438 100644 --- a/bacula/src/cats/protos.h +++ b/bacula/src/cats/protos.h @@ -94,7 +94,7 @@ int db_get_num_media_records(JCR *jcr, B_DB *mdb); int db_get_num_pool_records(JCR *jcr, B_DB *mdb); int db_get_pool_ids(JCR *jcr, B_DB *mdb, int *num_ids, DBId_t **ids); int db_get_client_ids(JCR *jcr, B_DB *mdb, int *num_ids, DBId_t **ids); -bool db_get_media_ids(JCR *jcr, B_DB *mdb, DBId_t PoolId, int *num_ids, uint32_t **ids); +bool db_get_media_ids(JCR *jcr, B_DB *mdb, MEDIA_DBR *mr, int *num_ids, uint32_t **ids); int db_get_job_volume_parameters(JCR *jcr, B_DB *mdb, JobId_t JobId, VOL_PARAMS **VolParams); int db_get_client_record(JCR *jcr, B_DB *mdb, CLIENT_DBR *cdbr); int db_get_counter_record(JCR *jcr, B_DB *mdb, COUNTER_DBR *cr); diff --git a/bacula/src/cats/sql_cmds.c b/bacula/src/cats/sql_cmds.c index 0222e1544f..442221748e 100644 --- a/bacula/src/cats/sql_cmds.c +++ b/bacula/src/cats/sql_cmds.c @@ -62,7 +62,11 @@ const char *cnt_DelCand = "SELECT count(*) FROM DelCandidates"; const char *del_Job = "DELETE FROM Job WHERE JobId=%s"; const char *del_JobMedia = "DELETE FROM JobMedia WHERE JobId=%s"; const char *cnt_JobMedia = "SELECT count(*) FROM JobMedia WHERE MediaId=%s"; -const char *sel_JobMedia = "SELECT JobId FROM JobMedia WHERE MediaId=%s"; + +const char *sel_JobMedia = + "SELECT DISTINCT JobMedia.JobId FROM JobMedia,Job " + "WHERE MediaId=%s AND Job.JobId=JobMedia.JobId " + "AND Job.JobTDate<%s"; /* Count Select JobIds for File deletion */ const char *count_select_job = @@ -74,7 +78,7 @@ const char *count_select_job = /* Select JobIds for File deletion. */ const char *select_job = - "SELECT JobId from Job " + "SELECT DISTINCT JobId from Job " "WHERE JobTDate<%s " "AND ClientId=%s " "AND PurgedFiles=0"; diff --git a/bacula/src/cats/sql_find.c b/bacula/src/cats/sql_find.c index 762a7bec2b..81dcfa3021 100644 --- a/bacula/src/cats/sql_find.c +++ b/bacula/src/cats/sql_find.c @@ -316,7 +316,7 @@ db_find_next_volume(JCR *jcr, B_DB *mdb, int item, bool InChanger, MEDIA_DBR *mr edit_int64(mr->PoolId, ed1), mr->MediaType, mr->VolStatus, changer, order, item); } - Dmsg1(100, "fnextvol=%s\n", mdb->cmd); + Dmsg1(050, "fnextvol=%s\n", mdb->cmd); if (!QUERY_DB(jcr, mdb, mdb->cmd)) { db_unlock(mdb); return 0; @@ -324,6 +324,7 @@ db_find_next_volume(JCR *jcr, B_DB *mdb, int item, bool InChanger, MEDIA_DBR *mr numrows = sql_num_rows(mdb); if (item > numrows || item < 1) { + Dmsg2(050, "item=%d got=%d\n", item, numrows); Mmsg2(&mdb->errmsg, _("Request for Volume item %d greater than max %d or less than 1\n"), item, numrows); db_unlock(mdb); @@ -338,6 +339,7 @@ db_find_next_volume(JCR *jcr, B_DB *mdb, int item, bool InChanger, MEDIA_DBR *mr */ while (item-- > 0) { if ((row = sql_fetch_row(mdb)) == NULL) { + Dmsg1(050, "Fail fetch item=%d\n", item+1); Mmsg1(&mdb->errmsg, _("No Volume record found for item %d.\n"), item); sql_free_result(mdb); db_unlock(mdb); @@ -377,6 +379,7 @@ db_find_next_volume(JCR *jcr, B_DB *mdb, int item, bool InChanger, MEDIA_DBR *mr sql_free_result(mdb); db_unlock(mdb); + Dmsg1(050, "Rtn numrows=%d\n", numrows); return numrows; } diff --git a/bacula/src/cats/sql_get.c b/bacula/src/cats/sql_get.c index 7e9ec061ac..62e49cec39 100644 --- a/bacula/src/cats/sql_get.c +++ b/bacula/src/cats/sql_get.c @@ -812,13 +812,13 @@ int db_get_num_media_records(JCR *jcr, B_DB *mdb) /* * This function returns a list of all the Media record ids for - * the current Pool. + * the current Pool with the correct Media Type. * The caller must free ids if non-NULL. * * Returns false: on failure * true: on success */ -bool db_get_media_ids(JCR *jcr, B_DB *mdb, uint32_t PoolId, int *num_ids, uint32_t *ids[]) +bool db_get_media_ids(JCR *jcr, B_DB *mdb, MEDIA_DBR *mr, int *num_ids, uint32_t *ids[]) { SQL_ROW row; int i = 0; @@ -828,8 +828,9 @@ bool db_get_media_ids(JCR *jcr, B_DB *mdb, uint32_t PoolId, int *num_ids, uint32 db_lock(mdb); *ids = NULL; - Mmsg(mdb->cmd, "SELECT MediaId FROM Media WHERE PoolId=%s", - edit_int64(PoolId, ed1)); + Mmsg(mdb->cmd, "SELECT DISTINCT MediaId FROM Media WHERE PoolId=%s " + " AND MediaType='%s'", + edit_int64(mr->PoolId, ed1), mr->MediaType); if (QUERY_DB(jcr, mdb, mdb->cmd)) { *num_ids = sql_num_rows(mdb); if (*num_ids > 0) { diff --git a/bacula/src/dird/autoprune.c b/bacula/src/dird/autoprune.c index 1259c205f9..45cfcabf2c 100644 --- a/bacula/src/dird/autoprune.c +++ b/bacula/src/dird/autoprune.c @@ -82,58 +82,76 @@ void do_autoprune(JCR *jcr) } /* - * Prune all volumes in current Pool. This is called from + * Prune at least on Volume in current Pool. This is called from * catreq.c when the Storage daemon is asking for another * volume and no appendable volumes are available. * - * Return 0: on error - * number of Volumes Purged + * Return: false if nothing pruned + * true if pruned, and mr is set to pruned volume */ -int prune_volumes(JCR *jcr) +bool prune_volumes(JCR *jcr, MEDIA_DBR *mr) { - int stat = 0; + int count; int i; uint32_t *ids = NULL; int num_ids = 0; - MEDIA_DBR mr; + struct del_ctx del; UAContext *ua; + bool ok = false; + Dmsg1(050, "Prune volumes PoolId=%d\n", jcr->jr.PoolId); if (!jcr->job->PruneVolumes && !jcr->pool->AutoPrune) { Dmsg0(100, "AutoPrune not set in Pool.\n"); return 0; } - memset(&mr, 0, sizeof(mr)); + memset(&del, 0, sizeof(del)); + del.max_ids = 1000; + del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids); + ua = new_ua_context(jcr); db_lock(jcr->db); /* Get the List of all media ids in the current Pool */ - if (!db_get_media_ids(jcr, jcr->db, jcr->jr.PoolId, &num_ids, &ids)) { + if (!db_get_media_ids(jcr, jcr->db, mr, &num_ids, &ids)) { Jmsg(jcr, M_ERROR, 0, "%s", db_strerror(jcr->db)); goto bail_out; } /* Visit each Volume and Prune it */ for (i=0; idb, &mr)) { + MEDIA_DBR lmr; + memset(&lmr, 0, sizeof(lmr)); + lmr.MediaId = ids[i]; + Dmsg1(150, "Get record MediaId=%d\n", (int)lmr.MediaId); + if (!db_get_media_record(jcr, jcr->db, &lmr)) { Jmsg(jcr, M_ERROR, 0, "%s", db_strerror(jcr->db)); continue; } /* Prune only Volumes from current Pool */ - if (jcr->jr.PoolId != mr.PoolId) { + if (mr->PoolId != lmr.PoolId) { continue; } /* Don't prune archived volumes */ - if (mr.Enabled == 2) { + if (lmr.Enabled == 2) { continue; } /* Prune only Volumes with status "Full", or "Used" */ - if (strcmp(mr.VolStatus, "Full") == 0 || - strcmp(mr.VolStatus, "Used") == 0) { - Dmsg1(200, "Prune Volume %s\n", mr.VolumeName); - stat += prune_volume(ua, &mr); - Dmsg1(200, "Num pruned = %d\n", stat); + if (strcmp(lmr.VolStatus, "Full") == 0 || + strcmp(lmr.VolStatus, "Used") == 0) { + Dmsg2(050, "Add prune list MediaId=%d Volume %s\n", (int)lmr.MediaId, lmr.VolumeName); + count = get_prune_list_for_volume(ua, &lmr, &del); + Dmsg1(050, "Num pruned = %d\n", count); + if (count != 0) { + purge_job_list_from_catalog(ua, del); + del.num_ids = 0; /* reset count */ + ok = is_volume_purged(ua, &lmr); + if (ok) { + Dmsg2(050, "Vol=%s MediaId=%d purged.\n", lmr.VolumeName, (int)lmr.MediaId); + mr = &lmr; /* struct copy */ + break; + } + } } } @@ -143,5 +161,8 @@ bail_out: if (ids) { free(ids); } - return stat; + if (del.JobId) { + free(del.JobId); + } + return ok; } diff --git a/bacula/src/dird/backup.c b/bacula/src/dird/backup.c index 9ff2db7c6c..c90f7b5141 100644 --- a/bacula/src/dird/backup.c +++ b/bacula/src/dird/backup.c @@ -1,18 +1,3 @@ -/* - * - * Bacula Director -- backup.c -- responsible for doing backup jobs - * - * Kern Sibbald, March MM - * - * Basic tasks done here: - * Open DB and create records for this job. - * Open Message Channel with Storage daemon to tell him a job will be starting. - * Open connection with File daemon and pass him commands - * to do the backup. - * When the File daemon finishes the job, update the DB. - * - * Version $Id$ - */ /* Bacula® - The Network Backup Solution @@ -40,6 +25,21 @@ (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich, Switzerland, email:ftf@fsfeurope.org. */ +/* + * + * Bacula Director -- backup.c -- responsible for doing backup jobs + * + * Kern Sibbald, March MM + * + * Basic tasks done here: + * Open DB and create records for this job. + * Open Message Channel with Storage daemon to tell him a job will be starting. + * Open connection with File daemon and pass him commands + * to do the backup. + * When the File daemon finishes the job, update the DB. + * + * Version $Id$ + */ #include "bacula.h" #include "dird.h" @@ -438,7 +438,7 @@ void backup_cleanup(JCR *jcr, int TermCode) // bmicrosleep(15, 0); /* for debugging SIGHUP */ - Jmsg(jcr, msg_type, 0, _("Bacula %s (%s): %s\n" + Jmsg(jcr, msg_type, 0, _("Bacula %s Version: %s (%s) %s %s %s at %s\n" " JobId: %d\n" " Job: %s\n" " Backup Level: %s%s\n" @@ -468,8 +468,7 @@ void backup_cleanup(JCR *jcr, int TermCode) " FD termination status: %s\n" " SD termination status: %s\n" " Termination: %s\n\n"), - VERSION, - LSMDATE, + my_name, VERSION, BDATE, HOST_OS, DISTNAME, DISTVER, edt, jcr->jr.JobId, jcr->jr.Job, diff --git a/bacula/src/dird/catreq.c b/bacula/src/dird/catreq.c index e884062e6f..443daa6ea0 100644 --- a/bacula/src/dird/catreq.c +++ b/bacula/src/dird/catreq.c @@ -142,7 +142,7 @@ void catalog_request(JCR *jcr, BSOCK *bs) mr.PoolId = pr.PoolId; mr.StorageId = jcr->wstore->StorageId; ok = find_next_volume_for_append(jcr, &mr, index, true /*permit create new vol*/); - Dmsg3(100, "find_media idx=%d ok=%d vol=%s\n", index, ok, mr.VolumeName); + Dmsg3(050, "find_media ok=%d idx=%d vol=%s\n", ok, index, mr.VolumeName); } /* * Send Find Media response to Storage daemon diff --git a/bacula/src/dird/dird.h b/bacula/src/dird/dird.h index 6f95d5c803..0dccce803f 100644 --- a/bacula/src/dird/dird.h +++ b/bacula/src/dird/dird.h @@ -43,8 +43,6 @@ #include "jcr.h" #include "bsr.h" #include "ua.h" -#include "protos.h" - #include "jobq.h" /* Globals that dird.c exports */ @@ -52,6 +50,21 @@ extern DIRRES *director; /* Director resource */ extern int FDConnectTimeout; extern int SDConnectTimeout; -/* From job.c */ -void dird_free_jcr(JCR *jcr); -void dird_free_jcr_pointers(JCR *jcr); +/* Used in ua_prune.c and ua_purge.c */ + +struct s_count_ctx { + int count; +}; + +#define MAX_DEL_LIST_LEN 2000000 + +struct del_ctx { + JobId_t *JobId; /* array of JobIds */ + char *PurgedFiles; /* Array of PurgedFile flags */ + int num_ids; /* ids stored */ + int max_ids; /* size of array */ + int num_del; /* number deleted */ + int tot_ids; /* total to process */ +}; + +#include "protos.h" diff --git a/bacula/src/dird/job.c b/bacula/src/dird/job.c index 91529724df..373033a03a 100644 --- a/bacula/src/dird/job.c +++ b/bacula/src/dird/job.c @@ -122,7 +122,7 @@ bool setup_job(JCR *jcr) /* * Open database */ - Dmsg0(50, "Open database\n"); + Dmsg0(150, "Open database\n"); jcr->db=db_init_database(jcr, jcr->catalog->db_name, jcr->catalog->db_user, jcr->catalog->db_password, jcr->catalog->db_address, jcr->catalog->db_port, jcr->catalog->db_socket, @@ -135,7 +135,7 @@ bool setup_job(JCR *jcr) } goto bail_out; } - Dmsg0(50, "DB opened\n"); + Dmsg0(150, "DB opened\n"); if (!jcr->fname) { jcr->fname = get_pool_memory(PM_FNAME); @@ -537,7 +537,7 @@ DBId_t get_or_create_pool_record(JCR *jcr, char *pool_name) memset(&pr, 0, sizeof(pr)); bstrncpy(pr.Name, pool_name, sizeof(pr.Name)); - Dmsg1(010, "get_or_create_pool=%s\n", pool_name); + Dmsg1(110, "get_or_create_pool=%s\n", pool_name); while (!db_get_pool_record(jcr, jcr->db, &pr)) { /* get by Name */ /* Try to create the pool */ @@ -1069,7 +1069,7 @@ void copy_wstorage(JCR *jcr, alist *storage, const char *where) } jcr->wstorage = New(alist(10, not_owned_by_alist)); foreach_alist(st, storage) { - Dmsg1(50, "storage=%s\n", st->name()); + Dmsg1(100, "wstorage=%s\n", st->name()); jcr->wstorage->append(st); } if (!jcr->wstore_source) { diff --git a/bacula/src/dird/migrate.c b/bacula/src/dird/migrate.c index 33a744488a..abe785e4b4 100644 --- a/bacula/src/dird/migrate.c +++ b/bacula/src/dird/migrate.c @@ -385,8 +385,11 @@ bool do_migration(JCR *jcr) migration_cleanup(jcr, jcr->JobStatus); if (mig_jcr) { + char jobid[50]; UAContext *ua = new_ua_context(jcr); - purge_job_records_from_catalog(ua, jcr->previous_jr.JobId); + edit_uint64(jcr->previous_jr.JobId, jobid); + /* Purge all old file records, but leave Job record */ + purge_files_from_jobs(ua, jobid); free_ua_context(ua); } return true; @@ -1149,7 +1152,7 @@ void migration_cleanup(JCR *jcr, int TermCode) jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg)); - Jmsg(jcr, msg_type, 0, _("Bacula %s (%s): %s\n" + Jmsg(jcr, msg_type, 0, _("Bacula %s Version: %s (%s) %s %s %s at %s\n" " Prev Backup JobId: %s\n" " New Backup JobId: %s\n" " Migration JobId: %s\n" @@ -1175,8 +1178,7 @@ void migration_cleanup(JCR *jcr, int TermCode) " SD Errors: %d\n" " SD termination status: %s\n" " Termination: %s\n\n"), - VERSION, - LSMDATE, + my_name, VERSION, BDATE, HOST_OS, DISTNAME, DISTVER, edt, edit_uint64(jcr->previous_jr.JobId, ec6), mig_jcr ? edit_uint64(mig_jcr->jr.JobId, ec7) : "0", diff --git a/bacula/src/dird/next_vol.c b/bacula/src/dird/next_vol.c index d615162035..0540e080b8 100644 --- a/bacula/src/dird/next_vol.c +++ b/bacula/src/dird/next_vol.c @@ -1,17 +1,7 @@ -/* - * - * Bacula Director -- next_vol -- handles finding the next - * volume for append. Split out of catreq.c August MMIII - * catalog request from the Storage daemon. - - * Kern Sibbald, March MMI - * - * Version $Id$ - */ /* Bacula® - The Network Backup Solution - Copyright (C) 2001-2006 Free Software Foundation Europe e.V. + Copyright (C) 2001-2007 Free Software Foundation Europe e.V. The main author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -35,6 +25,16 @@ (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich, Switzerland, email:ftf@fsfeurope.org. */ +/* + * + * Bacula Director -- next_vol -- handles finding the next + * volume for append. Split out of catreq.c August MMIII + * catalog request from the Storage daemon. + + * Kern Sibbald, March MMI + * + * Version $Id$ + */ #include "bacula.h" #include "dird.h" @@ -48,7 +48,7 @@ static bool get_scratch_volume(JCR *jcr, MEDIA_DBR *mr, bool InChanger); * jcr->wstore * jcr->db * jcr->pool - * MEDIA_DBR mr (zeroed out) + * MEDIA_DBR mr with PoolId set * create -- whether or not to create a new volume */ int find_next_volume_for_append(JCR *jcr, MEDIA_DBR *mr, int index, bool create) @@ -59,7 +59,7 @@ int find_next_volume_for_append(JCR *jcr, MEDIA_DBR *mr, int index, bool create) STORE *store = jcr->wstore; bstrncpy(mr->MediaType, store->media_type, sizeof(mr->MediaType)); - Dmsg2(100, "CatReq FindMedia: PoolId=%d, MediaType=%s\n", (int)mr->PoolId, mr->MediaType); + Dmsg2(150, "find_next_vol_for_append: PoolId=%d, MediaType=%s\n", (int)mr->PoolId, mr->MediaType); /* * If we are using an Autochanger, restrict Volume * search to the Autochanger on the first pass @@ -75,15 +75,15 @@ int find_next_volume_for_append(JCR *jcr, MEDIA_DBR *mr, int index, bool create) * 1. Look for volume with "Append" status. */ ok = db_find_next_volume(jcr, jcr->db, index, InChanger, mr); - Dmsg4(100, "after find_next_vol index=%d ok=%d InChanger=%d Vstat=%s\n", - index, ok, InChanger, mr->VolStatus); if (!ok) { + Dmsg4(050, "after find_next_vol ok=%d index=%d InChanger=%d Vstat=%s\n", + ok, index, InChanger, mr->VolStatus); /* * 2. Try finding a recycled volume */ ok = find_recycled_volume(jcr, InChanger, mr); - Dmsg2(100, "find_recycled_volume ok=%d FW=%d\n", ok, mr->FirstWritten); + Dmsg2(150, "find_recycled_volume ok=%d FW=%d\n", ok, mr->FirstWritten); if (!ok) { /* * 3. Try recycling any purged volume @@ -93,9 +93,12 @@ int find_next_volume_for_append(JCR *jcr, MEDIA_DBR *mr, int index, bool create) /* * 4. Try pruning Volumes */ - prune_volumes(jcr); + Dmsg0(150, "Call prune_volumes\n"); + prune_volumes(jcr, mr); ok = recycle_oldest_purged_volume(jcr, InChanger, mr); if (!ok) { + Dmsg4(050, "after prune volumes_vol ok=%d index=%d InChanger=%d Vstat=%s\n", + ok, index, InChanger, mr->VolStatus); /* * 5. Try pulling a volume from the Scratch pool */ @@ -171,6 +174,7 @@ int find_next_volume_for_append(JCR *jcr, MEDIA_DBR *mr, int index, bool create) break; } /* end for loop */ db_unlock(jcr->db); + Dmsg1(150, "return ok=%d find_next_vol\n", ok); return ok; } diff --git a/bacula/src/dird/protos.h b/bacula/src/dird/protos.h index fc910f84a8..ceec824345 100644 --- a/bacula/src/dird/protos.h +++ b/bacula/src/dird/protos.h @@ -44,7 +44,7 @@ extern int authenticate_user_agent(UAContext *ua); /* autoprune.c */ extern void do_autoprune(JCR *jcr); -extern int prune_volumes(JCR *jcr); +extern bool prune_volumes(JCR *jcr, MEDIA_DBR *mr); /* autorecycle.c */ extern bool recycle_oldest_purged_volume(JCR *jcr, bool InChanger, MEDIA_DBR *mr); @@ -126,6 +126,8 @@ extern void free_rstorage(JCR *jcr); extern bool setup_job(JCR *jcr); extern void create_clones(JCR *jcr); extern bool create_restore_bootstrap_file(JCR *jcr); +extern void dird_free_jcr(JCR *jcr); +extern void dird_free_jcr_pointers(JCR *jcr); /* migration.c */ extern bool do_migration(JCR *jcr); @@ -259,15 +261,20 @@ int insert_tree_handler(void *ctx, int num_fields, char **row); int prune_files(UAContext *ua, CLIENT *client); int prune_jobs(UAContext *ua, CLIENT *client, int JobType); bool prune_volume(UAContext *ua, MEDIA_DBR *mr); +int job_delete_handler(void *ctx, int num_fields, char **row); +int del_count_handler(void *ctx, int num_fields, char **row); +int file_delete_handler(void *ctx, int num_fields, char **row); +int get_prune_list_for_volume(UAContext *ua, MEDIA_DBR *mr, del_ctx *del); /* ua_purge.c */ +bool is_volume_purged(UAContext *ua, MEDIA_DBR *mr); bool mark_media_purged(UAContext *ua, MEDIA_DBR *mr); void purge_files_from_volume(UAContext *ua, MEDIA_DBR *mr ); -int purge_jobs_from_volume(UAContext *ua, MEDIA_DBR *mr); -void purge_files_from_job(UAContext *ua, JobId_t JobId); -void purge_job_from_catalog(UAContext *ua, JobId_t JobId); -void purge_job_records_from_catalog(UAContext *ua, JobId_t JobId); +bool purge_jobs_from_volume(UAContext *ua, MEDIA_DBR *mr); +void purge_files_from_jobs(UAContext *ua, char *jobs); void purge_jobs_from_catalog(UAContext *ua, char *jobs); +void purge_job_list_from_catalog(UAContext *ua, del_ctx &del); +void purge_files_from_job_list(UAContext *ua, del_ctx &del); /* ua_run.c */ diff --git a/bacula/src/dird/restore.c b/bacula/src/dird/restore.c index 6f4057eba0..b150a0d564 100644 --- a/bacula/src/dird/restore.c +++ b/bacula/src/dird/restore.c @@ -1,26 +1,7 @@ -/* - * Bacula Director -- restore.c -- responsible for restoring files - * - * Kern Sibbald, November MM - * - * This routine is run as a separate thread. - * - * Current implementation is Catalog verification only (i.e. no - * verification versus tape). - * - * Basic tasks done here: - * Open DB - * Open Message Channel with Storage daemon to tell him a job will be starting. - * Open connection with File daemon and pass him commands - * to do the restore. - * Update the DB according to what files where restored???? - * - * Version $Id$ - */ /* Bacula® - The Network Backup Solution - Copyright (C) 2000-2006 Free Software Foundation Europe e.V. + Copyright (C) 2000-2007 Free Software Foundation Europe e.V. The main author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -44,6 +25,25 @@ (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich, Switzerland, email:ftf@fsfeurope.org. */ +/* + * Bacula Director -- restore.c -- responsible for restoring files + * + * Kern Sibbald, November MM + * + * This routine is run as a separate thread. + * + * Current implementation is Catalog verification only (i.e. no + * verification versus tape). + * + * Basic tasks done here: + * Open DB + * Open Message Channel with Storage daemon to tell him a job will be starting. + * Open connection with File daemon and pass him commands + * to do the restore. + * Update the DB according to what files where restored???? + * + * Version $Id$ + */ #include "bacula.h" @@ -280,7 +280,7 @@ void restore_cleanup(JCR *jcr, int TermCode) jobstatus_to_ascii(jcr->FDJobStatus, fd_term_msg, sizeof(fd_term_msg)); jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg)); - Jmsg(jcr, msg_type, 0, _("Bacula %s (%s): %s\n" + Jmsg(jcr, msg_type, 0, _("Bacula %s Version: %s (%s) %s %s %s at %s\n" " JobId: %d\n" " Job: %s\n" " Client: %s\n" @@ -294,8 +294,7 @@ void restore_cleanup(JCR *jcr, int TermCode) " FD termination status: %s\n" " SD termination status: %s\n" " Termination: %s\n\n"), - VERSION, - LSMDATE, + my_name, VERSION, BDATE, HOST_OS, DISTNAME, DISTVER, edt, jcr->jr.JobId, jcr->jr.Job, diff --git a/bacula/src/dird/scheduler.c b/bacula/src/dird/scheduler.c index f06e2378a0..35c4178766 100644 --- a/bacula/src/dird/scheduler.c +++ b/bacula/src/dird/scheduler.c @@ -1,18 +1,7 @@ -/* - * - * Bacula scheduler - * It looks at what jobs are to be run and when - * and waits around until it is time to - * fire them up. - * - * Kern Sibbald, May MM, major revision December MMIII - * - * Version $Id$ - */ /* Bacula® - The Network Backup Solution - Copyright (C) 2000-2006 Free Software Foundation Europe e.V. + Copyright (C) 2000-2007 Free Software Foundation Europe e.V. The main author of Bacula is Kern Sibbald, with contributions from many others, a complete list can be found in the file AUTHORS. @@ -36,6 +25,17 @@ (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich, Switzerland, email:ftf@fsfeurope.org. */ +/* + * + * Bacula scheduler + * It looks at what jobs are to be run and when + * and waits around until it is time to + * fire them up. + * + * Kern Sibbald, May MM, major revision December MMIII + * + * Version $Id$ + */ #include "bacula.h" #include "dird.h" diff --git a/bacula/src/dird/ua.h b/bacula/src/dird/ua.h index a4145202b3..aabb2e1f6d 100644 --- a/bacula/src/dird/ua.h +++ b/bacula/src/dird/ua.h @@ -123,5 +123,4 @@ struct RESTORE_CTX { #define MAX_ID_LIST_LEN 2000000 - #endif diff --git a/bacula/src/dird/ua_cmds.c b/bacula/src/dird/ua_cmds.c index b09f3f7edd..db6174e44b 100644 --- a/bacula/src/dird/ua_cmds.c +++ b/bacula/src/dird/ua_cmds.c @@ -1346,15 +1346,13 @@ static void delete_job_id_range(UAContext *ua, char *tok) /* * do_job_delete now performs the actual delete operation atomically */ - static void do_job_delete(UAContext *ua, JobId_t JobId) { - POOL_MEM query(PM_MESSAGE); char ed1[50]; - purge_files_from_job(ua, JobId); - purge_job_from_catalog(ua, JobId); - bsendmsg(ua, _("Job %s and associated records deleted from the catalog.\n"), edit_int64(JobId, ed1)); + edit_int64(JobId, ed1); + purge_jobs_from_catalog(ua, ed1); + bsendmsg(ua, _("Job %s and associated records deleted from the catalog.\n"), ed1); } /* @@ -1690,7 +1688,8 @@ int qhelp_cmd(UAContext *ua, const char *cmd) static int version_cmd(UAContext *ua, const char *cmd) { - bsendmsg(ua, _("%s Version: %s (%s)\n"), my_name, VERSION, BDATE); + bsendmsg(ua, _("%s Version: %s (%s) %s %s %s\n"), my_name, VERSION, BDATE, + HOST_OS, DISTNAME, DISTVER); return 1; } diff --git a/bacula/src/dird/ua_output.c b/bacula/src/dird/ua_output.c index 83c4f23180..72a4ef9ed0 100644 --- a/bacula/src/dird/ua_output.c +++ b/bacula/src/dird/ua_output.c @@ -447,7 +447,6 @@ static bool list_nextvol(UAContext *ua, int ndays) { JOB *job; JCR *jcr = ua->jcr; - POOL *pool; USTORE store; RUN *run; time_t runtime; @@ -471,13 +470,12 @@ static bool list_nextvol(UAContext *ua, int ndays) } } for (run=NULL; (run = find_next_run(run, job, runtime, ndays)); ) { - pool = run->pool ? run->pool : NULL; - if (!complete_jcr_for_job(jcr, job, pool)) { + if (!complete_jcr_for_job(jcr, job, run->pool)) { return false; } memset(&pr, 0, sizeof(pr)); pr.PoolId = jcr->jr.PoolId; - if (! db_get_pool_record(ua->jcr, ua->db, &pr)) { + if (!db_get_pool_record(ua->jcr, ua->db, &pr)) { bstrncpy(pr.Name, "*UnknownPool*", sizeof(pr.Name)); } mr.PoolId = jcr->jr.PoolId; @@ -629,7 +627,7 @@ bool complete_jcr_for_job(JCR *jcr, JOB *job, POOL *pool) } return false; } - bstrncpy(pr.Name, jcr->pool->hdr.name, sizeof(pr.Name)); + bstrncpy(pr.Name, jcr->pool->name(), sizeof(pr.Name)); while (!db_get_pool_record(jcr, jcr->db, &pr)) { /* get by Name */ /* Try to create the pool */ if (create_pool(jcr, jcr->db, jcr->pool, POOL_OP_CREATE) < 0) { diff --git a/bacula/src/dird/ua_prune.c b/bacula/src/dird/ua_prune.c index 9486317ffb..c411106494 100644 --- a/bacula/src/dird/ua_prune.c +++ b/bacula/src/dird/ua_prune.c @@ -42,36 +42,10 @@ /* Forward referenced functions */ - -#define MAX_DEL_LIST_LEN 2000000 - -/* In memory list of JobIds */ -struct s_file_del_ctx { - JobId_t *JobId; - int num_ids; /* ids stored */ - int max_ids; /* size of array */ - int num_del; /* number deleted */ - int tot_ids; /* total to process */ -}; - -struct s_job_del_ctx { - JobId_t *JobId; /* array of JobIds */ - char *PurgedFiles; /* Array of PurgedFile flags */ - int num_ids; /* ids stored */ - int max_ids; /* size of array */ - int num_del; /* number deleted */ - int tot_ids; /* total to process */ -}; - -struct s_count_ctx { - int count; -}; - - /* * Called here to count entries to be deleted */ -static int count_handler(void *ctx, int num_fields, char **row) +int del_count_handler(void *ctx, int num_fields, char **row) { struct s_count_ctx *cnt = (struct s_count_ctx *)ctx; @@ -92,9 +66,9 @@ static int count_handler(void *ctx, int num_fields, char **row) * is allowed to get to MAX_DEL_LIST_LEN to limit the * maximum malloc'ed memory. */ -static int job_delete_handler(void *ctx, int num_fields, char **row) +int job_delete_handler(void *ctx, int num_fields, char **row) { - struct s_job_del_ctx *del = (struct s_job_del_ctx *)ctx; + struct del_ctx *del = (struct del_ctx *)ctx; if (del->num_ids == MAX_DEL_LIST_LEN) { return 1; @@ -105,13 +79,14 @@ static int job_delete_handler(void *ctx, int num_fields, char **row) del->PurgedFiles = (char *)brealloc(del->PurgedFiles, del->max_ids); } del->JobId[del->num_ids] = (JobId_t)str_to_int64(row[0]); +// Dmsg2(60, "row=%d val=%d\n", del->num_ids, del->JobId[del->num_ids]); del->PurgedFiles[del->num_ids++] = (char)str_to_int64(row[1]); return 0; } -static int file_delete_handler(void *ctx, int num_fields, char **row) +int file_delete_handler(void *ctx, int num_fields, char **row) { - struct s_file_del_ctx *del = (struct s_file_del_ctx *)ctx; + struct del_ctx *del = (struct del_ctx *)ctx; if (del->num_ids == MAX_DEL_LIST_LEN) { return 1; @@ -122,6 +97,7 @@ static int file_delete_handler(void *ctx, int num_fields, char **row) del->max_ids); } del->JobId[del->num_ids++] = (JobId_t)str_to_int64(row[0]); +// Dmsg2(150, "row=%d val=%d\n", del->num_ids-1, del->JobId[del->num_ids-1]); return 0; } @@ -205,10 +181,9 @@ int prunecmd(UAContext *ua, const char *cmd) */ int prune_files(UAContext *ua, CLIENT *client) { - struct s_file_del_ctx del; + struct del_ctx del; struct s_count_ctx cnt; - POOLMEM *query = get_pool_memory(PM_MESSAGE); - int i; + POOL_MEM query(PM_MESSAGE); utime_t now, period; CLIENT_DBR cr; char ed1[50], ed2[50]; @@ -228,9 +203,10 @@ int prune_files(UAContext *ua, CLIENT *client) /* Select Jobs -- for counting */ Mmsg(query, count_select_job, edit_uint64(now - period, ed1), edit_int64(cr.ClientId, ed2)); - Dmsg3(050, "select now=%u period=%u sql=%s\n", (uint32_t)now, (uint32_t)period, query); + Dmsg3(050, "select now=%u period=%u sql=%s\n", (uint32_t)now, + (uint32_t)period, query.c_str()); cnt.count = 0; - if (!db_sql_query(ua->db, query, count_handler, (void *)&cnt)) { + if (!db_sql_query(ua->db, query.c_str(), del_count_handler, (void *)&cnt)) { bsendmsg(ua, "%s", db_strerror(ua->db)); Dmsg0(050, "Count failed\n"); goto bail_out; @@ -255,15 +231,10 @@ int prune_files(UAContext *ua, CLIENT *client) /* Now process same set but making a delete list */ Mmsg(query, select_job, edit_uint64(now - period, ed1), edit_int64(cr.ClientId, ed2)); - db_sql_query(ua->db, query, file_delete_handler, (void *)&del); + db_sql_query(ua->db, query.c_str(), file_delete_handler, (void *)&del); + + purge_files_from_job_list(ua, del); - for (i=0; i < del.num_ids; i++) { - /* Don't prune current job */ - if (ua->jcr->JobId != del.JobId[i]) { - purge_files_from_job(ua, del.JobId[i]); - del.num_del++; - } - } edit_uint64_with_commas(del.num_del, ed1); bsendmsg(ua, _("Pruned Files from %s Jobs for client %s from catalog.\n"), ed1, client->name()); @@ -273,7 +244,6 @@ bail_out: if (del.JobId) { free(del.JobId); } - free_pool_memory(query); return 1; } @@ -316,10 +286,8 @@ static bool create_temp_tables(UAContext *ua) */ int prune_jobs(UAContext *ua, CLIENT *client, int JobType) { - struct s_job_del_ctx del; - struct s_count_ctx cnt; - POOLMEM *query = get_pool_memory(PM_MESSAGE); - int i; + struct del_ctx del; + POOL_MEM query(PM_MESSAGE); utime_t now, period; CLIENT_DBR cr; char ed1[50], ed2[50]; @@ -327,6 +295,7 @@ int prune_jobs(UAContext *ua, CLIENT *client, int JobType) db_lock(ua->db); memset(&cr, 0, sizeof(cr)); memset(&del, 0, sizeof(del)); + bstrncpy(cr.Name, client->hdr.name, sizeof(cr.Name)); if (!db_create_client_record(ua->jcr, ua->db, &cr)) { db_unlock(ua->db); @@ -351,7 +320,7 @@ int prune_jobs(UAContext *ua, CLIENT *client, int JobType) edit_uint64(now - period, ed1); Mmsg(query, insert_delcand, (char)JobType, ed1, edit_int64(cr.ClientId, ed2)); - if (!db_sql_query(ua->db, query, NULL, (void *)NULL)) { + if (!db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL)) { if (ua->verbose) { bsendmsg(ua, "%s", db_strerror(ua->db)); } @@ -359,28 +328,7 @@ int prune_jobs(UAContext *ua, CLIENT *client, int JobType) goto bail_out; } - /* Count Files to be deleted */ - pm_strcpy(query, cnt_DelCand); - Dmsg1(100, "select sql=%s\n", query); - cnt.count = 0; - if (!db_sql_query(ua->db, query, count_handler, (void *)&cnt)) { - bsendmsg(ua, "%s", db_strerror(ua->db)); - Dmsg0(050, "Count failed\n"); - goto bail_out; - } - - if (cnt.count == 0) { - if (ua->verbose) { - bsendmsg(ua, _("No Jobs found to prune.\n")); - } - goto bail_out; - } - - if (cnt.count < MAX_DEL_LIST_LEN) { - del.max_ids = cnt.count + 1; - } else { - del.max_ids = MAX_DEL_LIST_LEN; - } + del.max_ids = 100; del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids); del.PurgedFiles = (char *)malloc(del.max_ids); @@ -403,29 +351,20 @@ int prune_jobs(UAContext *ua, CLIENT *client, int JobType) Mmsg(query, select_migrate_del, ed1, ed2); break; } - if (!db_sql_query(ua->db, query, job_delete_handler, (void *)&del)) { + + Dmsg1(150, "Query=%s\n", query.c_str()); + if (!db_sql_query(ua->db, query.c_str(), job_delete_handler, (void *)&del)) { bsendmsg(ua, "%s", db_strerror(ua->db)); } - /* - * OK, now we have the list of JobId's to be pruned, send them - * off to be deleted batched 1000 at a time. - */ + purge_job_list_from_catalog(ua, del); - for (i=0; del.num_ids; ) { - for (int j=0; j<1000 && del.num_ids; j++) { - del.num_ids--; - if (ua->jcr->JobId == del.JobId[i]) { - continue; - } - pm_strcat(query, ","); - pm_strcpy(query, edit_int64(del.JobId[i++], ed1)); - del.num_del++; - } - purge_jobs_from_catalog(ua, query); - } - bsendmsg(ua, _("Pruned %d %s for client %s from catalog.\n"), del.num_del, - del.num_del==1?_("Job"):_("Jobs"), client->name()); + if (del.num_del > 0) { + bsendmsg(ua, _("Pruned %d %s for client %s from catalog.\n"), del.num_del, + del.num_del==1?_("Job"):_("Jobs"), client->name()); + } else if (ua->verbose) { + bsendmsg(ua, _("No Jobs found to prune.\n")); + } bail_out: drop_temp_tables(ua); @@ -436,7 +375,6 @@ bail_out: if (del.PurgedFiles) { free(del.PurgedFiles); } - free_pool_memory(query); return 1; } @@ -445,14 +383,14 @@ bail_out: */ bool prune_volume(UAContext *ua, MEDIA_DBR *mr) { - POOLMEM *query = get_pool_memory(PM_MESSAGE); + POOL_MEM query(PM_MESSAGE); struct s_count_ctx cnt; - struct s_file_del_ctx del; + struct del_ctx del; int i; bool ok = false; JOB_DBR jr; utime_t now, period; - char ed1[50]; + char ed1[50], ed2[50]; if (mr->Enabled == 2) { return false; /* Cannot prune archived volumes */ @@ -468,7 +406,8 @@ bool prune_volume(UAContext *ua, MEDIA_DBR *mr) */ cnt.count = 0; Mmsg(query, cnt_JobMedia, edit_int64(mr->MediaId, ed1)); - if (!db_sql_query(ua->db, query, count_handler, (void *)&cnt)) { + Dmsg1(150, "Query=%s\n", query.c_str()); + if (!db_sql_query(ua->db, query.c_str(), del_count_handler, (void *)&cnt)) { bsendmsg(ua, "%s", db_strerror(ua->db)); Dmsg0(050, "Count failed\n"); goto bail_out; @@ -499,11 +438,19 @@ bool prune_volume(UAContext *ua, MEDIA_DBR *mr) /* * Now get a list of JobIds for Jobs written to this Volume - * Could optimize here by adding JobTDate > (now - period). */ del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids); - Mmsg(query, sel_JobMedia, edit_int64(mr->MediaId, ed1)); - if (!db_sql_query(ua->db, query, file_delete_handler, (void *)&del)) { + + /* Use Volume Retention to prune Jobs and their Files */ + period = mr->VolRetention; + now = (utime_t)time(NULL); + Mmsg(query, sel_JobMedia, edit_int64(mr->MediaId, ed1), + edit_uint64(now-period, ed2)); + Dmsg3(250, "Now=%d period=%d now-period=%d\n", (int)now, (int)period, + (int)(now-period)); + + Dmsg1(150, "Query=%s\n", query.c_str()); + if (!db_sql_query(ua->db, query.c_str(), file_delete_handler, (void *)&del)) { if (ua->verbose) { bsendmsg(ua, "%s", db_strerror(ua->db)); } @@ -511,29 +458,24 @@ bool prune_volume(UAContext *ua, MEDIA_DBR *mr) goto bail_out; } - /* Use Volume Retention to prune Jobs and their Files */ - period = mr->VolRetention; - now = (utime_t)time(NULL); - - Dmsg3(200, "Now=%d period=%d now-period=%d\n", (int)now, (int)period, - (int)(now-period)); + cnt.count = 0; for (i=0; i < del.num_ids; i++) { - jr.JobId = del.JobId[i]; - if (!db_get_job_record(ua->jcr, ua->db, &jr)) { + if (ua->jcr->JobId == del.JobId[i]) { + Dmsg2(250, "skip same job JobId[%d]=%d\n", i, (int)del.JobId[i]); + del.JobId[i] = 0; continue; } - Dmsg2(200, "Looking at %s JobTdate=%d\n", jr.Job, (int)jr.JobTDate); - if (jr.JobTDate >= (now - period) || ua->jcr->JobId == del.JobId[i]) { - continue; - } - purge_files_from_job(ua, del.JobId[i]); - purge_job_from_catalog(ua, del.JobId[i]); - del.num_del++; + Dmsg2(250, "accept JobId[%d]=%d\n", i, (int)del.JobId[i]); + cnt.count++; } - if (del.JobId) { - free(del.JobId); + if (cnt.count != 0) { + purge_job_list_from_catalog(ua, del); + } else { + Dmsg0(050, "No jobs to prune.\n"); + goto bail_out; } + if (ua->verbose && del.num_del != 0) { bsendmsg(ua, _("Pruned %d %s on Volume \"%s\" from catalog.\n"), del.num_del, del.num_del == 1 ? "Job" : "Jobs", mr->VolumeName); @@ -545,7 +487,8 @@ bool prune_volume(UAContext *ua, MEDIA_DBR *mr) */ cnt.count = 0; Mmsg(query, cnt_JobMedia, edit_int64(mr->MediaId, ed1)); - if (!db_sql_query(ua->db, query, count_handler, (void *)&cnt)) { + Dmsg1(150, "Query=%s\n", query.c_str()); + if (!db_sql_query(ua->db, query.c_str(), del_count_handler, (void *)&cnt)) { bsendmsg(ua, "%s", db_strerror(ua->db)); Dmsg0(050, "Count failed\n"); goto bail_out; @@ -557,6 +500,60 @@ bool prune_volume(UAContext *ua, MEDIA_DBR *mr) bail_out: db_unlock(ua->db); - free_pool_memory(query); + if (del.JobId) { + free(del.JobId); + } return ok; } + +/* + * Get prune list for a volume + */ +int get_prune_list_for_volume(UAContext *ua, MEDIA_DBR *mr, del_ctx *del) +{ + POOL_MEM query(PM_MESSAGE); + int count = 0; + int i; + utime_t now, period; + char ed1[50], ed2[50]; + + if (mr->Enabled == 2) { + return 0; /* cannot prune Archived volumes */ + } + + db_lock(ua->db); + + /* + * Now add to the list of JobIds for Jobs written to this Volume + */ + edit_int64(mr->MediaId, ed1); + period = mr->VolRetention; + now = (utime_t)time(NULL); + edit_uint64(now-period, ed2); + Mmsg(query, sel_JobMedia, ed1, ed2); + Dmsg3(250, "Now=%d period=%d now-period=%d\n", (int)now, (int)period, + (int)(now-period)); + + Dmsg1(050, "Query=%s\n", query.c_str()); + if (!db_sql_query(ua->db, query.c_str(), file_delete_handler, (void *)del)) { + if (ua->verbose) { + bsendmsg(ua, "%s", db_strerror(ua->db)); + } + Dmsg0(050, "Count failed\n"); + goto bail_out; + } + + for (i=0; i < del->num_ids; i++) { + if (ua->jcr->JobId == del->JobId[i]) { + Dmsg2(150, "skip same job JobId[%d]=%d\n", i, (int)del->JobId[i]); + del->JobId[i] = 0; + continue; + } + Dmsg2(150, "accept JobId[%d]=%d\n", i, (int)del->JobId[i]); + count++; + } + +bail_out: + db_unlock(ua->db); + return count; +} diff --git a/bacula/src/dird/ua_purge.c b/bacula/src/dird/ua_purge.c index b909fe32f0..c08fe06c55 100644 --- a/bacula/src/dird/ua_purge.c +++ b/bacula/src/dird/ua_purge.c @@ -1,15 +1,3 @@ -/* - * - * Bacula Director -- User Agent Database Purge Command - * - * Purges Files from specific JobIds - * or - * Purges Jobs from Volumes - * - * Kern Sibbald, February MMII - * - * Version $Id$ - */ /* Bacula® - The Network Backup Solution @@ -37,6 +25,18 @@ (FSFE), Fiduciary Program, Sumatrastrasse 25, 8006 Zürich, Switzerland, email:ftf@fsfeurope.org. */ +/* + * + * Bacula Director -- User Agent Database Purge Command + * + * Purges Files from specific JobIds + * or + * Purges Jobs from Volumes + * + * Kern Sibbald, February MMII + * + * Version $Id$ + */ #include "bacula.h" #include "dird.h" @@ -45,8 +45,6 @@ static int purge_files_from_client(UAContext *ua, CLIENT *client); static int purge_jobs_from_client(UAContext *ua, CLIENT *client); -#define MAX_DEL_LIST_LEN 1000000 - static const char *select_jobsfiles_from_client = "SELECT JobId FROM Job " "WHERE ClientId=%s " @@ -56,104 +54,6 @@ static const char *select_jobs_from_client = "SELECT JobId, PurgedFiles FROM Job " "WHERE ClientId=%s"; - -/* In memory list of JobIds */ -struct s_file_del_ctx { - JobId_t *JobId; - int num_ids; /* ids stored */ - int max_ids; /* size of array */ - int num_del; /* number deleted */ - int tot_ids; /* total to process */ -}; - -struct s_job_del_ctx { - JobId_t *JobId; /* array of JobIds */ - char *PurgedFiles; /* Array of PurgedFile flags */ - int num_ids; /* ids stored */ - int max_ids; /* size of array */ - int num_del; /* number deleted */ - int tot_ids; /* total to process */ -}; - -struct s_count_ctx { - int count; -}; - -/* - * Called here to count entries to be deleted - */ -static int count_handler(void *ctx, int num_fields, char **row) -{ - struct s_count_ctx *cnt = (struct s_count_ctx *)ctx; - - if (row[0]) { - cnt->count = str_to_int64(row[0]); - } else { - cnt->count = 0; - } - return 0; -} - -/* - * Called here to count entries to be deleted - */ -static int file_count_handler(void *ctx, int num_fields, char **row) -{ - struct s_file_del_ctx *del = (struct s_file_del_ctx *)ctx; - del->tot_ids++; - return 0; -} - - -static int job_count_handler(void *ctx, int num_fields, char **row) -{ - struct s_job_del_ctx *del = (struct s_job_del_ctx *)ctx; - del->tot_ids++; - return 0; -} - - -/* - * Called here to make in memory list of JobIds to be - * deleted and the associated PurgedFiles flag. - * The in memory list will then be transversed - * to issue the SQL DELETE commands. Note, the list - * is allowed to get to MAX_DEL_LIST_LEN to limit the - * maximum malloc'ed memory. - */ -static int job_delete_handler(void *ctx, int num_fields, char **row) -{ - struct s_job_del_ctx *del = (struct s_job_del_ctx *)ctx; - - if (del->num_ids == MAX_DEL_LIST_LEN) { - return 1; - } - if (del->num_ids == del->max_ids) { - del->max_ids = (del->max_ids * 3) / 2; - del->JobId = (JobId_t *)brealloc(del->JobId, sizeof(JobId_t) * del->max_ids); - del->PurgedFiles = (char *)brealloc(del->PurgedFiles, del->max_ids); - } - del->JobId[del->num_ids] = (JobId_t)str_to_int64(row[0]); - del->PurgedFiles[del->num_ids++] = (char)str_to_int64(row[1]); - return 0; -} - -static int file_delete_handler(void *ctx, int num_fields, char **row) -{ - struct s_file_del_ctx *del = (struct s_file_del_ctx *)ctx; - - if (del->num_ids == MAX_DEL_LIST_LEN) { - return 1; - } - if (del->num_ids == del->max_ids) { - del->max_ids = (del->max_ids * 3) / 2; - del->JobId = (JobId_t *)brealloc(del->JobId, sizeof(JobId_t) * - del->max_ids); - } - del->JobId[del->num_ids++] = (JobId_t)str_to_int64(row[0]); - return 0; -} - /* * Purge records from database * @@ -204,7 +104,9 @@ int purgecmd(UAContext *ua, const char *cmd) case 0: /* Job */ case 1: /* JobId */ if (get_job_dbr(ua, &jr)) { - purge_files_from_job(ua, jr.JobId); + char jobid[50]; + edit_int64(jr.JobId, jobid); + purge_files_from_jobs(ua, jobid); } return 1; case 2: /* client */ @@ -279,58 +181,40 @@ int purgecmd(UAContext *ua, const char *cmd) */ static int purge_files_from_client(UAContext *ua, CLIENT *client) { - struct s_file_del_ctx del; - POOLMEM *query = get_pool_memory(PM_MESSAGE); - int i; + struct del_ctx del; + POOL_MEM query(PM_MESSAGE); CLIENT_DBR cr; char ed1[50]; memset(&cr, 0, sizeof(cr)); - memset(&del, 0, sizeof(del)); - bstrncpy(cr.Name, client->name(), sizeof(cr.Name)); if (!db_create_client_record(ua->jcr, ua->db, &cr)) { return 0; } + + memset(&del, 0, sizeof(del)); + del.max_ids = 1000; + del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids); + bsendmsg(ua, _("Begin purging files for Client \"%s\"\n"), cr.Name); - Mmsg(query, select_jobsfiles_from_client, edit_int64(cr.ClientId, ed1)); - Dmsg1(050, "select sql=%s\n", query); + Mmsg(query, select_jobsfiles_from_client, edit_int64(cr.ClientId, ed1)); + Dmsg1(050, "select sql=%s\n", query.c_str()); + db_sql_query(ua->db, query.c_str(), file_delete_handler, (void *)&del); - if (!db_sql_query(ua->db, query, file_count_handler, (void *)&del)) { - bsendmsg(ua, "%s", db_strerror(ua->db)); - Dmsg0(050, "Count failed\n"); - goto bail_out; - } + purge_files_from_job_list(ua, del); - if (del.tot_ids == 0) { + if (del.num_ids == 0) { bsendmsg(ua, _("No Files found for client %s to purge from %s catalog.\n"), client->name(), client->catalog->name()); - goto bail_out; - } - - if (del.tot_ids < MAX_DEL_LIST_LEN) { - del.max_ids = del.tot_ids + 1; } else { - del.max_ids = MAX_DEL_LIST_LEN; - } - del.tot_ids = 0; - - del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids); - - db_sql_query(ua->db, query, file_delete_handler, (void *)&del); - - for (i=0; i < del.num_ids; i++) { - purge_files_from_job(ua, del.JobId[i]); + bsendmsg(ua, _("Files for %d Jobs for client \"%s\" purged from %s catalog.\n"), del.num_ids, + client->name(), client->catalog->name()); } - bsendmsg(ua, _("%d Files for client \"%s\" purged from %s catalog.\n"), del.num_ids, - client->name(), client->catalog->name()); -bail_out: if (del.JobId) { free(del.JobId); } - free_pool_memory(query); return 1; } @@ -345,143 +229,143 @@ bail_out: */ static int purge_jobs_from_client(UAContext *ua, CLIENT *client) { - struct s_job_del_ctx del; - POOLMEM *query = get_pool_memory(PM_MESSAGE); - int i; + struct del_ctx del; + POOL_MEM query(PM_MESSAGE); CLIENT_DBR cr; char ed1[50]; memset(&cr, 0, sizeof(cr)); - memset(&del, 0, sizeof(del)); bstrncpy(cr.Name, client->name(), sizeof(cr.Name)); if (!db_create_client_record(ua->jcr, ua->db, &cr)) { return 0; } + memset(&del, 0, sizeof(del)); + del.max_ids = 1000; + del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids); + del.PurgedFiles = (char *)malloc(del.max_ids); + bsendmsg(ua, _("Begin purging jobs from Client \"%s\"\n"), cr.Name); + Mmsg(query, select_jobs_from_client, edit_int64(cr.ClientId, ed1)); + Dmsg1(150, "select sql=%s\n", query.c_str()); + db_sql_query(ua->db, query.c_str(), job_delete_handler, (void *)&del); - Dmsg1(050, "select sql=%s\n", query); + purge_job_list_from_catalog(ua, del); - if (!db_sql_query(ua->db, query, job_count_handler, (void *)&del)) { - bsendmsg(ua, "%s", db_strerror(ua->db)); - Dmsg0(050, "Count failed\n"); - goto bail_out; - } - if (del.tot_ids == 0) { - bsendmsg(ua, _("No Jobs found for client %s to purge from %s catalog.\n"), + if (del.num_ids == 0) { + bsendmsg(ua, _("No Files found for client %s to purge from %s catalog.\n"), client->name(), client->catalog->name()); - goto bail_out; - } - - if (del.tot_ids < MAX_DEL_LIST_LEN) { - del.max_ids = del.tot_ids + 1; } else { - del.max_ids = MAX_DEL_LIST_LEN; - } - - del.tot_ids = 0; - - del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids); - del.PurgedFiles = (char *)malloc(del.max_ids); - - db_sql_query(ua->db, query, job_delete_handler, (void *)&del); - - /* - * OK, now we have the list of JobId's to be purged, first check - * if the Files have been purged, if not, purge (delete) them. - * Then delete the Job entry, and finally and JobMedia records. - */ - for (i=0; i < del.num_ids; i++) { - Dmsg1(050, "Delete Files JobId=%s\n", ed1); - if (!del.PurgedFiles[i]) { - purge_files_from_job(ua, del.JobId[i]); - } - purge_job_from_catalog(ua, del.JobId[i]); + bsendmsg(ua, _("%d Jobs for client %s purged from %s catalog.\n"), del.num_ids, + client->name(), client->catalog->name()); } - bsendmsg(ua, _("%d Jobs for client %s purged from %s catalog.\n"), del.num_ids, - client->name(), client->catalog->name()); -bail_out: if (del.JobId) { free(del.JobId); } if (del.PurgedFiles) { free(del.PurgedFiles); } - free_pool_memory(query); return 1; } -void purge_job_from_catalog(UAContext *ua, JobId_t JobId) + +/* + * Remove File records from a list of JobIds + */ +void purge_files_from_jobs(UAContext *ua, char *jobs) { POOL_MEM query(PM_MESSAGE); - char ed1[50]; - /* Delete (or purge) records associated with the job */ - purge_job_records_from_catalog(ua, JobId); + Mmsg(query, "DELETE FROM File WHERE JobId IN (%s)", jobs); + db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL); + Dmsg1(050, "Delete File sql=%s\n", query.c_str()); - /* Now remove the Job record itself */ - edit_int64(JobId, ed1); - Mmsg(query, "DELETE FROM Job WHERE JobId=%s", ed1); + /* + * Now mark Job as having files purged. This is necessary to + * avoid having too many Jobs to process in future prunings. If + * we don't do this, the number of JobId's in our in memory list + * could grow very large. + */ + Mmsg(query, "UPDATE Job SET PurgedFiles=1 WHERE JobId IN (%s)", jobs); db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL); - Dmsg1(050, "Delete Job sql=%s\n", query.c_str()); + Dmsg1(050, "Mark purged sql=%s\n", query.c_str()); } /* - * This removes all the records associated with a Job from - * the catalog (i.e. prunes it) without removing the Job - * record itself. + * Delete jobs (all records) from the catalog in groups of 1000 + * at a time. */ -void purge_job_records_from_catalog(UAContext *ua, JobId_t JobId) +void purge_job_list_from_catalog(UAContext *ua, del_ctx &del) { - POOL_MEM query(PM_MESSAGE); + POOL_MEM jobids(PM_MESSAGE); char ed1[50]; - purge_files_from_job(ua, JobId); - - edit_int64(JobId, ed1); - Mmsg(query, "DELETE FROM JobMedia WHERE JobId=%s", ed1); - db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL); - Dmsg1(050, "Delete JobMedia sql=%s\n", query.c_str()); - - Mmsg(query, "DELETE FROM Log WHERE JobId=%s", ed1); - db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL); - Dmsg1(050, "Delete Log sql=%s\n", query.c_str()); - + for (int i=0; del.num_ids; ) { + Dmsg1(150, "num_ids=%d\n", del.num_ids); + pm_strcat(jobids, ""); + for (int j=0; j<1000 && del.num_ids>0; j++) { + del.num_ids--; + if (del.JobId[i] == 0 || ua->jcr->JobId == del.JobId[i]) { + Dmsg2(150, "skip JobId[%d]=%d\n", i, (int)del.JobId[i]); + i++; + continue; + } + if (*jobids.c_str() != 0) { + pm_strcat(jobids, ","); + } + pm_strcat(jobids, edit_int64(del.JobId[i++], ed1)); + Dmsg1(150, "Add id=%s\n", ed1); + del.num_del++; + } + Dmsg1(150, "num_ids=%d\n", del.num_ids); + purge_jobs_from_catalog(ua, jobids.c_str()); + } } /* - * Remove File records for a particular Job. + * Delete files from a list of jobs in groups of 1000 + * at a time. */ -void purge_files_from_job(UAContext *ua, JobId_t JobId) +void purge_files_from_job_list(UAContext *ua, del_ctx &del) { - POOL_MEM query(PM_MESSAGE); + POOL_MEM jobids(PM_MESSAGE); char ed1[50]; - - edit_int64(JobId, ed1); - Mmsg(query, del_File, ed1); - db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL); - /* - * Now mark Job as having files purged. This is necessary to - * avoid having too many Jobs to process in future prunings. If - * we don't do this, the number of JobId's in our in memory list - * could grow very large. + * OK, now we have the list of JobId's to be pruned, send them + * off to be deleted batched 1000 at a time. */ - Mmsg(query, upd_Purged, ed1); - db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL); + for (int i=0; del.num_ids; ) { + pm_strcat(jobids, ""); + for (int j=0; j<1000 && del.num_ids>0; j++) { + del.num_ids--; + if (del.JobId[i] == 0 || ua->jcr->JobId == del.JobId[i]) { + Dmsg2(150, "skip JobId[%d]=%d\n", i, (int)del.JobId[i]); + i++; + continue; + } + if (*jobids.c_str() != 0) { + pm_strcat(jobids, ","); + } + pm_strcat(jobids, edit_int64(del.JobId[i++], ed1)); + Dmsg1(150, "Add id=%s\n", ed1); + del.num_del++; + } + purge_files_from_jobs(ua, jobids.c_str()); + } } +/* + * Remove all records from catalog for a list of JobIds + */ void purge_jobs_from_catalog(UAContext *ua, char *jobs) { POOL_MEM query(PM_MESSAGE); /* Delete (or purge) records associated with the job */ - Mmsg(query, "DELETE FROM File WHERE JobId IN (%s)", jobs); - db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL); - Dmsg1(050, "Delete File sql=%s\n", query.c_str()); + purge_files_from_jobs(ua, jobs); Mmsg(query, "DELETE FROM JobMedia WHERE JobId IN (%s)", jobs); db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL); @@ -495,15 +379,6 @@ void purge_jobs_from_catalog(UAContext *ua, char *jobs) Mmsg(query, "DELETE FROM Job WHERE JobId IN (%s)", jobs); db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL); Dmsg1(050, "Delete Job sql=%s\n", query.c_str()); - - /* - * Now mark Job as having files purged. This is necessary to - * avoid having too many Jobs to process in future prunings. If - * we don't do this, the number of JobId's in our in memory list - * could grow very large. - */ - Mmsg(query, "UPDATE Job SET PurgedFiles=1 WHERE JobId IN (%s)", jobs); - db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL); } @@ -514,12 +389,13 @@ void purge_files_from_volume(UAContext *ua, MEDIA_DBR *mr ) * Returns: 1 if Volume purged * 0 if Volume not purged */ -int purge_jobs_from_volume(UAContext *ua, MEDIA_DBR *mr) +bool purge_jobs_from_volume(UAContext *ua, MEDIA_DBR *mr) { - POOLMEM *query = get_pool_memory(PM_MESSAGE); - struct s_count_ctx cnt; - struct s_file_del_ctx del; - int i, stat = 0; + POOL_MEM query(PM_MESSAGE); + struct del_ctx del; + int i; + bool purged = false; + bool stat; JOB_DBR jr; char ed1[50]; @@ -537,70 +413,66 @@ int purge_jobs_from_volume(UAContext *ua, MEDIA_DBR *mr) memset(&jr, 0, sizeof(jr)); memset(&del, 0, sizeof(del)); - cnt.count = 0; - Mmsg(query, "SELECT count(*) FROM JobMedia WHERE MediaId=%s", - edit_int64(mr->MediaId, ed1)); - if (!db_sql_query(ua->db, query, count_handler, (void *)&cnt)) { - bsendmsg(ua, "%s", db_strerror(ua->db)); - Dmsg0(050, "Count failed\n"); - goto bail_out; - } - - if (cnt.count == 0) { - bsendmsg(ua, _("There are no Jobs associated with Volume \"%s\". Marking it purged.\n"), - mr->VolumeName); - if (!mark_media_purged(ua, mr)) { - bsendmsg(ua, "%s", db_strerror(ua->db)); - goto bail_out; - } - goto bail_out; - } - - if (cnt.count < MAX_DEL_LIST_LEN) { - del.max_ids = cnt.count + 1; - } else { - del.max_ids = MAX_DEL_LIST_LEN; - } + del.max_ids = 1000; + del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids); /* * Check if he wants to purge a single jobid */ i = find_arg_with_value(ua, "jobid"); if (i >= 0) { - del.JobId = (JobId_t *)malloc(sizeof(JobId_t)); del.num_ids = 1; del.JobId[0] = str_to_int64(ua->argv[i]); } else { /* * Purge ALL JobIds */ - del.JobId = (JobId_t *)malloc(sizeof(JobId_t) * del.max_ids); - - Mmsg(query, "SELECT JobId FROM JobMedia WHERE MediaId=%s", + Mmsg(query, "SELECT DISTINCT JobId FROM JobMedia WHERE MediaId=%s", edit_int64(mr->MediaId, ed1)); - if (!db_sql_query(ua->db, query, file_delete_handler, (void *)&del)) { + if (!db_sql_query(ua->db, query.c_str(), file_delete_handler, (void *)&del)) { bsendmsg(ua, "%s", db_strerror(ua->db)); Dmsg0(050, "Count failed\n"); goto bail_out; } } - for (i=0; i < del.num_ids; i++) { - purge_files_from_job(ua, del.JobId[i]); - purge_job_from_catalog(ua, del.JobId[i]); - del.num_del++; - } + purge_job_list_from_catalog(ua, del); + + bsendmsg(ua, _("%d File%s on Volume \"%s\" purged from catalog.\n"), del.num_del, + del.num_del==1?"":"s", mr->VolumeName); + + purged = is_volume_purged(ua, mr); + +bail_out: if (del.JobId) { free(del.JobId); } - bsendmsg(ua, _("%d File%s on Volume \"%s\" purged from catalog.\n"), del.num_del, - del.num_del==1?"":"s", mr->VolumeName); + return purged; +} +/* + * This routine will check the JobMedia records to see if the + * Volume has been purged. If so, it marks it as such and + * + * Returns: true if volume purged + * false if not + */ +bool is_volume_purged(UAContext *ua, MEDIA_DBR *mr) +{ + POOL_MEM query(PM_MESSAGE); + struct s_count_ctx cnt; + bool purged = false; + char ed1[50]; + + if (strcmp(mr->VolStatus, "Purged") == 0) { + purged = true; + goto bail_out; + } /* If purged, mark it so */ cnt.count = 0; Mmsg(query, "SELECT count(*) FROM JobMedia WHERE MediaId=%s", edit_int64(mr->MediaId, ed1)); - if (!db_sql_query(ua->db, query, count_handler, (void *)&cnt)) { + if (!db_sql_query(ua->db, query.c_str(), del_count_handler, (void *)&cnt)) { bsendmsg(ua, "%s", db_strerror(ua->db)); Dmsg0(050, "Count failed\n"); goto bail_out; @@ -609,15 +481,12 @@ int purge_jobs_from_volume(UAContext *ua, MEDIA_DBR *mr) if (cnt.count == 0) { bsendmsg(ua, _("There are no more Jobs associated with Volume \"%s\". Marking it purged.\n"), mr->VolumeName); - if (!(stat = mark_media_purged(ua, mr))) { + if (!(purged = mark_media_purged(ua, mr))) { bsendmsg(ua, "%s", db_strerror(ua->db)); - goto bail_out; } } - bail_out: - free_pool_memory(query); - return stat; + return purged; } /* diff --git a/bacula/src/dird/ua_restore.c b/bacula/src/dird/ua_restore.c index 6cde0eb41b..aa78db7d5a 100644 --- a/bacula/src/dird/ua_restore.c +++ b/bacula/src/dird/ua_restore.c @@ -68,7 +68,7 @@ static bool insert_dir_into_findex_list(UAContext *ua, RESTORE_CTX *rx, char *di static void insert_one_file_or_dir(UAContext *ua, RESTORE_CTX *rx, char *date, bool dir); static int get_client_name(UAContext *ua, RESTORE_CTX *rx); static int get_date(UAContext *ua, char *date, int date_len); -static int count_handler(void *ctx, int num_fields, char **row); +static int restore_count_handler(void *ctx, int num_fields, char **row); static bool insert_table_into_findex_list(UAContext *ua, RESTORE_CTX *rx, char *table); /* @@ -901,7 +901,7 @@ static bool build_directory_tree(UAContext *ua, RESTORE_CTX *rx) if (get_next_jobid_from_list(&p, &JobId) > 0) { /* Use first JobId as estimate of the number of files to restore */ Mmsg(rx->query, uar_count_files, edit_int64(JobId, ed1)); - if (!db_sql_query(ua->db, rx->query, count_handler, (void *)rx)) { + if (!db_sql_query(ua->db, rx->query, restore_count_handler, (void *)rx)) { ua->error_msg("%s\n", db_strerror(ua->db)); } if (rx->found) { @@ -1180,7 +1180,7 @@ int get_next_jobid_from_list(char **p, JobId_t *JobId) return 1; } -static int count_handler(void *ctx, int num_fields, char **row) +static int restore_count_handler(void *ctx, int num_fields, char **row) { RESTORE_CTX *rx = (RESTORE_CTX *)ctx; rx->JobId = str_to_int64(row[0]); diff --git a/bacula/src/dird/ua_status.c b/bacula/src/dird/ua_status.c index feb3aa790a..3b7c04c878 100644 --- a/bacula/src/dird/ua_status.c +++ b/bacula/src/dird/ua_status.c @@ -393,6 +393,7 @@ static void prt_runtime(UAContext *ua, sched_pkt *sp) if (sp->job->JobType == JT_BACKUP) { jcr->db = NULL; ok = complete_jcr_for_job(jcr, sp->job, sp->pool); + Dmsg1(250, "Using pool=%s\n", jcr->pool->name()); if (jcr->db) { close_db = true; /* new db opened, remember to close it */ } @@ -400,6 +401,7 @@ static void prt_runtime(UAContext *ua, sched_pkt *sp) mr.PoolId = jcr->jr.PoolId; mr.StorageId = sp->store->StorageId; jcr->wstore = sp->store; + Dmsg0(250, "call find_next_volume_for_append\n"); ok = find_next_volume_for_append(jcr, &mr, 1, false/*no create*/); } if (!ok) { @@ -501,6 +503,7 @@ static void list_scheduled_jobs(UAContext *ua) sp->pool = run->pool; get_job_storage(&store, job, run); sp->store = store.store; + Dmsg3(250, "job=%s store=%s MediaType=%s\n", job->name(), sp->store->name(), sp->store->media_type); sched.binary_insert_multiple(sp, my_compare); num_jobs++; } diff --git a/bacula/src/dird/verify.c b/bacula/src/dird/verify.c index 1080bf9a28..54450a2d2c 100644 --- a/bacula/src/dird/verify.c +++ b/bacula/src/dird/verify.c @@ -412,7 +412,7 @@ void verify_cleanup(JCR *jcr, int TermCode) jobstatus_to_ascii(jcr->FDJobStatus, fd_term_msg, sizeof(fd_term_msg)); if (jcr->JobLevel == L_VERIFY_VOLUME_TO_CATALOG) { jobstatus_to_ascii(jcr->SDJobStatus, sd_term_msg, sizeof(sd_term_msg)); - Jmsg(jcr, msg_type, 0, _("Bacula %s (%s): %s\n" + Jmsg(jcr, msg_type, 0, _("Bacula %s Version: %s (%s) %s %s %s at %s\n" " JobId: %d\n" " Job: %s\n" " FileSet: %s\n" @@ -428,8 +428,7 @@ void verify_cleanup(JCR *jcr, int TermCode) " FD termination status: %s\n" " SD termination status: %s\n" " Termination: %s\n\n"), - VERSION, - LSMDATE, + my_name, VERSION, BDATE, HOST_OS, DISTNAME, DISTVER, edt, jcr->jr.JobId, jcr->jr.Job, @@ -447,7 +446,7 @@ void verify_cleanup(JCR *jcr, int TermCode) sd_term_msg, term_msg); } else { - Jmsg(jcr, msg_type, 0, _("Bacula %s (%s): %s\n" + Jmsg(jcr, msg_type, 0, _("Bacula %s Version: %s (%s) %s %s %s\n" " JobId: %d\n" " Job: %s\n" " FileSet: %s\n" @@ -461,8 +460,7 @@ void verify_cleanup(JCR *jcr, int TermCode) " Non-fatal FD errors: %d\n" " FD termination status: %s\n" " Termination: %s\n\n"), - VERSION, - LSMDATE, + my_name, VERSION, BDATE, HOST_OS, DISTNAME, DISTVER, edt, jcr->jr.JobId, jcr->jr.Job, diff --git a/bacula/src/lib/bsnprintf.c b/bacula/src/lib/bsnprintf.c index 1d1d2553c2..07b94bb135 100644 --- a/bacula/src/lib/bsnprintf.c +++ b/bacula/src/lib/bsnprintf.c @@ -326,6 +326,7 @@ int bvsnprintf(char *buffer, int32_t maxlen, const char *format, va_list args) currlen = fmtstr(buffer, currlen, maxlen, strvalue, flags, min, max); break; case 'p': + flags |= DP_F_UNSIGNED; strvalue = va_arg(args, char *); currlen = fmtint(buffer, currlen, maxlen, (long)strvalue, 16, min, max, flags); break; diff --git a/bacula/src/stored/status.c b/bacula/src/stored/status.c index b671936270..091cb6278f 100644 --- a/bacula/src/stored/status.c +++ b/bacula/src/stored/status.c @@ -366,7 +366,7 @@ static void list_running_jobs(void sendit(const char *msg, int len, void *sarg), } if (rdcr && rdcr->device) { len = Mmsg(msg, _("Reading: %s %s job %s JobId=%d Volume=\"%s\"\n" - " pool=\"%s\" device=\"%s\"\n"), + " pool=\"%s\" device=%s\n"), job_level_to_str(jcr->JobLevel), job_type_to_str(jcr->JobType), JobName, @@ -379,7 +379,7 @@ static void list_running_jobs(void sendit(const char *msg, int len, void *sarg), } if (dcr && dcr->device) { len = Mmsg(msg, _("Writing: %s %s job %s JobId=%d Volume=\"%s\"\n" - " pool=\"%s\" device=\"%s\"\n"), + " pool=\"%s\" device=%s\n"), job_level_to_str(jcr->JobLevel), job_type_to_str(jcr->JobType), JobName, diff --git a/bacula/technotes-2.1 b/bacula/technotes-2.1 index fdaab93b0c..869037f85f 100644 --- a/bacula/technotes-2.1 +++ b/bacula/technotes-2.1 @@ -2,6 +2,19 @@ General: 23Mar07 +kes Write new subroutine is_volume_purged() that explicitly checks + if the Volume is purged, and if so marks it as such. This should + resolve problems reported about needing to mount twice to recycle + volumes. +kes Rewrite pruning algorithm to do more work in the SQL engine, and + to pass a list of JobIds to be deleted to SQL. Also, minimize the + amount of duplicated code. +kes Do volume pruning only for the Media Type desired (reduces pruning time + if multiple Media Types are in the same pool). +kes Implement more detailed info in the Job report for the Bacula version + and architecture. +kes Switch from POOLMEM to POOL_MEM (a real class) in ua_prune.c and + ua_purge.c. ebl Add bbatch tool to bench database and insert mode. Fix sql quote stuff in batch mode Fix sql stuff for mysql -- 2.39.5