2 Bacula(R) - The Network Backup Solution
4 Copyright (C) 2000-2017 Kern Sibbald
6 The original author of Bacula is Kern Sibbald, with contributions
7 from many others, a complete list can be found in the file AUTHORS.
9 You may use this file and others of this release according to the
10 license defined in the LICENSE file, which includes the Affero General
11 Public License, v3.0 ("AGPLv3") and some additional permissions and
12 terms pursuant to its AGPLv3 Section 7.
14 This notice must be preserved when any source code is
15 conveyed and/or propagated.
17 Bacula(R) is a registered trademark of Kern Sibbald.
20 * Bacula Director -- next_vol -- handles finding the next
21 * volume for append. Split out of catreq.c August MMIII
22 * catalog request from the Storage daemon.
24 * Kern Sibbald, March MMI
30 static int const dbglvl = 50; /* debug level */
33 * We setup the StorageId or StorageId group if it is
34 * an autochanger from the Storage and put it in
36 * store == NULL => use existing StorageId
38 void set_storageid_in_mr(STORE *store, MEDIA_DBR *mr)
41 /* Just use the plain (single) StorageId */
42 mr->sid_group = edit_int64(mr->StorageId, mr->sid);
46 /* At this point we know store != NULL */
47 mr->StorageId = store->StorageId;
48 /* Get to the parent of the autochanger (if any) */
50 store = store->changer;
51 mr->StorageId = store->StorageId;
53 /* Go to the master shared storage head (if any) */
54 if (store->shared_storage && store->shared_storage->ac_group) {
55 store = store->shared_storage;
57 /* If it is an autochanger we should have an ac_group */
58 if (store->autochanger && store->ac_group) {
59 /* Note we keep the StorageId of the local autochanger */
60 mr->sid_group = store->ac_group;
62 /* Otherwise, we just use the plain (single) StorageId */
63 mr->sid_group = edit_int64(mr->StorageId, mr->sid);
67 static void add_volume_to_exclude_list(JCR *jcr, int index, MEDIA_DBR *mr)
71 *jcr->next_vol_list = 0;
73 } else if (*jcr->next_vol_list) {
74 pm_strcat(jcr->next_vol_list, ",");
76 pm_strcat(jcr->next_vol_list, edit_int64(mr->MediaId, ed1));
78 /* The list is valid only in find_next_volume_for_append() */
79 mr->exclude_list = NULL;
82 static void set_volume_to_exclude_list(JCR *jcr, int index, MEDIA_DBR *mr)
85 *jcr->next_vol_list = 0;
87 mr->exclude_list = jcr->next_vol_list;
92 * mr.PoolId must be set
93 * mr.ScratchPoolId could be set (used if create==true)
97 * MEDIA_DBR mr with PoolId set
98 * create -- whether or not to create a new volume
100 int find_next_volume_for_append(JCR *jcr, MEDIA_DBR *mr, int index,
101 bool create, bool prune)
106 STORE *store = jcr->wstore;
108 bstrncpy(mr->MediaType, store->media_type, sizeof(mr->MediaType));
109 Dmsg6(dbglvl, "find_next_vol_for_append: JobId=%u PoolId=%d, MediaType=%s index=%d create=%d prune=%d\n",
110 (uint32_t)jcr->JobId, (int)mr->PoolId, mr->MediaType, index,
113 * If we are using an Autochanger, restrict Volume
114 * search to the Autochanger on the first pass
116 InChanger = (store->autochanger)? true : false;
118 /* Make sure we don't send two times the same volume in the same session */
119 set_volume_to_exclude_list(jcr, index, mr);
122 * Find the Next Volume for Append
126 bstrncpy(mr->VolStatus, "Append", sizeof(mr->VolStatus)); /* want only appendable volumes */
128 * 1. Look for volume with "Append" status.
130 set_storageid_in_mr(store, mr); /* put StorageId in new record */
131 ok = db_find_next_volume(jcr, jcr->db, index, InChanger, mr);
135 * No volume found, apply algorithm
137 Dmsg4(dbglvl, "after find_next_vol ok=%d index=%d InChanger=%d Vstat=%s\n",
138 ok, index, InChanger, mr->VolStatus);
140 * 2. Try finding a recycled volume
142 ok = find_recycled_volume(jcr, InChanger, mr, store);
143 set_storageid_in_mr(store, mr); /* put StorageId in new record */
144 Dmsg2(dbglvl, "find_recycled_volume ok=%d FW=%d\n", ok, mr->FirstWritten);
147 * 3. Try recycling any purged volume
149 ok = recycle_oldest_purged_volume(jcr, InChanger, mr, store);
150 set_storageid_in_mr(store, mr); /* put StorageId in new record */
153 * 4. Try pruning Volumes
156 Dmsg0(dbglvl, "Call prune_volumes\n");
157 prune_volumes(jcr, InChanger, mr, store);
159 ok = recycle_oldest_purged_volume(jcr, InChanger, mr, store);
160 set_storageid_in_mr(store, mr); /* put StorageId in new record */
162 Dmsg4(dbglvl, "after prune volumes_vol ok=%d index=%d InChanger=%d Vstat=%s\n",
163 ok, index, InChanger, mr->VolStatus);
165 * 5. Try pulling a volume from the Scratch pool
167 ok = get_scratch_volume(jcr, InChanger, mr, store);
168 set_storageid_in_mr(store, mr); /* put StorageId in new record */
169 Dmsg4(dbglvl, "after get scratch volume ok=%d index=%d InChanger=%d Vstat=%s\n",
170 ok, index, InChanger, mr->VolStatus);
173 * If we are using an Autochanger and have not found
174 * a volume, retry looking for any volume.
176 if (!ok && InChanger) {
178 continue; /* retry again accepting any volume */
186 * 6. Try "creating" a new Volume
188 ok = newVolume(jcr, mr, store);
191 * Look at more drastic ways to find an Appendable Volume
193 if (!ok && (jcr->pool->purge_oldest_volume ||
194 jcr->pool->recycle_oldest_volume)) {
195 Dmsg2(dbglvl, "No next volume found. PurgeOldest=%d\n RecyleOldest=%d",
196 jcr->pool->purge_oldest_volume, jcr->pool->recycle_oldest_volume);
197 /* Find oldest volume to recycle */
198 set_storageid_in_mr(store, mr); /* update storage id */
199 ok = db_find_next_volume(jcr, jcr->db, -1, InChanger, mr);
200 set_storageid_in_mr(store, mr); /* update storageid */
201 Dmsg1(dbglvl, "Find oldest=%d Volume\n", ok);
204 Dmsg0(dbglvl, "Try purge Volume.\n");
206 * 7. Try to purging oldest volume only if not UA calling us.
208 ua = new_ua_context(jcr);
209 if (jcr->pool->purge_oldest_volume && create) {
210 Jmsg(jcr, M_INFO, 0, _("Purging oldest volume \"%s\"\n"), mr->VolumeName);
211 ok = purge_jobs_from_volume(ua, mr);
213 * 8. or try recycling the oldest volume
215 } else if (jcr->pool->recycle_oldest_volume) {
216 Jmsg(jcr, M_INFO, 0, _("Pruning oldest volume \"%s\"\n"), mr->VolumeName);
217 ok = prune_volume(ua, mr);
221 ok = recycle_volume(jcr, mr);
222 Dmsg1(dbglvl, "Recycle after purge oldest=%d\n", ok);
227 Dmsg2(dbglvl, "VolJobs=%d FirstWritten=%d\n", mr->VolJobs, mr->FirstWritten);
229 /* If we can use the volume, check if it is expired */
230 if (has_volume_expired(jcr, mr)) {
231 if (retry++ < 200) { /* sanity check */
232 continue; /* try again from the top */
234 Jmsg(jcr, M_ERROR, 0, _(
235 "We seem to be looping trying to find the next volume. I give up.\n"));
242 Dmsg1(dbglvl, "return ok=%d find_next_vol\n", ok);
244 /* We keep the record of all previous volumes requested */
246 add_volume_to_exclude_list(jcr, index, mr);;
252 * Check if any time limits or use limits have expired
253 * if so, set the VolStatus appropriately.
255 bool has_volume_expired(JCR *jcr, MEDIA_DBR *mr)
257 bool expired = false;
260 * Check limits and expirations if "Append" and it has been used
261 * i.e. mr->VolJobs > 0
264 if (strcmp(mr->VolStatus, "Append") == 0 && mr->VolJobs > 0) {
265 /* First handle Max Volume Bytes */
266 if ((mr->MaxVolBytes > 0 && mr->VolBytes >= mr->MaxVolBytes)) {
267 Jmsg(jcr, M_INFO, 0, _("Max Volume bytes=%s exceeded. "
268 "Marking Volume \"%s\" as Full.\n"),
269 edit_uint64_with_commas(mr->MaxVolBytes, ed1), mr->VolumeName);
270 bstrncpy(mr->VolStatus, "Full", sizeof(mr->VolStatus));
273 /* Now see if Volume should only be used once */
274 } else if (mr->VolBytes > 1000 && jcr->pool->use_volume_once) {
275 Jmsg(jcr, M_INFO, 0, _("Volume used once. "
276 "Marking Volume \"%s\" as Used.\n"), mr->VolumeName);
277 bstrncpy(mr->VolStatus, "Used", sizeof(mr->VolStatus));
280 /* Now see if Max Jobs written to volume */
281 } else if (mr->MaxVolJobs > 1000 && mr->MaxVolJobs <= mr->VolJobs) {
282 Jmsg(jcr, M_INFO, 0, _("Max Volume jobs=%s exceeded. "
283 "Marking Volume \"%s\" as Used.\n"),
284 edit_uint64_with_commas(mr->MaxVolJobs, ed1), mr->VolumeName);
285 Dmsg3(dbglvl, "MaxVolJobs=%d JobId=%d Vol=%s\n", mr->MaxVolJobs,
286 (uint32_t)jcr->JobId, mr->VolumeName);
287 bstrncpy(mr->VolStatus, "Used", sizeof(mr->VolStatus));
290 /* Now see if Max Files written to volume */
291 } else if (mr->MaxVolFiles > 0 && mr->MaxVolFiles <= mr->VolFiles) {
292 Jmsg(jcr, M_INFO, 0, _("Max Volume files=%s exceeded. "
293 "Marking Volume \"%s\" as Used.\n"),
294 edit_uint64_with_commas(mr->MaxVolFiles, ed1), mr->VolumeName);
295 bstrncpy(mr->VolStatus, "Used", sizeof(mr->VolStatus));
298 /* Finally, check Use duration expiration */
299 } else if (mr->VolUseDuration > 0) {
300 utime_t now = time(NULL);
301 /* See if Vol Use has expired */
302 if (mr->VolUseDuration <= (now - mr->FirstWritten)) {
303 Jmsg(jcr, M_INFO, 0, _("Max configured use duration=%s sec. exceeded. "
304 "Marking Volume \"%s\" as Used.\n"),
305 edit_uint64_with_commas(mr->VolUseDuration, ed1), mr->VolumeName);
306 bstrncpy(mr->VolStatus, "Used", sizeof(mr->VolStatus));
312 /* Need to update media */
313 Dmsg1(dbglvl, "Vol=%s has expired update media record\n", mr->VolumeName);
314 set_storageid_in_mr(NULL, mr);
315 if (!db_update_media_record(jcr, jcr->db, mr)) {
316 Jmsg(jcr, M_ERROR, 0, _("Catalog error updating volume \"%s\". ERR=%s"),
317 mr->VolumeName, db_strerror(jcr->db));
320 Dmsg2(dbglvl, "Vol=%s expired=%d\n", mr->VolumeName, expired);
325 * Try hard to recycle the current volume
327 * Returns: on failure - reason = NULL
328 * on success - reason - pointer to reason
330 void check_if_volume_valid_or_recyclable(JCR *jcr, MEDIA_DBR *mr, const char **reason)
336 /* Check if a duration or limit has expired */
337 if (has_volume_expired(jcr, mr)) {
338 *reason = _("volume has expired");
339 if (!mr->Recycle) { /* cannot recycle */
342 /* Keep going because we may be able to recycle volume */
346 * Now see if we can use the volume as is
348 if (strcmp(mr->VolStatus, "Append") == 0 ||
349 strcmp(mr->VolStatus, "Recycle") == 0) {
355 * Check if the Volume is already marked for recycling
357 if (strcmp(mr->VolStatus, "Purged") == 0) {
358 if (recycle_volume(jcr, mr)) {
359 Jmsg(jcr, M_INFO, 0, _("Recycled current volume \"%s\"\n"), mr->VolumeName);
363 /* In principle this shouldn't happen */
364 *reason = _("and recycling of current volume failed");
369 /* At this point, the volume is not valid for writing */
370 *reason = _("but should be Append, Purged or Recycle");
373 * What we're trying to do here is see if the current volume is
374 * "recyclable" - ie. if we prune all expired jobs off it, is
375 * it now possible to reuse it for the job that it is currently
379 *reason = _("volume has recycling disabled");
383 * Check retention period from last written, but recycle to within
384 * a minute to try to catch close calls ...
386 if ((mr->LastWritten + mr->VolRetention - 60) < (utime_t)time(NULL)
387 && jcr->pool->recycle_current_volume
388 && (strcmp(mr->VolStatus, "Full") == 0 ||
389 strcmp(mr->VolStatus, "Used") == 0)) {
391 * Attempt prune of current volume to see if we can
392 * recycle it for use.
396 ua = new_ua_context(jcr);
397 ok = prune_volume(ua, mr);
401 /* If fully purged, recycle current volume */
402 if (recycle_volume(jcr, mr)) {
403 Jmsg(jcr, M_INFO, 0, _("Recycled current volume \"%s\"\n"), mr->VolumeName);
406 *reason = _("but should be Append, Purged or Recycle (recycling of the "
407 "current volume failed)");
410 *reason = _("but should be Append, Purged or Recycle (cannot automatically "
411 "recycle current volume, as it still contains unpruned data "
412 "or the Volume Retention time has not expired.)");
417 static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
419 bool get_scratch_volume(JCR *jcr, bool InChanger, MEDIA_DBR *mr,
422 MEDIA_DBR smr; /* for searching scratch pool */
427 /* Only one thread at a time can pull from the scratch pool */
430 * Get Pool record for Scratch Pool
431 * choose between ScratchPoolId and Scratch
432 * db_get_pool_numvols will first try ScratchPoolId,
433 * and then try the pool named Scratch
435 memset(&spr, 0, sizeof(spr));
436 bstrncpy(spr.Name, "Scratch", sizeof(spr.Name));
437 spr.PoolId = mr->ScratchPoolId;
438 if (db_get_pool_record(jcr, jcr->db, &spr)) {
439 smr.PoolId = spr.PoolId;
440 bstrncpy(smr.VolStatus, "Append", sizeof(smr.VolStatus)); /* want only appendable volumes */
441 bstrncpy(smr.MediaType, mr->MediaType, sizeof(smr.MediaType));
444 * If we do not find a valid Scratch volume, try
445 * recycling any existing purged volumes, then
446 * try to take the oldest volume.
448 set_storageid_in_mr(store, &smr); /* put StorageId in new record */
449 if (db_find_next_volume(jcr, jcr->db, 1, InChanger, &smr)) {
452 } else if (find_recycled_volume(jcr, InChanger, &smr, store)) {
455 } else if (recycle_oldest_purged_volume(jcr, InChanger, &smr, store)) {
460 POOL_MEM query(PM_MESSAGE);
463 * Get pool record where the Scratch Volume will go to ensure
464 * that we can add a Volume.
466 memset(&pr, 0, sizeof(pr));
467 bstrncpy(pr.Name, jcr->pool->name(), sizeof(pr.Name));
469 if (!db_get_pool_numvols(jcr, jcr->db, &pr)) {
470 Jmsg(jcr, M_WARNING, 0, _("Unable to get Pool record: ERR=%s"),
471 db_strerror(jcr->db));
475 /* Make sure there is room for another volume */
476 if (pr.MaxVols > 0 && pr.NumVols >= pr.MaxVols) {
477 Jmsg(jcr, M_WARNING, 0, _("Unable add Scratch Volume, Pool \"%s\" full MaxVols=%d\n"),
478 jcr->pool->name(), pr.MaxVols);
483 set_storageid_in_mr(store, mr);
485 /* Set default parameters from current pool */
486 set_pool_dbr_defaults_in_media_dbr(mr, &pr);
489 * set_pool_dbr_defaults_in_media_dbr set VolStatus to Append,
490 * we could have Recycled media, also, we retain the old
493 bstrncpy(mr->VolStatus, smr.VolStatus, sizeof(smr.VolStatus));
494 mr->RecyclePoolId = smr.RecyclePoolId;
496 if (!db_update_media_record(jcr, jcr->db, mr)) {
497 Jmsg(jcr, M_WARNING, 0, _("Failed to move Scratch Volume. ERR=%s\n"),
498 db_strerror(jcr->db));
502 Jmsg(jcr, M_INFO, 0, _("Using Volume \"%s\" from '%s' %spool.\n"),
503 mr->VolumeName, spr.Name,
504 ((strcmp(spr.Name, "Scratch") == 0) ? "" : "Scratch "));