From 4ea1b913118be209dc49b78c195e19fbec54ed21 Mon Sep 17 00:00:00 2001 From: Kern Sibbald Date: Tue, 1 Mar 2005 21:54:22 +0000 Subject: [PATCH] - Implement setting DIR Storage device to Autochanger name. - Select first available device in Autochanger. - Pass back actual device name used. - Allow Query of AutoChanger. - Modify Query to include name of AutoChanger if Device belongs to one. - Remove old Pool code in jobq.c - Add Autoselect flag to query and DEVICE class (still need Directive). git-svn-id: https://bacula.svn.sourceforge.net/svnroot/bacula/trunk@1854 91ce42f0-d328-0410-95d8-f526ca767f89 --- bacula/kernstodo | 46 ++++++++++++---- bacula/src/dird/dird_conf.h | 3 ++ bacula/src/dird/jobq.c | 24 ++------- bacula/src/dird/msgchan.c | 32 +++++++----- bacula/src/dird/ua_run.c | 7 +-- bacula/src/stored/dev.c | 6 +-- bacula/src/stored/dev.h | 2 + bacula/src/stored/job.c | 101 +++++++++++++++++++++++++++++++----- bacula/src/version.h | 4 +- 9 files changed, 161 insertions(+), 64 deletions(-) diff --git a/bacula/kernstodo b/bacula/kernstodo index 8172390982..186742d3e2 100644 --- a/bacula/kernstodo +++ b/bacula/kernstodo @@ -24,17 +24,11 @@ Version 1.37 Kern (see below) Suggestions for Preben: - Look at adding Client run command that will use the port opened by the client. -- Implement WildFile and WildDir to solve problem of - saving only *.doc files. -- Bootstrap from JobMedia records. - Optimized bootstrap. Autochangers: -- Locking (done) -- Key on Storage rather than Pool (done) -- Allow multiple drives to use same Pool -- Synchronize multiple drives so that no more - than one loads a tape and any time. +- Copy Changer Device and Changer Command from Autochanger + to Device resource in SD if none given in Device resource. - Doc the following to activate, check or disable the hardware compression feature on my exb-8900 i use the exabyte "MammothTool" you can get it here: @@ -42,6 +36,35 @@ Autochangers: There is a solaris version of this tool. With option -C 0 or 1 you can disable or activate compression. Start this tool without any options for a small reference. +- 3.Prevent two drives requesting the same Volume in any given + autochanger. +- 4. Use Changer Device and Changer Command specified in the + Autochanger resource, if none is found in the Device resource. + You can continue to specify them in the Device resource if you want + or need them to be different for each device. +- 5. Implement a new Device directive (perhaps "Autoselect = yes/no") + that can allow a Device be part of an Autochanger, and hence the changer + script protected, but if set to no, will prevent the Device from being + automatically selected from the changer. This allows the device to + be directly accessed through its Device name, but not through the + AutoChanger name. +- 7. Implement new Console commands to allow offlining/reserving drives, + and possibly manipulating the autochanger (much asked for). +- 8. Automatic updating of Drive status from SD to DIR when something + changes (Volume, offline, append, read, ...). + +Autochangers Done: +- 1. Automatic use of more than one drive in an autochanger (done) +- 2. Automatic selection of the correct drive for each Job (i.e. + selects a drive with an appropriate Volume for the Job) (done) +- 6. Allow multiple simultaneous Jobs referencing the same pool write + to several tapes (some new directive(s) are are probably needed for + this) (done) +- Locking (done) +- Key on Storage rather than Pool (done) +- Allow multiple drives to use same Pool (change jobq.c DIR) (done). +- Synchronize multiple drives so that not more + than one loads a tape and any time (done) For 1.37: @@ -68,7 +91,8 @@ For 1.37: > ..\findlib\../../findlib/create_file.c:275 Could not open e:/: ERR=Der > Prozess kann nicht auf die Datei zugreifen, da sie von einem anderen > Prozess verwendet wird. - Restore restores all files, but then fails at the end. + Restore restores all files, but then fails at the end trying + to set the attributes of e: - Add better documentation on how restores can be done from failed jobs. - Resolve the problem between Device name and Archive name, @@ -1240,3 +1264,7 @@ Block Position: 0 for operator intervention. - Cancel command should include JobId in list of Jobs. - Add performance testing hooks +- Bootstrap from JobMedia records. +- Implement WildFile and WildDir to solve problem of + saving only *.doc files. + diff --git a/bacula/src/dird/dird_conf.h b/bacula/src/dird/dird_conf.h index 19cc098330..3bc8ec5a51 100644 --- a/bacula/src/dird/dird_conf.h +++ b/bacula/src/dird/dird_conf.h @@ -133,6 +133,8 @@ public: bool labeled; bool autochanger; bool offline; + bool autoselect; + char ChangerName[MAX_NAME_LENGTH]; char VolumeName[MAX_NAME_LENGTH]; char MediaType[MAX_NAME_LENGTH]; }; @@ -224,6 +226,7 @@ public: int64_t StorageId; /* Set from Storage DB record */ + /* Methods */ char *dev_name() const; char *name() const; }; diff --git a/bacula/src/dird/jobq.c b/bacula/src/dird/jobq.c index 1345e30398..0bf994bce0 100755 --- a/bacula/src/dird/jobq.c +++ b/bacula/src/dird/jobq.c @@ -652,27 +652,7 @@ static bool acquire_resources(JCR *jcr) /* Simple case, first job */ jcr->store->NumConcurrentJobs = 1; } else if (jcr->store->NumConcurrentJobs < jcr->store->MaxConcurrentJobs) { - /* - * At this point, we already have at least one Job running - * for this Storage daemon, so we must ensure that there - * is no Volume conflict. In general, it should be OK, if - * all Jobs pull from the same Pool, so we check the Pools. - */ - JCR *njcr; - lock_jcr_chain(); - for (njcr=jobs; njcr; njcr=njcr->next) { - if (njcr->JobId == 0 || njcr == jcr) { - continue; - } - if (njcr->pool != jcr->pool) { - skip_this_jcr = true; - break; - } - } - unlock_jcr_chain(); - if (!skip_this_jcr) { - jcr->store->NumConcurrentJobs++; - } + jcr->store->NumConcurrentJobs++; } else { skip_this_jcr = true; } @@ -698,5 +678,7 @@ static bool acquire_resources(JCR *jcr) set_jcr_job_status(jcr, JS_WaitJobRes); return false; } + /* Check actual device availability */ + /* ***FIXME****/ return true; } diff --git a/bacula/src/dird/msgchan.c b/bacula/src/dird/msgchan.c index 5461d466ba..12233796ee 100644 --- a/bacula/src/dird/msgchan.c +++ b/bacula/src/dird/msgchan.c @@ -48,10 +48,11 @@ static char query_device[] = "query device=%s"; /* Response from Storage daemon */ static char OKjob[] = "3000 OK Job SDid=%d SDtime=%d Authorization=%100s\n"; -static char OK_device[] = "3000 OK use device\n"; +static char OK_device[] = "3000 OK use device device=%s\n"; static char OK_query[] = "3001 OK query append=%d read=%d num_writers=%d " - "num_waiting=%d open=%d use_count=%d labeled=%d offline=%d autochanger=%d " - "media_type=%127s volume_name=%127s"; + "num_waiting=%d open=%d use_count=%d labeled=%d offline=%d " + "autoselect=%d autochanger=%d " + "changer_name=%127s media_type=%127s volume_name=%127s"; /* Storage Daemon requests */ static char Job_start[] = "3010 Job %127s start\n"; @@ -104,9 +105,9 @@ bool connect_to_storage_daemon(JCR *jcr, int retry_interval, */ bool update_device_res(JCR *jcr, DEVICE *dev) { - POOL_MEM device_name, media_type, volume_name; + POOL_MEM device_name, changer_name, media_type, volume_name; int dev_open, dev_append, dev_read, dev_labeled; - int dev_offline, dev_autochanger; + int dev_offline, dev_autochanger, dev_autoselect; BSOCK *sd; if (!connect_to_storage_daemon(jcr, 5, 30, 0)) { return false; @@ -119,13 +120,15 @@ bool update_device_res(JCR *jcr, DEVICE *dev) Dmsg1(400, "msg); if (sscanf(sd->msg, OK_query, &dev_append, &dev_read, &dev->num_writers, &dev->num_waiting, &dev_open, - &dev->use_count, &dev_labeled, &dev_offline, - &dev_autochanger, media_type.c_str(), - volume_name.c_str()) != 11) { + &dev->use_count, &dev_labeled, &dev_offline, &dev_autoselect, + &dev_autochanger, changer_name.c_str(), media_type.c_str(), + volume_name.c_str()) != 13) { return false; } + unbash_spaces(changer_name); unbash_spaces(media_type); unbash_spaces(volume_name); + bstrncpy(dev->ChangerName, changer_name.c_str(), sizeof(dev->ChangerName)); bstrncpy(dev->MediaType, media_type.c_str(), sizeof(dev->MediaType)); bstrncpy(dev->VolumeName, volume_name.c_str(), sizeof(dev->VolumeName)); dev->open = dev_open; @@ -133,6 +136,7 @@ bool update_device_res(JCR *jcr, DEVICE *dev) dev->read = dev_read; dev->labeled = dev_labeled; dev->offline = dev_offline; + dev->autoselect = dev_autoselect; dev->autochanger = dev_autochanger; dev->found = true; } else { @@ -201,12 +205,16 @@ int start_storage_daemon_job(JCR *jcr, alist *store, int append) media_type.c_str(), pool_name.c_str(), pool_type.c_str(), append); Dmsg1(200, ">stored: %s", sd->msg); - ok = response(jcr, sd, OK_device, "Use Device", NO_DISPLAY); - if (!ok) { - pm_strcpy(pool_type, sd->msg); /* save message */ + if (bget_dirmsg(sd) > 0) { + Dmsg1(400, "msg); + ok = sscanf(sd->msg, OK_device, device_name.c_str()) == 1; + } else { + POOL_MEM err_msg; + ok = false; + pm_strcpy(err_msg, sd->msg); /* save message */ Jmsg(jcr, M_FATAL, 0, _("\n" " Storage daemon didn't accept Device \"%s\" because:\n %s"), - device_name.c_str(), pool_type.c_str()/* sd->msg */); + device_name.c_str(), err_msg.c_str()/* sd->msg */); } // } if (ok) { diff --git a/bacula/src/dird/ua_run.c b/bacula/src/dird/ua_run.c index f757f8ba77..29fba3371d 100644 --- a/bacula/src/dird/ua_run.c +++ b/bacula/src/dird/ua_run.c @@ -617,10 +617,7 @@ try_again: /* * At user request modify parameters of job to be run. */ - if (ua->cmd[0] == 0) { - goto bail_out; - } - if (strncasecmp(ua->cmd, _("mod"), strlen(ua->cmd)) == 0) { + if (ua->cmd[0] != 0 && strncasecmp(ua->cmd, _("mod"), strlen(ua->cmd)) == 0) { FILE *fd; start_prompt(ua, _("Parameters to modify:\n")); @@ -843,7 +840,7 @@ try_again: goto bail_out; } - if (strncasecmp(ua->cmd, _("yes"), strlen(ua->cmd)) == 0) { + if (ua->cmd[0] == 0 || strncasecmp(ua->cmd, _("yes"), strlen(ua->cmd)) == 0) { JobId_t JobId; Dmsg1(800, "Calling run_job job=%x\n", jcr->job); start_job: diff --git a/bacula/src/stored/dev.c b/bacula/src/stored/dev.c index 326f09e991..cc3316fe78 100644 --- a/bacula/src/stored/dev.c +++ b/bacula/src/stored/dev.c @@ -116,7 +116,7 @@ init_dev(JCR *jcr, DEVICE *dev, DEVRES *device) if (dev) { dev->dev_errno = errno; } - Jmsg2(jcr, M_FATAL, 0, _("Unable to stat device %s: ERR=%s\n"), + Jmsg2(jcr, M_ERROR, 0, _("Unable to stat device %s: ERR=%s\n"), device->device_name, be.strerror()); return NULL; } @@ -134,7 +134,7 @@ init_dev(JCR *jcr, DEVICE *dev, DEVRES *device) if (dev) { dev->dev_errno = ENODEV; } - Jmsg2(jcr, M_FATAL, 0, _("%s is an unknown device type. Must be tape or directory. st_mode=%x\n"), + Jmsg2(jcr, M_ERROR, 0, _("%s is an unknown device type. Must be tape or directory. st_mode=%x\n"), device->device_name, statp.st_mode); return NULL; } @@ -190,7 +190,7 @@ init_dev(JCR *jcr, DEVICE *dev, DEVRES *device) if (stat(device->mount_point, &statp) < 0) { berrno be; dev->dev_errno = errno; - Jmsg2(jcr, M_FATAL, 0, _("Unable to stat mount point %s: ERR=%s\n"), + Jmsg2(jcr, M_ERROR, 0, _("Unable to stat mount point %s: ERR=%s\n"), device->mount_point, be.strerror()); return NULL; } diff --git a/bacula/src/stored/dev.h b/bacula/src/stored/dev.h index d439e8c144..a88f14291f 100644 --- a/bacula/src/stored/dev.h +++ b/bacula/src/stored/dev.h @@ -194,6 +194,7 @@ public: int dev_errno; /* Our own errno */ int mode; /* read/write modes */ int openmode; /* parameter passed to open_dev (useful to reopen the device) */ + bool autoselect; /* Autoselect in autochanger */ int label_type; /* Bacula/ANSI/IBM label types */ uint32_t drive_index; /* Autochanger drive index */ POOLMEM *dev_name; /* device name */ @@ -243,6 +244,7 @@ public: int rem_wait_sec; int num_wait; + /* Methods */ int is_tape() const; int is_file() const; int is_fifo() const; diff --git a/bacula/src/stored/job.c b/bacula/src/stored/job.c index d6dfa0e631..e6b8c560d3 100644 --- a/bacula/src/stored/job.c +++ b/bacula/src/stored/job.c @@ -49,14 +49,15 @@ static char query_device[] = "query device=%127s"; /* Responses sent to Director daemon */ static char OKjob[] = "3000 OK Job SDid=%u SDtime=%u Authorization=%s\n"; -static char OK_device[] = "3000 OK use device\n"; +static char OK_device[] = "3000 OK use device device=%s\n"; static char NO_device[] = "3924 Device \"%s\" not in SD Device resources.\n"; static char NOT_open[] = "3925 Device \"%s\" could not be opened or does not exist.\n"; static char BAD_use[] = "3913 Bad use command: %s\n"; static char BAD_job[] = "3915 Bad Job command: %s\n"; static char OK_query[] = "3001 OK query append=%d read=%d num_writers=%d " - "num_waiting=%d open=%d use_count=%d labeled=%d offline=%d autochanger=%d " - "media_type=%s volume_name=%s"; + "num_waiting=%d open=%d use_count=%d labeled=%d offline=%d " + "autoselect=%d autochanger=%d " + "changer_name=%s media_type=%s volume_name=%s"; static char BAD_query[] = "3917 Bad query command: %s\n"; /* @@ -142,6 +143,7 @@ bool use_cmd(JCR *jcr) /* * Wait for the device, media, and pool information */ + Dmsg1(100, "Use_cmd: %s\n", jcr->dir_bsock->msg); if (!use_device_cmd(jcr)) { set_jcr_job_status(jcr, JS_ErrorTerminated); memset(jcr->sd_auth_key, 0, strlen(jcr->sd_auth_key)); @@ -157,6 +159,7 @@ bool run_cmd(JCR *jcr) struct timespec timeout; int errstat; + Dmsg1(100, "Run_cmd: %s\n", jcr->dir_bsock->msg); /* The following jobs don't need the FD */ switch (jcr->JobType) { case JT_MIGRATION: @@ -260,10 +263,11 @@ static bool use_device_cmd(JCR *jcr) POOL_MEM dev_name, media_type, pool_name, pool_type; BSOCK *dir = jcr->dir_bsock; DEVRES *device; + AUTOCHANGER *changer; int append; bool ok; - Dmsg1(120, "Use device: %s", dir->msg); + Dmsg1(100, "Use_device_cmd: %s", jcr->dir_bsock->msg); /* * If there are multiple devices, the director sends us * use_device for each device that it wants to use. @@ -288,7 +292,7 @@ static bool use_device_cmd(JCR *jcr) } if (!device->dev) { Jmsg(jcr, M_FATAL, 0, _("\n" - " Archive \"%s\" requested by DIR could not be opened or does not exist.\n"), + " Device \"%s\" requested by DIR could not be opened or does not exist.\n"), dev_name.c_str()); bnet_fsend(dir, NOT_open, dev_name.c_str()); return false; @@ -298,7 +302,7 @@ static bool use_device_cmd(JCR *jcr) bnet_fsend(dir, _("3926 Could not get dcr for device: %s\n"), dev_name.c_str()); return false; } - Dmsg1(120, "Found device %s\n", device->hdr.name); + Dmsg1(100, "Found device %s\n", device->hdr.name); bstrncpy(dcr->pool_name, pool_name, name_len); bstrncpy(dcr->pool_type, pool_type, name_len); bstrncpy(dcr->media_type, media_type, name_len); @@ -315,9 +319,61 @@ static bool use_device_cmd(JCR *jcr) return false; } Dmsg1(220, "Got: %s", dir->msg); - return bnet_fsend(dir, OK_device); + bash_spaces(dev_name); + return bnet_fsend(dir, OK_device, dev_name.c_str()); } } + + foreach_res(changer, R_AUTOCHANGER) { + /* Find resource, and make sure we were able to open it */ + if (fnmatch(dev_name.c_str(), changer->hdr.name, 0) == 0) { + const int name_len = MAX_NAME_LENGTH; + DCR *dcr; + /* Try each device in this AutoChanger */ + foreach_alist(device, changer->device) { + Dmsg1(100, "Try changer device %s\n", device->hdr.name); + if (!device->dev) { + device->dev = init_dev(jcr, NULL, device); + } + if (!device->dev) { + Dmsg1(100, "Device %s could not be opened. Skipped\n", dev_name.c_str()); + Jmsg(jcr, M_WARNING, 0, _("\n" + " Device \"%s\" in changer \"%s\" requested by DIR could not be opened or does not exist.\n"), + device->hdr.name, dev_name.c_str()); + continue; + } + dcr = new_dcr(jcr, device->dev); + if (!dcr) { + bnet_fsend(dir, _("3926 Could not get dcr for device: %s\n"), dev_name.c_str()); + UnlockRes(); + return false; + } + Dmsg1(100, "Found changer device %s\n", device->hdr.name); + bstrncpy(dcr->pool_name, pool_name, name_len); + bstrncpy(dcr->pool_type, pool_type, name_len); + bstrncpy(dcr->media_type, media_type, name_len); + bstrncpy(dcr->dev_name, dev_name, name_len); + jcr->dcr = dcr; + if (append == SD_APPEND) { + ok = reserve_device_for_append(jcr, device->dev); + } else { + ok = reserve_device_for_read(jcr, device->dev); + } + if (!ok) { + Jmsg(jcr, M_WARNING, 0, _("Could not reserve device: %s\n"), dev_name.c_str()); + free_dcr(jcr->dcr); + continue; + } + Dmsg1(100, "Device %s opened.\n", dev_name.c_str()); + UnlockRes(); + pm_strcpy(dev_name, device->hdr.name); + bash_spaces(dev_name); + return bnet_fsend(dir, OK_device, dev_name.c_str()); + } + break; /* we found it but could not open a device */ + } + } + UnlockRes(); if (verbose) { unbash_spaces(dir->msg); @@ -351,12 +407,13 @@ static bool use_device_cmd(JCR *jcr) */ bool query_cmd(JCR *jcr) { - POOL_MEM dev_name; + POOL_MEM dev_name; BSOCK *dir = jcr->dir_bsock; DEVRES *device; + AUTOCHANGER *changer; bool ok; - Dmsg1(120, "Query: %s", dir->msg); + Dmsg1(100, "Query_cmd: %s", dir->msg); ok = sscanf(dir->msg, query_device, dev_name.c_str()) == 1; if (ok) { @@ -372,7 +429,7 @@ bool query_cmd(JCR *jcr) break; } DEVICE *dev = device->dev; - POOL_MEM VolumeName, MediaType; + POOL_MEM VolumeName, MediaType, ChangerName; UnlockRes(); if (dev->is_labeled()) { pm_strcpy(VolumeName, dev->VolHdr.VolName); @@ -382,13 +439,32 @@ bool query_cmd(JCR *jcr) bash_spaces(VolumeName); pm_strcpy(MediaType, device->media_type); bash_spaces(MediaType); + if (device->changer_res) { + pm_strcpy(ChangerName, device->changer_res->hdr.name); + bash_spaces(ChangerName); + } else { + pm_strcpy(ChangerName, ""); + } return bnet_fsend(dir, OK_query, dev->can_append()!=0, dev->can_read()!=0, dev->num_writers, dev->num_waiting, dev->is_open()!=0, dev->use_count, dev->is_labeled()!=0, - dev->is_offline()!=0, device->changer_res!=NULL, - MediaType.c_str(), VolumeName.c_str()); + dev->is_offline()!=0, 0, dev->autoselect, + ChangerName.c_str(), MediaType.c_str(), VolumeName.c_str()); } } + foreach_res(changer, R_AUTOCHANGER) { + /* Find resource, and make sure we were able to open it */ + if (fnmatch(dev_name.c_str(), changer->hdr.name, 0) == 0) { + UnlockRes(); + /* This is mostly to indicate that we are here */ + return bnet_fsend(dir, OK_query, 0, + 0, 0, 0, + 0, 0, 0, + 0, 1, "*", /* Set AutoChanger = 1 */ + "*", "*"); + } + } + /* If we get here, the device/autochanger was not found */ UnlockRes(); unbash_spaces(dir->msg); pm_strcpy(jcr->errmsg, dir->msg); @@ -398,6 +474,7 @@ bool query_cmd(JCR *jcr) pm_strcpy(jcr->errmsg, dir->msg); bnet_fsend(dir, BAD_query, jcr->errmsg); } + return true; } diff --git a/bacula/src/version.h b/bacula/src/version.h index baf5e9ad62..a31b2f3939 100644 --- a/bacula/src/version.h +++ b/bacula/src/version.h @@ -1,8 +1,8 @@ /* */ #undef VERSION #define VERSION "1.37.4" -#define BDATE "28 February 2005" -#define LSMDATE "28Feb05" +#define BDATE "01 March 2005" +#define LSMDATE "01Mar05" /* Debug flags */ #undef DEBUG -- 2.39.5