From: Kern Sibbald Date: Mon, 2 Dec 2002 20:30:35 +0000 (+0000) Subject: See kes02Dec02 X-Git-Tag: Release-1.28~37 X-Git-Url: https://git.sur5r.net/?a=commitdiff_plain;h=5ff92994cf19da2fc01de499e489529a1ad7a222;p=bacula%2Fbacula See kes02Dec02 git-svn-id: https://bacula.svn.sourceforge.net/svnroot/bacula/trunk@229 91ce42f0-d328-0410-95d8-f526ca767f89 --- diff --git a/bacula/ChangeLog b/bacula/ChangeLog index c0a814a687..f702b670e9 100644 --- a/bacula/ChangeLog +++ b/bacula/ChangeLog @@ -1,3 +1,39 @@ +2002-mm-dd Version 1.28 (not yet released) + +General: from kes02Dec02 +- Wrote a general purpose bi-directional pipe command. This replaces + previous use of pipes as well as the run_program previously used. +- Make BSRs stop if no more matches are possible. +- Allow unliminted number of devices in Storage daemon. +- Allow connections to Storage daemon before all devices are initialized. + +Changes submitted this submission: +- Added a readme and an afs-bacula script to the examples directory + that permits Bacula to backup an AFS filesystem. Thanks to + Lucas Mingarro for the submission. +- Added A Sun-desktop autoloader script and Device definition to the + examples/devices subdirectory. Thanks to Lucas Mingarro for the + submission. +- If the WriteBootStrap fails, the job will now be marked in error. +- Added a week position to the scheduler syntax that allows you to + specify 1st, 2nd, 3rd, 4th, or 4th, or first, ... fifth as a week + position specification in front of a day. So if you say + + 1st sun ... + + the scheduler will start only on the first sunday of the month. The + day specification can also be a day range e.g. sun-fri. + This code is untested. +- Implemented bpipe.h and bpipe.c in src/lib, which defines a bi-directional + pipe. This allows executing other programs and sending them information + as well as getting info from them. +- Replaced the previous pipe usage with bpipes in RunBeforeJob and + RunAfterJob. +- The mail program now uses bpipes rather than pipes, which means that any + error output will appear in the job output (truly bi-directional). +- Modified BSR to handle counts and to stop when no more matches are possible. + This is untested. +- Improved error messages in smtp. ============================================================================= 2002-11-29 Version 1.27c diff --git a/bacula/examples/AFS-README b/bacula/examples/AFS-README new file mode 100644 index 0000000000..233ede4897 --- /dev/null +++ b/bacula/examples/AFS-README @@ -0,0 +1,45 @@ +From: Lucas Mingarro +To: bacula-users@lists.sourceforge.net +Subject: [Bacula-users] OpenAFS with bacula + +Hi, + I'm using Bacula for backing up an AFS file set. I don't know if +anyone else is doing it, but here is my explaination about how Bacula +works with it. + +I'm using Bacula 1.26a on a RedHat Linux 7.1 on the Bacula Director +machine and RedHat Linux 7.1 and OpenAFS 1.26 on the Bacula client machine. + +First make a user bacula in your kas server an give him rl permission on +all volumes that you want to backup with Bacula. + +In order for bacula-fd to reads the files on your +AFS server you have to give the client a kerberos ticket with +the right privileges to read the volumes. Here my script to +obtain the ticket. + +I made a script that obtains the tiket and then runs bacula-fd, + +I put this script in /sbin/afs-bacula with permissions 700. +(See current directory for a copy) + +Then you have to change the bacula-fd start/stop script. +Replace the line + +daemon /usr/local/sbin/bacula-fd $2 -c /usr/local/etc/bacula-fd.conf + +with + +/sbin/afs_bacula daemon /usr/local/sbin/bacula-fd $2 -c \ + /usr/local/etc/bacula-fd.conf + + +Ok that's all. You've got a bacula-fd validated with the kerberos server. + +Lucas Mingarro +lucas@easytech.com.ar + +Note: Don't forget that kerberos tickets have a life time :) + +See: http://www.angelfire.com/hi/plutonic/afs-faq.html for FAQ on AFS. + diff --git a/bacula/examples/afs-bacula b/bacula/examples/afs-bacula new file mode 100755 index 0000000000..ebbb2c1b2c --- /dev/null +++ b/bacula/examples/afs-bacula @@ -0,0 +1,296 @@ +#!/usr/afsws/bin/pagsh +# +# Get a Kerbos authentication ticket for AFS for Bacula, then run +# the Bacula client. See AFS-README for documentation. +# +# NAME afs_bacula +# AUTHOR Lucas Mingarro +# PURPOSE Run an AFS authenticated program. +# Get a PAG, get the user's token, +# then exec user's command +# +TEXTDOMAIN=initscripts +TEXTDOMAINDIR=/etc/locale + +# Make sure umask is sane +umask 022 + +# First set up a default search path. +export PATH="/sbin:/usr/sbin:/bin:/usr/bin:/usr/X11R6/bin" + +# Get a sane screen width +[ -z "$COLUMNS" ] && COLUMNS=80 + +if [ -f /etc/sysconfig/i18n -a -z "$NOLOCALE" ] ; then + . /etc/sysconfig/i18n + if [ "$LANG" = "ja_JP.eucJP" -a "`/sbin/consoletype`" != "pty" ]; then + unset LANG + else + export LANG + fi +fi + +# Read in our configuration +if [ -z "$BOOTUP" ]; then + if [ -f /etc/sysconfig/init ]; then + . /etc/sysconfig/init + else + # This all seem confusing? Look in /etc/sysconfig/init, + # or in /usr/doc/initscripts-*/sysconfig.txt + BOOTUP=color + RES_COL=60 + MOVE_TO_COL="echo -en \\033[${RES_COL}G" + SETCOLOR_SUCCESS="echo -en \\033[1;32m" + SETCOLOR_FAILURE="echo -en \\033[1;31m" + SETCOLOR_WARNING="echo -en \\033[1;33m" + SETCOLOR_NORMAL="echo -en \\033[0;39m" + LOGLEVEL=1 + fi + if [ -x /sbin/consoletype ]; then + if [ "`consoletype`" = "serial" ]; then + BOOTUP=serial + MOVE_TO_COL= + SETCOLOR_SUCCESS= + SETCOLOR_FAILURE= + SETCOLOR_WARNING= + SETCOLOR_NORMAL= + fi + fi +fi + +if [ "$BOOTUP" != "verbose" ]; then + INITLOG_ARGS="-q" +else + INITLOG_ARGS= +fi + +# Check if $pid (could be plural) are running +checkpid() { + while [ "$1" ]; do + [ -d /proc/$1 ] && return 0 + shift + done + return 1 +} + + +# A function to start a program. +daemon() { + # Test syntax. + local gotbase= + local base= user= nice= bg= pid + nicelevel=0 + while [ "$1" != "${1##[-+]}" ]; do + case $1 in + '') echo $"$0: Usage: daemon [+/-nicelevel] {program}" + return 1;; + --check) + base=$2 + gotbase="yes" + shift 2 + ;; + --check=?*) + base=${1#--user=} + shift + ;; + --user) + user=$2 + shift 2 + ;; + --user=?*) + user=${1#--user=} + shift + ;; + [-+][0-9]*) + nice="nice -n $1" + shift + ;; + *) echo $"$0: Usage: daemon [+/-nicelevel] {program}" + return 1;; + esac + done + + # Save basename. + [ -z $gotbase ] && base=${1##*/} + + # See if it's already running. Look *only* at the pid file. + pidlist=`pidfileofproc $base` + + [ -n "$pid" ] && return + + # make sure it doesn't core dump anywhere; while this could mask + # problems with the daemon, it also closes some security problems + ulimit -S -c 0 >/dev/null 2>&1 + + # Echo daemon + [ "$BOOTUP" = "verbose" ] && echo -n " $base" + + # And start it up. + if [ -z "$user" ]; then + $nice initlog $INITLOG_ARGS -c "$*" + else + $nice initlog $INITLOG_ARGS -c "su - $user -c \"$*\"" && +success $"$base startup" || failure $"$base startup" + fi + [ $? = 0 ] && success $"$base startup" || failure $"$base startup" +} + + +# A function to find the pid of a program. Looks *only* at the pidfile +pidfileofproc() { + local base=${1##*/} + local pid + + # Test syntax. + if [ $# = 0 ] ; then + echo $"Usage: pidfileofproc {program}" + return 1 + fi + + # First try "/var/run/*.pid" files + if [ -f /var/run/${base}.pid ] ; then + read pid < /var/run/${base}.pid + for p in $line ; do + [ -z "${p//[0-9]/}" -a -d /proc/$p ] && +pid="$pid $p" + done + if [ -n "$pid" ] ; then + echo $pid + return 0 + fi + fi +} + +echo_success() { + [ "$BOOTUP" = "color" ] && $MOVE_TO_COL + echo -n "[ " + [ "$BOOTUP" = "color" ] && $SETCOLOR_SUCCESS + echo -n $"OK" + [ "$BOOTUP" = "color" ] && $SETCOLOR_NORMAL + echo -n " ]" + echo -ne "\r" + return 0 +} + +echo_failure() { + [ "$BOOTUP" = "color" ] && $MOVE_TO_COL + echo -n "[" + [ "$BOOTUP" = "color" ] && $SETCOLOR_FAILURE + echo -n $"FAILED" + [ "$BOOTUP" = "color" ] && $SETCOLOR_NORMAL + echo -n "]" + echo -ne "\r" + return 1 +} + +echo_passed() { + [ "$BOOTUP" = "color" ] && $MOVE_TO_COL + echo -n "[" + [ "$BOOTUP" = "color" ] && $SETCOLOR_WARNING + echo -n $"PASSED" + [ "$BOOTUP" = "color" ] && $SETCOLOR_NORMAL + echo -n "]" + echo -ne "\r" + return 1 +} + +# Log that something succeeded +success() { + if [ -z "$IN_INITLOG" ]; then + initlog $INITLOG_ARGS -n $0 -s "$1" -e 1 + else + # silly hack to avoid EPIPE killing rc.sysinit + trap "" SIGPIPE + echo "$INITLOG_ARGS -n $0 -s \"$1\" -e 1" >&21 + trap - SIGPIPE + fi + [ "$BOOTUP" != "verbose" ] && echo_success + return 0 +} + +# Log that something failed +failure() { + rc=$? + if [ -z "$IN_INITLOG" ]; then + initlog $INITLOG_ARGS -n $0 -s "$1" -e 2 + else + trap "" SIGPIPE + echo "$INITLOG_ARGS -n $0 -s \"$1\" -e 2" >&21 + trap - SIGPIPE + fi + [ "$BOOTUP" != "verbose" ] && echo_failure + return $rc +} + +# Log that something passed, but may have had errors. Useful for fsck +passed() { + rc=$? + if [ -z "$IN_INITLOG" ]; then + initlog $INITLOG_ARGS -n $0 -s "$1" -e 1 + else + trap "" SIGPIPE + echo "$INITLOG_ARGS -n $0 -s \"$1\" -e 1" >&21 + trap - SIGPIPE + fi + [ "$BOOTUP" != "verbose" ] && echo_passed + return $rc +} + +# Run some action. Log its output. +action() { + STRING=$1 + echo -n "$STRING " + shift + initlog $INITLOG_ARGS -c "$*" && success $"$STRING" || failure $"$STRING" + rc=$? + echo + return $rc +} + +# returns OK if $1 contains $2 +strstr() { + [ "$1" = "$2" ] && return 0 + slice=${1#*$2*} + [ "$slice" = "$1" ] && return 1 + return 0 +} + +# Confirm whether we really want to run this service +confirm() { + local YES=$"yY" + local NO=$"nN" + local CONT=$"cC" + + while : ; do + echo -n $"Start service $1 (Y)es/(N)o/(C)ontinue? [Y] " + read answer + if strstr "$YES" "$answer" || [ "$answer" = "" ] ; then + return 0 + elif strstr "$CONT" "$answer" ; then + return 2 + elif strstr "$NO" "$answer" ; then + return 1 + fi + done +} + +# Here is the authentication with the kas server + +CMD=`basename ${0}` +PRINCIPAL='bacula' +passwordfile='/etc/security/afs_bacula.pw' +klog $PRINCIPAL -pipe < ${passwordfile} +command_line="$*" +command=`echo ${command_line} | awk '{print $1}'` +# Check if we can run the command. +# If we got this far, it is likely that the command name is correct +# but there may be a problem in accessing the command file. +# If there is an error, log it via syslog (logger) rather than ">&2" + + #if [ ! -x "${command}" ]; then + #M="error: unable to execute command ${command}" + #logger -i -t "${CMD}" "${M}" + #exit 1 + #fi +#fi +$command_line diff --git a/bacula/kernstodo b/bacula/kernstodo index f2eb2e15c1..deaf6ac588 100644 --- a/bacula/kernstodo +++ b/bacula/kernstodo @@ -1,5 +1,5 @@ Kern's ToDo List - 27 November 2002 + 2 December 2002 Documentation to do: (a little bit at a time) - Document running a test version. @@ -11,17 +11,29 @@ Testing to do: (painful) - that mod of restore options works. - that console command line options work - blocksize recognition code. +- Test new BSR code + For 1.28 release: -- Think about how to make Bacula work better with File archives. +- Make BSR accept count (total files to be restored). +- Make BSR return next_block when it knows record is not + in block, done when count is reached, and possibly other + optimizations. I.e. add a state word. +- Continue improving the restore process (handling + of tapes, efficiency improvements e.g. use FSF to + position the tape, ...) +- Add code to fast seek to proper place on tape/file + when doing Restore. If it doesn't work, try linear + search as before. +- Add code to reject whole blocks if not wanted on restore. + +- Figure out how to allow multiple simultaneous file Volumes on + a single device. - Start working on Base jobs. - Implement FileOptions (see end of this document) -- Test a second language e.g. french. - Replace popen() and pclose() -- fail safe and timeout, no SIG dep. -- Enhance schedule to have 1stSat, ... - Ensure that restore of differential jobs works (check SQL). - Make sure the MaxVolFiles is fully implemented in SD -- Make Job err if WriteBootstrap fails. - Flush all the daemon messages at the end of every job. - Check if both CatalogFiles and UseCatalog are set to SD. - Check if we can bump Bacula FD priorty in Win2000 @@ -29,9 +41,8 @@ For 1.28 release: - Need return status on read_cb() from read_records(). Need multiple records -- one per Job, maybe a JCR or some other structure with a block and a record. -- Continue improving the restore process (handling - of tapes, efficiency improvements e.g. use FSF to - position the tape, ...) +- Think about how to make Bacula work better with File archives. + - Work more on how to to a Bacula restore beginning with just a Bacula tape and a boot floppy (bare metal recovery). - Try bare metal Windows restore @@ -39,10 +50,6 @@ For 1.28 release: - Program files (i.e. execute a program to read/write files). Pass read date of last backup, size of file last time. - Put system type returned by FD into catalog. -- Add code to fast seek to proper place on tape/file - when doing Restore. If it doesn't work, try linear - search as before. -- Add code to reject whole blocks if not wanted on restore. - Possibly add email to Watchdog if drive is unmounted too long and a job is waiting on the drive. - Strip trailing slashes from Include directory names in the FD. @@ -53,43 +60,10 @@ For 1.28 release: mark the link so that the data will be reloaded. - Restore program that errors in SD due to no tape reports OK incorrectly in output. -- Make BSR accept count (total files to be restored). -- Make BSR return next_block when it knows record is not - in block, done when count is reached, and possibly other - optimizations. I.e. add a state word. - -- Running multiple simultaneous jobs has a problem when one Job - must label the volume -- the others apparently do not wait. - > - > 13-Nov-2002 02:06 dump01-dir: Start Backup JobId 20, Job=save-rho.2002-11-13_02.06.00 - > 13-Nov-2002 02:06 dump01-dir: Start Backup JobId 21, Job=save-beta.2002-11-13_02.06.01 - > 13-Nov-2002 02:06 dump01-dir: Start Backup JobId 22, Job=save-dump01.2002-11-13_02.06.02 - > 13-Nov-2002 02:06 dump01-sd: Created Volume label File0013 on device /dump7/bacula. - > 13-Nov-2002 02:06 dump01-sd: Wrote label to prelabeled Volume File0013 on device /dump7/bacula - > 13-Nov-2002 02:06 dump01-sd: save-beta.2002-11-13_02.06.01 Fatal error: Device /dump7/bacula is - > busy writing with another Volume. - > 13-Nov-2002 02:06 dump01-sd: save-dump01.2002-11-13_02.06.02 Fatal error: Device /dump7/bacula is - > busy writing with another Volume. - > 13-Nov-2002 02:06 beta-fd: save-beta.2002-11-13_02.06.01 Error: bnet.c:292 Write error sending to - > Storage daemon:dump01:9103: ERR=Broken pipe - > 13-Nov-2002 02:06 dump01-fd: save-dump01.2002-11-13_02.06.02 Error: bnet.c:319 Write error sending - > to Storage daemon:dump01:9103: ERR=Broken pipe - > 13-Nov-2002 02:08 dump01-dir: 13-Nov-2002 02:08 - > - -- Figure out how compress everything except .gz,... files. -- Make bcopy copy with a single tape drive. -- Make sure catalog doesn't keep growing. -- Permit changing ownership during restore. - After unmount, if restore job started, ask to mount. - Fix db_get_fileset in cats/sql_get.c for multiple records. -- Fix start/end blocks for File devices -- Add new code to scheduler.c and run_conf.c - schedule options (1-sat, 2-sat, ...). - Fix catalog filename truncation in sql_get and sql_create. Use only a single filename split routine. -- Add command to reset VolFiles to a larger value (don't allow - a smaller number or print big warning). - Make Restore report an error if FD or SD term codes are not OK. - Convert all %x substitution variables, which are hard to remember and read to %(variable-name). Idea from TMDA. @@ -110,6 +84,174 @@ For 1.28 release: - Fix Win32 config file definition name on /install - No READLINE_SRC if found in alternate directory. - Add Client FS/OS id (Linux, Win95/98, ...). +- Test a second language e.g. french. +- Compare tape to Client files (attributes, or attributes and data) +- Restore options (overwrite, overwrite if older, + overwrite if newer, never overwrite, ...) +- Restore to a particular time -- e.g. before date, after date. +- Make all database Ids 64 bit. +- Write an applet for Linux. +- Add estimate to Console commands +- Find solution to blank filename (i.e. path only) problem. +- Implement new daemon communications protocol. +- Remove PoolId from Job table, it exists in Media. +- Allow console commands to detach or run in background. +- Fix status delay on storage daemon during rewind. +- Add SD message variables to control operator wait time + - Maximum Operator Wait + - Minimum Message Interval + - Maximum Message Interval +- Send Operator message when cannot read tape label. +- Verify level=Volume (scan only), level=Data (compare of data to file). + Verify level=Catalog, level=InitCatalog +- Events file +- Add keyword search to show command in Console. +- Fix Win2000 error with no messages during startup. +- Events : tape has more than xxx bytes. +- Restrict characters permitted in a Resource name. +- Complete code in Bacula Resources -- this will permit + reading a new config file at any time. +- Handle ctl-c in Console +- Implement LabelTemplate (at least first cut). +- Implement script driven addition of File daemon to config files. + +- see setgroup and user for Bacula p4-5 of stunnel.c +- Implement new serialize subroutines + send(socket, "string", &Vol, "uint32", &i, NULL) +- Audit all UA commands to ensure that we always prompt where possible. +- If ./btape is called without /dev, assume argument is a Storage resource name. +- Put memory utilization in Status output of each daemon + if full status requested or if some level of debug on. +- Make database type selectable by .conf files i.e. at runtime +- gethostbyname failure in bnet_connect() continues + generating errors -- should stop. +- Add HOST to Volume label. +- Set flag for uname -a. Add to Volume label. +- Implement throttled work queue. +- Check for EOT at ENOSPC or EIO or ENXIO (unix Pc) +- Allow multiple Storage specifications (or multiple names on + a single Storage specification) in the Job record. Thus a job + can be backed up to a number of storage devices. +- Implement dump label to UA +- Copy volume using single drive. +- Concept of VolumeSet during restore which is a list + of Volume names needed. +- Restore files modified after date +- Restore file modified before date +- Emergency restore info: + - Backup Bacula + - Backup working directory + - Backup Catalog +- Restore -- do nothing but show what would happen +- SET LD_RUN_PATH=$HOME/mysql/lib/mysql +- Implement Restore FileSet= +- Create a protocol.h and protocol.c where all protocol messages + are concentrated. +- If SD cannot open a drive, make it periodically retry. +- Remove duplicate fields from jcr (e.g. jcr.level and jcr.jr.Level, ...). +- Timout a job or terminate if link goes down, or reopen link and query. +- Fill all fields in Vol/Job Header -- ensure that everything + needed is written to tape. Think about restore to Catalog + from tape. Client record needs improving. +- Find general solution for sscanf size problems (as well + as sprintf. Do at run time? +- Concept of precious tapes (cannot be reused). +- Make bcopy copy with a single tape drive. +- Permit changing ownership during restore. + +- Restore should get Device and Pool information from + job record rather than from config. +- Autolabel should be specified by DR instead of SD. +- Find out how to get the system tape block limits, e.g.: + Apr 22 21:22:10 polymatou kernel: st1: Block limits 1 - 245760 bytes. + Apr 22 21:22:10 polymatou kernel: st0: Block limits 2 - 16777214 bytes. +- Storage daemon + - Add media capacity + - AutoScan (check checksum of tape) + - Format command = "format /dev/nst0" + - MaxRewindTime + - MinRewindTime + - MaxBufferSize + - Seek resolution (usually corresponds to buffer size) + - EODErrorCode=ENOSPC or code + - Partial Read error code + - Partial write error code + - Nonformatted read error + - Nonformatted write error + - WriteProtected error + - IOTimeout + - OpenRetries + - OpenTimeout + - IgnoreCloseErrors=yes + - Tape=yes + - NoRewind=yes +- Pool + - Maxwrites + - Recycle period +- Job + - MaxWarnings + - MaxErrors (job?) +===== +- FD sends unsaved file list to Director at end of job. +- Write a Storage daemon that uses pipes and + standard Unix programs to write to the tape. + See afbackup. +- Need something that monitors the JCR queue and + times out jobs by asking the deamons where they are. + +- Enhance Jmsg code to permit buffering and saving to disk. +- device driver = "xxxx" for drives. +- restart: paranoid: read label fsf to + eom read append block, and go + super-paranoid: read label, read all files + in between, read append block, and go + verify: backspace, read append block, and go + permissive: same as above but frees drive + if tape is not valid. +- Verify from Volume +- Ensure that /dev/null works +- File daemon should build list of files skipped, and then + at end of save retry and report any errors. +- Need report class for messages. Perhaps + report resource where report=group of messages +- enhance scan_attrib and rename scan_jobtype, and + fill in code for "since" option +- Need to save contents of FileSet to tape? +- Director needs a time after which the report status is sent + anyway -- or better yet, a retry time for the job. + Don't reschedule a job if previous incarnation is still running. +- Figure out how to save the catalog (possibly a special FileSet). +- Figure out how to restore the catalog. +- Some way to automatically backup everything is needed???? +- Need a structure for pending actions: + - buffered messages + - termination status (part of buffered msgs?) +- Concept of grouping Storage devices and job can use + any of a number of devices +- Drive management + Read, Write, Clean, Delete +- Login to Bacula; Bacula users with different permissions: + owner, group, user, quotas +- Store info on each file system type (probably in the job header on tape. + This could be the output of df; or perhaps some sort of /etc/mtab record. + +Longer term to do: +- Implement FSM (File System Modules). +- Identify unchanged or "system" files and save them to a + special tape thus removing them from the standard + backup FileSet -- BASE backup. +- Turn virutally all sprintfs into snprintfs. +- Heartbeat between daemons. +- Audit M_ error codes to ensure they are correct and consistent. +- Add variable break characters to lex analyzer. + Either a bit mask or a string of chars so that + the caller can change the break characters. +- Make a single T_BREAK to replace T_COMMA, etc. +- Ensure that File daemon and Storage daemon can + continue a save if the Director goes down (this + is NOT currently the case). Must detect socket error, + buffer messages for later. +- Enhance time/duration input to allow multiple qualifiers e.g. 3d2h Projects: @@ -298,195 +440,6 @@ Item 11: New daemon communication protocol. -I haven't put these in any particular order. - -Small projects: -- Compare tape to Client files (attributes, or attributes and data) -- Restore options (overwrite, overwrite if older, - overwrite if newer, never overwrite, ...) -- Restore to a particular time -- e.g. before date, after date. -- Make all database Ids 64 bit. -- Write an applet for Linux. -- Add estimate to Console commands -- Find solution to blank filename (i.e. path only) problem. -- Implement new daemon communications protocol. - -To be done: -- Remove PoolId from Job table, it exists in Media. -- Allow console commands to detach or run in background. -- Fix status delay on storage daemon during rewind. -- Add SD message variables to control operator wait time - - Maximum Operator Wait - - Minimum Message Interval - - Maximum Message Interval -- Send Operator message when cannot read tape label. -- Think about how to handle I/O error on MTEOM. -- Verify level=Volume (scan only), level=Data (compare of data to file). - Verify level=Catalog, level=InitCatalog -- Events file -- Add keyword search to show command in Console. -- Fix Win2000 error with no messages during startup. -- Events : tape has more than xxx bytes. -- Write general list maintenance subroutines. -- Restrict characters permitted in a Resource name. -- Complete code in Bacula Resources -- this will permit - reading a new config file at any time. -- Handle ctl-c in Console -- Implement LabelTemplate (at least first cut). -- Implement script driven addition of File daemon to config files. - -- see setgroup and user for Bacula p4-5 of stunnel.c -- Implement new serialize subroutines - send(socket, "string", &Vol, "uint32", &i, NULL) -- On I/O error, write EOF, then try to write again ???? -- Audit all UA commands to ensure that we always prompt where possible. -- If ./btape is called without /dev, assume argument is a Storage resource name. -- Put memory utilization in Status output of each daemon - if full status requested or if some level of debug on. -- Make database type selectable by .conf files i.e. at runtime -- gethostbyname failure in bnet_connect() continues - generating errors -- should stop. -- Add HOST to Volume label. -- Set flag for uname -a. Add to Volume label. -- Implement throttled work queue. -- Check for EOT at ENOSPC or EIO or ENXIO (unix Pc) -- Allow multiple Storage specifications (or multiple names on - a single Storage specification) in the Job record. Thus a job - can be backed up to a number of storage devices. -- Implement full MediaLabel code. -- Implement dump label to UA -- Copy volume using single drive. -- Concept of VolumeSet during restore which is a list - of Volume names needed. -- Restore files modified after date -- Restore file modified before date -- Emergency restore info: - - Backup Bacula - - Backup working directory - - Backup Catalog -- Restore -- do nothing but show what would happen -- SET LD_RUN_PATH=$HOME/mysql/lib/mysql -- Implement Restore FileSet= -- Create a protocol.h and protocol.c where all protocol messages - are concentrated. -- If SD cannot open a drive, make it periodically retry. -- Put Bacula version somewhere in Job stream, probably Start Session - Labels. -- Remove duplicate fields from jcr (e.g. jcr.level and jcr.jr.Level, ...). -- Timout a job or terminate if link goes down, or reopen link and query. -- Fill all fields in Vol/Job Header -- ensure that everything - needed is written to tape. Think about restore to Catalog - from tape. Client record needs improving. -- Find general solution for sscanf size problems (as well - as sprintf. Do at run time? -- Concept of precious tapes (cannot be reused). - -- Restore should get Device and Pool information from - job record rather than from config. -- Autolabel should be specified by DR instead of SD. -- Find out how to get the system tape block limits, e.g.: - Apr 22 21:22:10 polymatou kernel: st1: Block limits 1 - 245760 bytes. - Apr 22 21:22:10 polymatou kernel: st0: Block limits 2 - 16777214 bytes. -- Storage daemon - - Add media capacity - - AutoScan (check checksum of tape) - - Format command = "format /dev/nst0" - - MaxRewindTime - - MinRewindTime - - MaxBufferSize - - Seek resolution (usually corresponds to buffer size) - - EODErrorCode=ENOSPC or code - - Partial Read error code - - Partial write error code - - Nonformatted read error - - Nonformatted write error - - WriteProtected error - - IOTimeout - - OpenRetries - - OpenTimeout - - IgnoreCloseErrors=yes - - Tape=yes - - NoRewind=yes -- Pool - - Maxwrites - - Recycle period -- Job - - MaxWarnings - - MaxErrors (job?) -===== -- FD sends unsaved file list to Director at end of job. -- Write a Storage daemon that uses pipes and - standard Unix programs to write to the tape. - See afbackup. -- Need something that monitors the JCR queue and - times out jobs by asking the deamons where they are. - -- Enhance Jmsg code to permit buffering and saving to disk. -- device driver = "xxxx" for drives. -- restart: paranoid: read label fsf to - eom read append block, and go - super-paranoid: read label, read all files - in between, read append block, and go - verify: backspace, read append block, and go - permissive: same as above but frees drive - if tape is not valid. -- Verify from Volume -- Ensure that /dev/null works -- File daemon should build list of files skipped, and then - at end of save retry and report any errors. -- Need report class for messages. Perhaps - report resource where report=group of messages -- enhance scan_attrib and rename scan_jobtype, and - fill in code for "since" option -- Need to save contents of FileSet to tape? -- Director needs a time after which the report status is sent - anyway -- or better yet, a retry time for the job. - Don't reschedule a job if previous incarnation is still running. -- Figure out how to save the catalog (possibly a special FileSet). -- Figure out how to restore the catalog. -- Some way to automatically backup everything is needed???? -- Need a structure for pending actions: - - buffered messages - - termination status (part of buffered msgs?) -- Concept of grouping Storage devices and job can use - any of a number of devices -- Drive management - Read, Write, Clean, Delete -- Login to Bacula; Bacula users with different permissions: - owner, group, user, quotas -- Store info on each file system type (probably in the job header on tape. - This could be the output of df; or perhaps some sort of /etc/mtab record. - -Longer term to do: -- Implement FSM (File System Modules). -- Identify unchanged or "system" files and save them to a - special tape thus removing them from the standard - backup FileSet -- BASE backup. -- Turn virutally all sprintfs into snprintfs. -- Heartbeat between daemons. -- Audit M_ error codes to ensure they are correct and consistent. -- Add variable break characters to lex analyzer. - Either a bit mask or a string of chars so that - the caller can change the break characters. -- Make a single T_BREAK to replace T_COMMA, etc. -- Ensure that File daemon and Storage daemon can - continue a save if the Director goes down (this - is NOT currently the case). Must detect socket error, - buffer messages for later. -- Enhance time/duration input to allow multiple qualifiers e.g. 3d2h - - -Done: (see kernsdone for more) -- Add EOM records? No, not at this time. The current system works and - above all is simple. -- Add VolumeUseDuration and MaximumVolumeJobs to Pool db record and - to Media db record. -- Add VOLUME_CAT_INFO to the EOS tape record (as - well as to the EOD record). -- No, not at this time. -- Put MaximumVolumeSize in Director (MaximumVolumeJobs, MaximumVolumeFiles, - MaximumFileSize). - - ==================================== @@ -646,3 +599,22 @@ Results: After implementing the above, the user will be able to specify on a file by file basis (using regular expressions) what options are applied for the backup. +==================================== + +Done: (see kernsdone for more) +- Add EOM records? No, not at this time. The current system works and + above all is simple. +- Add VolumeUseDuration and MaximumVolumeJobs to Pool db record and + to Media db record. +- Add VOLUME_CAT_INFO to the EOS tape record (as + well as to the EOD record). -- No, not at this time. +- Put MaximumVolumeSize in Director (MaximumVolumeJobs, MaximumVolumeFiles, + MaximumFileSize). +- Enhance schedule to have 1stSat, ... +- Make sure catalog doesn't keep growing. +- On I/O error, write EOF, then try to write again ? No, keep it simple. +- Figure out how compress everything except .gz,... files. + Implement FileOptions. +- Put Bacula version somewhere in Job stream, probably Start Session Labels. +- Fix start/end blocks for File devices +- Make Job err if WriteBootstrap fails. diff --git a/bacula/src/dird/backup.c b/bacula/src/dird/backup.c index 8213c6b0d0..322387ada0 100644 --- a/bacula/src/dird/backup.c +++ b/bacula/src/dird/backup.c @@ -320,12 +320,46 @@ static void backup_cleanup(JCR *jcr, int TermCode, char *since) if (!db_get_job_record(jcr->db, &jcr->jr)) { Jmsg(jcr, M_WARNING, 0, _("Error getting job record for stats: %s"), db_strerror(jcr->db)); + TermCode = JS_ErrorTerminated; } strcpy(mr.VolumeName, jcr->VolumeName); if (!db_get_media_record(jcr->db, &mr)) { Jmsg(jcr, M_WARNING, 0, _("Error getting Media record for stats: %s"), db_strerror(jcr->db)); + TermCode = JS_ErrorTerminated; + } + + /* Now update the bootstrap file if any */ + if (TermCode == JS_Terminated && jcr->job->WriteBootstrap) { + FILE *fd; + BPIPE *bpipe = NULL; + int got_pipe = 0; + char *fname = jcr->job->WriteBootstrap; + + if (*fname == '|') { + fname++; + got_pipe = 1; + bpipe = open_bpipe(fname, 0, "w"); + fd = bpipe ? bpipe->wfd : NULL; + } else { + fd = fopen(fname, jcr->JobLevel==L_FULL?"w+":"a+"); + } + if (fd) { + /* Write the record */ + fprintf(fd, "Volume=\"%s\"\n", jcr->VolumeName); + fprintf(fd, "VolSessionId=%u\n", jcr->VolSessionId); + fprintf(fd, "VolSessionTime=%u\n", jcr->VolSessionTime); + if (got_pipe) { + close_bpipe(bpipe); + } else { + fclose(fd); + } + } else { + Jmsg(jcr, M_ERROR, 0, _("Could not open WriteBootstrap file:\n" + "%s: ERR=%s\n"), fname, strerror(errno)); + TermCode = JS_ErrorTerminated; + } } msg_type = M_INFO; /* by default INFO message */ @@ -422,32 +456,5 @@ Termination: %s\n\n"), term_msg); - /* Now update the bootstrap file if any */ - if (jcr->job->WriteBootstrap) { - FILE *fd; - int got_pipe = 0; - char *fname = jcr->job->WriteBootstrap; - - if (*fname == '|') { - fname++; - got_pipe = 1; - fd = popen(fname, "w"); - } else { - fd = fopen(fname, jcr->JobLevel==L_FULL?"w+":"a+"); - } - if (!fd) { - Jmsg(jcr, M_ERROR, 0, _("Could not open WriteBootstrap file:\n" - "%s: ERR=%s\n"), fname, strerror(errno)); - return; - } - fprintf(fd, "Volume=\"%s\"\n", jcr->VolumeName); - fprintf(fd, "VolSessionId=%u\n", jcr->VolSessionId); - fprintf(fd, "VolSessionTime=%u\n", jcr->VolSessionTime); - if (got_pipe) { - pclose(fd); - } else { - fclose(fd); - } - } Dmsg0(100, "Leave backup_cleanup()\n"); } diff --git a/bacula/src/dird/dird_conf.h b/bacula/src/dird/dird_conf.h index bb5c7b5a75..a8fa1e449d 100644 --- a/bacula/src/dird/dird_conf.h +++ b/bacula/src/dird/dird_conf.h @@ -295,5 +295,6 @@ struct s_run { char mday[nbytes_for_bits(31)]; /* bit set for each day of month */ char month[nbytes_for_bits(12)]; /* bit set for each month */ char wday[nbytes_for_bits(7)]; /* bit set for each day of the week */ + char wpos[nbytes_for_bits(5)]; /* week position */ }; typedef struct s_run RUN; diff --git a/bacula/src/dird/run_conf.c b/bacula/src/dird/run_conf.c index 3efb8f24b6..032ce1037f 100644 --- a/bacula/src/dird/run_conf.c +++ b/bacula/src/dird/run_conf.c @@ -47,6 +47,7 @@ enum e_state { s_weekly, s_monthly, s_hourly, + s_wpos, /* 1st, 2nd, ...*/ }; struct s_keyw { @@ -103,30 +104,44 @@ static struct s_keyw keyw[] = { {N_("weekly"), s_weekly, 0}, {N_("monthly"), s_monthly, 0}, {N_("hourly"), s_hourly, 0}, + + {N_("1st"), s_wpos, 0}, + {N_("2nd"), s_wpos, 1}, + {N_("3rd"), s_wpos, 2}, + {N_("4th"), s_wpos, 3}, + {N_("5th"), s_wpos, 4}, + + {N_("first"), s_wpos, 0}, + {N_("second"), s_wpos, 1}, + {N_("third"), s_wpos, 2}, + {N_("fourth"), s_wpos, 3}, + {N_("fifth"), s_wpos, 4}, {NULL, s_none, 0} }; -static int have_hour, have_mday, have_wday, have_month; +static int have_hour, have_mday, have_wday, have_month, have_wpos; static int have_at; static RUN lrun; static void clear_defaults() { - have_hour = have_mday = have_wday = have_month = TRUE; + have_hour = have_mday = have_wday = have_month = have_wpos = TRUE; clear_bit(0,lrun.hour); clear_bits(0, 30, lrun.mday); clear_bits(0, 6, lrun.wday); clear_bits(0, 11, lrun.month); + clear_bits(0, 4, lrun.wpos); } static void set_defaults() { - have_hour = have_mday = have_wday = have_month = FALSE; + have_hour = have_mday = have_wday = have_month = have_wpos = FALSE; have_at = FALSE; set_bit(0,lrun.hour); set_bits(0, 30, lrun.mday); set_bits(0, 6, lrun.wday); set_bits(0, 11, lrun.month); + set_bits(0, 4, lrun.wpos); } @@ -337,6 +352,13 @@ void store_run(LEX *lc, struct res_items *item, int index, int pass) } set_bit(code, lrun.wday); break; + case s_wpos: /* Week position 1st, ... */ + if (!have_wpos) { + clear_bits(0, 4, lrun.wpos); + have_wpos = TRUE; + } + set_bit(code, lrun.wpos); + break; case s_time: /* time */ if (!have_at) { scan_err0(lc, _("Time must be preceded by keyword AT.")); @@ -418,8 +440,8 @@ void store_run(LEX *lc, struct res_items *item, int index, int pass) break; } } - if (i != 0 || (state != s_month && state != s_wday)) { - scan_err0(lc, _("Invalid month or week day range")); + if (i != 0 || (state != s_month && state != s_wday && state != s_wpos)) { + scan_err0(lc, _("Invalid month, week or position day range")); /* NOT REACHED */ } @@ -433,9 +455,8 @@ void store_run(LEX *lc, struct res_items *item, int index, int pass) break; } } - if (i != 0 || state != state2 || - (state2 != s_month && state2 != s_wday) || code == code2) { - scan_err0(lc, _("Invalid month or weekday range")); + if (i != 0 || state != state2 || code == code2) { + scan_err0(lc, _("Invalid month, weekday or position range")); /* NOT REACHED */ } if (state == s_wday) { @@ -450,8 +471,7 @@ void store_run(LEX *lc, struct res_items *item, int index, int pass) set_bits(code, 6, lrun.wday); set_bits(0, code2, lrun.wday); } - } else { - /* must be s_month */ + } else if (state == s_month) { if (!have_month) { clear_bits(0, 30, lrun.month); have_month = TRUE; @@ -463,27 +483,43 @@ void store_run(LEX *lc, struct res_items *item, int index, int pass) set_bits(code, 30, lrun.month); set_bits(0, code2, lrun.month); } - } + } else { + /* Must be position */ + if (!have_wpos) { + clear_bits(0, 4, lrun.wpos); + have_wpos = TRUE; + } + if (code < code2) { + set_bits(code, code2, lrun.wpos); + } else { + set_bits(code, 4, lrun.wpos); + set_bits(0, code2, lrun.wpos); + } + } break; case s_hourly: clear_defaults(); set_bits(0, 23, lrun.hour); set_bits(0, 30, lrun.mday); set_bits(0, 11, lrun.month); + set_bits(0, 4, lrun.wpos); break; case s_weekly: clear_defaults(); set_bit(0, lrun.wday); set_bits(0, 11, lrun.month); + set_bits(0, 4, lrun.wpos); break; case s_daily: clear_defaults(); set_bits(0, 30, lrun.mday); set_bits(0, 11, lrun.month); + set_bits(0, 4, lrun.wpos); break; case s_monthly: clear_defaults(); set_bits(0, 11, lrun.month); + set_bits(0, 4, lrun.wpos); break; default: scan_err0(lc, _("Unexpected run state\n")); diff --git a/bacula/src/dird/scheduler.c b/bacula/src/dird/scheduler.c index 19cb11ff91..f0488f71b8 100644 --- a/bacula/src/dird/scheduler.c +++ b/bacula/src/dird/scheduler.c @@ -188,7 +188,7 @@ static void find_runs() JOB *job; SCHED *sched; struct tm tm; - int hour, next_hour, minute, mday, wday, month; + int hour, next_hour, minute, mday, wday, month, wpos; Dmsg0(200, "enter find_runs()\n"); num_runjobs = 0; @@ -204,6 +204,7 @@ static void find_runs() mday = tm.tm_mday - 1; wday = tm.tm_wday; month = tm.tm_mon; + wpos = (tm.tm_mday - 1) / 7; /* Loop through all jobs */ LockRes(); @@ -219,7 +220,7 @@ static void find_runs() */ if ((bit_is_set(hour, run->hour) || bit_is_set(next_hour, run->hour)) && (bit_is_set(mday, run->mday) || bit_is_set(wday, run->wday)) && - bit_is_set(month, run->month)) { + bit_is_set(month, run->month) && bit_is_set(wpos, run->wpos)) { /* find time (time_t) job is to be run */ localtime_r(&now, &tm); diff --git a/bacula/src/dird/ua_status.c b/bacula/src/dird/ua_status.c index 927e71b520..cdea6d75d0 100644 --- a/bacula/src/dird/ua_status.c +++ b/bacula/src/dird/ua_status.c @@ -372,7 +372,7 @@ static void print_jobs_scheduled(UAContext *ua) JOB *job; SCHED *sched; struct tm tm; - int mday, wday, month, tmday, twday, tmonth, i, hour; + int mday, wday, month, wpos, tmday, twday, tmonth, twpos, i, hour; int tod, tom; int found; @@ -383,12 +383,14 @@ static void print_jobs_scheduled(UAContext *ua) mday = tm.tm_mday - 1; wday = tm.tm_wday; month = tm.tm_mon; + wpos = (tm.tm_mday - 1) / 7; tomorrow = now + 60 * 60 * 24; localtime_r(&tomorrow, &tm); tmday = tm.tm_mday - 1; twday = tm.tm_wday; tmonth = tm.tm_mon; + twpos = (tm.tm_mday - 1) / 7; /* Loop through all jobs */ LockRes(); @@ -402,10 +404,10 @@ static void print_jobs_scheduled(UAContext *ua) * Find runs in next 24 hours */ tod = (bit_is_set(mday, run->mday) || bit_is_set(wday, run->wday)) && - bit_is_set(month, run->month); + bit_is_set(month, run->month) && bit_is_set(wpos, run->wpos); tom = (bit_is_set(tmday, run->mday) || bit_is_set(twday, run->wday)) && - bit_is_set(tmonth, run->month); + bit_is_set(tmonth, run->month) && bit_is_set(wpos, run->wpos); Dmsg2(200, "tod=%d tom=%d\n", tod, tom); found = FALSE; diff --git a/bacula/src/lib/Makefile.in b/bacula/src/lib/Makefile.in index 1cf4e07363..6375186ee0 100644 --- a/bacula/src/lib/Makefile.in +++ b/bacula/src/lib/Makefile.in @@ -32,7 +32,7 @@ first_rule: all dummy: LIBSRCS = alloc.c base64.c bmisc.c bnet.c bnet_server.c \ - bshm.c btime.c \ + bpipe.c bshm.c btime.c \ cram-md5.c crc32.c daemon.c fnmatch.c \ hmac.c idcache.c jcr.c lex.c \ md5.c message.c mem_pool.c parse_conf.c \ @@ -42,7 +42,7 @@ LIBSRCS = alloc.c base64.c bmisc.c bnet.c bnet_server.c \ # immortal.c filesys.c LIBOBJS = alloc.o base64.o bmisc.o bnet.o bnet_server.o \ - bshm.o btime.o \ + bpipe.o bshm.o btime.o \ cram-md5.o crc32.o daemon.o fnmatch.o \ hmac.o idcache.o jcr.o lex.o \ md5.o message.o mem_pool.o parse_conf.o \ diff --git a/bacula/src/lib/bpipe.c b/bacula/src/lib/bpipe.c new file mode 100644 index 0000000000..8c415d65e1 --- /dev/null +++ b/bacula/src/lib/bpipe.c @@ -0,0 +1,247 @@ +/* + * bpipe.c bi-directional pipe + * + * Kern Sibbald, November MMII + * + * Version $Id$ + */ + +/* + Copyright (C) 2000, 2001, 2002 Kern Sibbald and John Walker + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of + the License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public + License along with this program; if not, write to the Free + Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, + MA 02111-1307, USA. + + */ + +#include "bacula.h" +#include "jcr.h" + + +#define MAX_ARGV 100 +static void build_argc_argv(char *cmd, int *bargc, char *bargv[], int max_arg); + + + +/* + * Run an external program. Optionally wait a specified number + * of seconds. Program killed if wait exceeded. We open + * a bi-directional pipe so that the user can read from and + * write to the program. + */ +BPIPE *open_bpipe(char *prog, int wait, char *mode) +{ + char *bargv[MAX_ARGV]; + int bargc; + int readp[2], writep[2]; + POOLMEM *tprog; + int mode_read, mode_write; + BPIPE *bpipe; + + bpipe = (BPIPE *)malloc(sizeof(BPIPE)); + memset(bpipe, 0, sizeof(BPIPE)); + mode_read = (mode[0] == 'r'); + mode_write = (mode[0] == 'w' || mode[1] == 'w'); + /* Build arguments for running program. */ + tprog = get_pool_memory(PM_FNAME); + pm_strcpy(&tprog, prog); + build_argc_argv(tprog, &bargc, bargv, MAX_ARGV); +#ifdef xxxxxx + printf("argc=%d\n", bargc); + int i; + for (i=0; iworker_pid = fork()) { + case -1: + free(bpipe); + return NULL; + + case 0: /* child */ + if (mode_write) { + close(writep[1]); + dup2(writep[0], 0); /* Dup our write to his stdin */ + } + if (mode_read) { + close(readp[0]); /* Close unused child fds */ + dup2(readp[1], 1); /* dup our read to his stdout */ + dup2(readp[1], 2); /* and his stderr */ + } + execvp(bargv[0], bargv); /* call the program */ + exit(errno); /* shouldn't get here */ + + default: /* parent */ + break; + } + if (mode_read) { + close(readp[1]); /* close unused parent fds */ + bpipe->rfd = fdopen(readp[0], "r"); /* open file descriptor */ + } + if (mode_write) { + close(writep[0]); + bpipe->wfd = fdopen(writep[1], "w"); + } + bpipe->worker_stime = time(NULL); + bpipe->wait = wait; + return bpipe; +} + +/* Close the write pipe only */ +int close_wpipe(BPIPE *bpipe) +{ + int stat = 1; + + if (bpipe->wfd) { + if (fclose(bpipe->wfd) != 0) { + stat = 0; + } + bpipe->wfd = NULL; + } + return stat; +} + +/* Close both pipes and free resources */ +int close_bpipe(BPIPE *bpipe) +{ + int chldstatus = 0; + int stat = ETIME; + int wait_option; + int remaining_wait; + + /* Close pipes */ + if (bpipe->rfd) { + fclose(bpipe->rfd); + bpipe->rfd = NULL; + } + if (bpipe->wfd) { + fclose(bpipe->wfd); + bpipe->wfd = NULL; + } + + if (bpipe->wait == 0) { + wait_option = 0; /* wait indefinitely */ + } else { + wait_option = WNOHANG; /* don't hang */ + } + remaining_wait = bpipe->wait; + + /* wait for worker child to exit */ + for ( ;; ) { + pid_t wpid; + wpid = waitpid(bpipe->worker_pid, &chldstatus, wait_option); + if (wpid == bpipe->worker_pid || (wpid == -1 && errno != EINTR)) { + break; + } + if (remaining_wait > 0) { + sleep(1); /* wait one second */ + remaining_wait--; + } else { + break; /* don't wait any longer */ + } + } + if (WIFEXITED(chldstatus)) { + stat = WEXITSTATUS(chldstatus); + } + free(bpipe); + return stat; +} + + +/* + * Run an external program. Optionally wait a specified number + * of seconds. Program killed if wait exceeded. Optionally + * return the output from the program (normally a single line). + */ +int run_program(char *prog, int wait, POOLMEM *results) +{ + BPIPE *bpipe; + int stat1, stat2; + char *mode; + + mode = (char *)(results != NULL ? "r" : ""); + bpipe = open_bpipe(prog, wait, mode); + if (!bpipe) { + return 0; + } + if (results) { + results[0] = 0; + stat1 = fgets(results, sizeof_pool_memory(results), bpipe->rfd) != NULL; + } else { + stat1 = 1; + } + stat2 = close_bpipe(bpipe); + return stat1 && stat2; +} + + +/* + * Build argc and argv from a string + */ +static void build_argc_argv(char *cmd, int *bargc, char *bargv[], int max_argv) +{ + int i, quote; + char *p, *q; + int argc = 0; + + argc = 0; + for (i=0; imail_cmd && jcr) { *cmd = edit_job_codes(jcr, *cmd, d->mail_cmd, d->where); @@ -429,11 +429,11 @@ static FILE *open_mail_pipe(JCR *jcr, POOLMEM **cmd, DEST *d) Mmsg(cmd, "mail -s \"Bacula Message\" %s", d->where); } fflush(stdout); - pfd = popen(*cmd, "w"); - if (!pfd) { - Jmsg(jcr, M_ERROR, 0, "mail popen %s failed: ERR=%s\n", *cmd, strerror(errno)); + + if (!(bpipe = open_bpipe(*cmd, 120, "rw"))) { + Jmsg(jcr, M_ERROR, 0, "open mail pipe %s failed: ERR=%s\n", *cmd, strerror(errno)); } - return pfd; + return bpipe; } /* @@ -445,7 +445,7 @@ void close_msg(void *vjcr) MSGS *msgs; JCR *jcr = (JCR *)vjcr; DEST *d; - FILE *pfd; + BPIPE *bpipe; POOLMEM *cmd, *line; int len, stat; @@ -483,8 +483,8 @@ void close_msg(void *vjcr) goto rem_temp_file; } - pfd = open_mail_pipe(jcr, &cmd, d); - if (!pfd) { + if (!(bpipe=open_mail_pipe(jcr, &cmd, d))) { + Dmsg0(000, "open mail pipe failed.\n"); goto rem_temp_file; } Dmsg0(150, "Opened mail pipe\n"); @@ -492,19 +492,30 @@ void close_msg(void *vjcr) line = get_memory(len); rewind(d->fd); while (fgets(line, len, d->fd)) { - fputs(line, pfd); + fputs(line, bpipe->wfd); + } + if (!close_wpipe(bpipe)) { /* close write pipe sending mail */ + Dmsg1(000, "close error: ERR=%s\n", strerror(errno)); } - stat = pclose(pfd); /* close pipe, sending mail */ - Dmsg1(150, "Close mail pipe stat=%d\n", stat); + /* * Since we are closing all messages, before "recursing" * make sure we are not closing the daemon messages, otherwise * kaboom. */ - if (stat < 0 && msgs != daemon_msgs && errno != ECHILD) { + if (msgs != daemon_msgs) { + /* read what mail prog returned -- should be nothing */ + while (fgets(line, len, bpipe->rfd)) { +// Dmsg1(000, "Mail prog got: %s", line); + Jmsg1(jcr, M_INFO, 0, _("Mail prog: %s"), line); + } + } + + stat = close_bpipe(bpipe); + if (stat != 0 && msgs != daemon_msgs) { Dmsg1(150, "Calling emsg. CMD=%s\n", cmd); - Emsg1(M_ERROR, 0, _("Mail program terminated in error.\nCMD=%s\n"), - cmd); + Jmsg2(jcr, M_ERROR, 0, _("Mail program terminated in error. stat=%d\n" + "CMD=%s\n"), stat, cmd); } free_memory(line); rem_temp_file: @@ -592,6 +603,7 @@ void dispatch_message(void *vjcr, int type, int level, char *msg) JCR *jcr = (JCR *) vjcr; int len; MSGS *msgs; + BPIPE *bpipe; Dmsg2(200, "Enter dispatch_msg type=%d msg=%s\n", type, msg); @@ -642,14 +654,12 @@ void dispatch_message(void *vjcr, int type, int level, char *msg) case MD_OPERATOR: Dmsg1(400, "OPERATOR for collowing msg: %s\n", msg); mcmd = get_pool_memory(PM_MESSAGE); - d->fd = open_mail_pipe(jcr, &mcmd, d); - if (d->fd) { + if ((bpipe=open_mail_pipe(jcr, &mcmd, d))) { int stat; - fputs(msg, d->fd); + fputs(msg, bpipe->wfd); /* Messages to the operator go one at a time */ - stat = pclose(d->fd); - d->fd = NULL; - if (stat < 0 && errno != ECHILD) { + stat = close_bpipe(bpipe); + if (stat != 0) { Emsg1(M_ERROR, 0, _("Operator mail program terminated in error.\nCMD=%s\n"), mcmd); } diff --git a/bacula/src/lib/protos.h b/bacula/src/lib/protos.h index 6466074891..d82ffab399 100644 --- a/bacula/src/lib/protos.h +++ b/bacula/src/lib/protos.h @@ -24,125 +24,127 @@ */ /* base64.c */ -void base64_init (void); -int to_base64 (intmax_t value, char *where); -int from_base64 (intmax_t *value, char *where); -int bin_to_base64 (char *buf, char *bin, int len); +void base64_init (void); +int to_base64 (intmax_t value, char *where); +int from_base64 (intmax_t *value, char *where); +int bin_to_base64 (char *buf, char *bin, int len); /* bmisc.c */ -char *bstrncpy (char *dest, const char *src, int maxlen); -char *bstrncat (char *dest, const char *src, int maxlen); -void *b_malloc (char *file, int line, size_t size); +char *bstrncpy (char *dest, const char *src, int maxlen); +char *bstrncat (char *dest, const char *src, int maxlen); +void *b_malloc (char *file, int line, size_t size); #ifndef DEBUG -void *bmalloc (size_t size); +void *bmalloc (size_t size); #endif -void *brealloc (void *buf, size_t size); -void *bcalloc (size_t size1, size_t size2); -int bsnprintf (char *str, size_t size, const char *format, ...); -int bvsnprintf (char *str, size_t size, const char *format, va_list ap); -int pool_sprintf (char *pool_buf, char *fmt, ...); -void create_pid_file (char *dir, char *progname, int port); -int delete_pid_file (char *dir, char *progname, int port); +void *brealloc (void *buf, size_t size); +void *bcalloc (size_t size1, size_t size2); +int bsnprintf (char *str, size_t size, const char *format, ...); +int bvsnprintf (char *str, size_t size, const char *format, va_list ap); +int pool_sprintf (char *pool_buf, char *fmt, ...); +void create_pid_file (char *dir, char *progname, int port); +int delete_pid_file (char *dir, char *progname, int port); /* bnet.c */ -int32_t bnet_recv (BSOCK *bsock); -int bnet_send (BSOCK *bsock); -int bnet_fsend (BSOCK *bs, char *fmt, ...); -int bnet_set_buffer_size (BSOCK *bs, uint32_t size, int rw); -int bnet_sig (BSOCK *bs, int sig); -BSOCK * bnet_connect (void *jcr, int retry_interval, - int max_retry_time, char *name, char *host, char *service, - int port, int verbose); -int bnet_wait_data (BSOCK *bsock, int sec); -void bnet_close (BSOCK *bsock); -BSOCK * init_bsock (void *jcr, int sockfd, char *who, char *ip, int port); -BSOCK * dup_bsock (BSOCK *bsock); -void term_bsock (BSOCK *bsock); -char * bnet_strerror (BSOCK *bsock); -char * bnet_sig_to_ascii (BSOCK *bsock); -int bnet_wait_data (BSOCK *bsock, int sec); -int bnet_despool (BSOCK *bsock); -int is_bnet_stop (BSOCK *bsock); -int is_bnet_error (BSOCK *bsock); +int32_t bnet_recv (BSOCK *bsock); +int bnet_send (BSOCK *bsock); +int bnet_fsend (BSOCK *bs, char *fmt, ...); +int bnet_set_buffer_size (BSOCK *bs, uint32_t size, int rw); +int bnet_sig (BSOCK *bs, int sig); +BSOCK * bnet_connect (void *jcr, int retry_interval, + int max_retry_time, char *name, char *host, char *service, + int port, int verbose); +int bnet_wait_data (BSOCK *bsock, int sec); +void bnet_close (BSOCK *bsock); +BSOCK * init_bsock (void *jcr, int sockfd, char *who, char *ip, int port); +BSOCK * dup_bsock (BSOCK *bsock); +void term_bsock (BSOCK *bsock); +char * bnet_strerror (BSOCK *bsock); +char * bnet_sig_to_ascii (BSOCK *bsock); +int bnet_wait_data (BSOCK *bsock, int sec); +int bnet_despool (BSOCK *bsock); +int is_bnet_stop (BSOCK *bsock); +int is_bnet_error (BSOCK *bsock); /* cram-md5.c */ int cram_md5_get_auth(BSOCK *bs, char *password); int cram_md5_auth(BSOCK *bs, char *password); void hmac_md5(uint8_t* text, int text_len, uint8_t* key, - int key_len, uint8_t *hmac); + int key_len, uint8_t *hmac); /* crc32.c */ uint32_t bcrc32(uint8_t *buf, int len); /* daemon.c */ -void daemon_start (); +void daemon_start (); /* lex.c */ -LEX * lex_close_file (LEX *lf); -LEX * lex_open_file (LEX *lf, char *fname, LEX_ERROR_HANDLER *scan_error); -int lex_get_char (LEX *lf); -void lex_unget_char (LEX *lf); -char * lex_tok_to_str (int token); -int lex_get_token (LEX *lf, int expect); +LEX * lex_close_file (LEX *lf); +LEX * lex_open_file (LEX *lf, char *fname, LEX_ERROR_HANDLER *scan_error); +int lex_get_char (LEX *lf); +void lex_unget_char (LEX *lf); +char * lex_tok_to_str (int token); +int lex_get_token (LEX *lf, int expect); /* message.c */ -void my_name_is (int argc, char *argv[], char *name); -void init_msg (void *jcr, MSGS *msg); -void term_msg (void); -void close_msg (void *jcr); -void add_msg_dest (MSGS *msg, int dest, int type, char *where, char *dest_code); -void rem_msg_dest (MSGS *msg, int dest, int type, char *where); -void Jmsg (void *jcr, int type, int level, char *fmt, ...); -void dispatch_message (void *jcr, int type, int level, char *buf); -void init_console_msg (char *wd); -void free_msgs_res (MSGS *msgs); -int open_spool_file (void *jcr, BSOCK *bs); -int close_spool_file (void *vjcr, BSOCK *bs); +void my_name_is (int argc, char *argv[], char *name); +void init_msg (void *jcr, MSGS *msg); +void term_msg (void); +void close_msg (void *jcr); +void add_msg_dest (MSGS *msg, int dest, int type, char *where, char *dest_code); +void rem_msg_dest (MSGS *msg, int dest, int type, char *where); +void Jmsg (void *jcr, int type, int level, char *fmt, ...); +void dispatch_message (void *jcr, int type, int level, char *buf); +void init_console_msg (char *wd); +void free_msgs_res (MSGS *msgs); +int open_spool_file (void *jcr, BSOCK *bs); +int close_spool_file (void *vjcr, BSOCK *bs); /* bnet_server.c */ -void bnet_thread_server(char *bind_addr, int port, int max_clients, workq_t *client_wq, - void handle_client_request(void *bsock)); -void bnet_server (int port, void handle_client_request(BSOCK *bsock)); -int net_connect (int port); -BSOCK * bnet_bind (int port); -BSOCK * bnet_accept (BSOCK *bsock, char *who); +void bnet_thread_server(char *bind_addr, int port, int max_clients, workq_t *client_wq, + void handle_client_request(void *bsock)); +void bnet_server (int port, void handle_client_request(BSOCK *bsock)); +int net_connect (int port); +BSOCK * bnet_bind (int port); +BSOCK * bnet_accept (BSOCK *bsock, char *who); /* signal.c */ -void init_signals (void terminate(int sig)); -void init_stack_dump (void); +void init_signals (void terminate(int sig)); +void init_stack_dump (void); /* util.c */ -void lcase (char *str); -void bash_spaces (char *str); -void unbash_spaces (char *str); -void strip_trailing_junk (char *str); -void strip_trailing_slashes (char *dir); -int skip_spaces (char **msg); -int skip_nonspaces (char **msg); -int fstrsch (char *a, char *b); -char * encode_time (time_t time, char *buf); -char * encode_mode (mode_t mode, char *buf); -char * edit_uint64_with_commas (uint64_t val, char *buf); -char * add_commas (char *val, char *buf); -char * edit_uint64 (uint64_t val, char *buf); -int do_shell_expansion (char *name); -int is_a_number (const char *num); -int is_buf_zero (char *buf, int len); -int duration_to_utime (char *str, utime_t *value); -int size_to_uint64(char *str, int str_len, uint64_t *rtn_value); -char *edit_utime (utime_t val, char *buf); -void jobstatus_to_ascii (int JobStatus, char *msg, int maxlen); -void pm_strcat (POOLMEM **pm, char *str); -void pm_strcpy (POOLMEM **pm, char *str); -int run_program (char *prog, int wait, POOLMEM *results); -char * job_type_to_str (int type); -char * job_status_to_str (int stat); -char * job_level_to_str (int level); -void makeSessionKey (char *key, char *seed, int mode); - +void lcase (char *str); +void bash_spaces (char *str); +void unbash_spaces (char *str); +void strip_trailing_junk (char *str); +void strip_trailing_slashes (char *dir); +int skip_spaces (char **msg); +int skip_nonspaces (char **msg); +int fstrsch (char *a, char *b); +char * encode_time (time_t time, char *buf); +char * encode_mode (mode_t mode, char *buf); +char * edit_uint64_with_commas (uint64_t val, char *buf); +char * add_commas (char *val, char *buf); +char * edit_uint64 (uint64_t val, char *buf); +int do_shell_expansion (char *name); +int is_a_number (const char *num); +int is_buf_zero (char *buf, int len); +int duration_to_utime (char *str, utime_t *value); +int size_to_uint64(char *str, int str_len, uint64_t *rtn_value); +char *edit_utime (utime_t val, char *buf); +void jobstatus_to_ascii (int JobStatus, char *msg, int maxlen); +void pm_strcat (POOLMEM **pm, char *str); +void pm_strcpy (POOLMEM **pm, char *str); +int run_program (char *prog, int wait, POOLMEM *results); +char * job_type_to_str (int type); +char * job_status_to_str (int stat); +char * job_level_to_str (int level); +void makeSessionKey (char *key, char *seed, int mode); +BPIPE * open_bpipe(char *prog, int wait, char *mode); +int close_wpipe(BPIPE *bpipe); +int close_bpipe(BPIPE *bpipe); /* watchdog.c */ diff --git a/bacula/src/lib/util.c b/bacula/src/lib/util.c index 5fed0e9e91..cc0da0539c 100644 --- a/bacula/src/lib/util.c +++ b/bacula/src/lib/util.c @@ -462,8 +462,7 @@ void jobstatus_to_ascii(int JobStatus, char *msg, int maxlen) termstat = jstat; break; } - strncpy(msg, termstat, maxlen); - msg[maxlen-1] = 0; + bstrncpy(msg, termstat, maxlen); } /* @@ -704,149 +703,6 @@ int do_shell_expansion(char *name) } -#define MAX_ARGV 100 -static void build_argc_argv(char *cmd, int *bargc, char *bargv[], int max_arg); - -/* - * Run an external program. Optionally wait a specified number - * of seconds. Program killed if wait exceeded. Optionally - * return the output from the program (normally a single line). - */ -int run_program(char *prog, int wait, POOLMEM *results) -{ - int stat = ETIME; - int chldstatus = 0; - pid_t pid1, pid2 = 0; - int pfd[2]; - char *bargv[MAX_ARGV]; - int bargc; - - - build_argc_argv(prog, &bargc, bargv, MAX_ARGV); -#ifdef xxxxxxxxxx - printf("argc=%d\n", bargc); - int i; - for (i=0; i 0) { - switch (pid2=fork()) { - case -1: - break; - case 0: /* child 2 */ - /* Time the worker process */ - sleep(wait); - if (kill(pid1, SIGTERM) == 0) { /* time expired kill it */ - exit(0); - } - sleep(3); - kill(pid1, SIGKILL); - exit(0); - default: /* parent */ - break; - } - } - - /* Parent continues here */ - int i; - if (results) { - i = read(pfd[0], results, sizeof_pool_memory(results) - 1); - if (--i < 0) { - i = 0; - } - results[i] = 0; /* set end of string */ - } - /* wait for worker child to exit */ - for ( ;; ) { - pid_t wpid; - wpid = waitpid(pid1, &chldstatus, 0); - if (wpid == pid1 || (errno != EINTR)) { - break; - } - } - if (WIFEXITED(chldstatus)) - stat = WEXITSTATUS(chldstatus); - - if (wait > 0) { - kill(pid2, SIGKILL); /* kill off timer process */ - waitpid(pid2, &chldstatus, 0); /* reap timer process */ - } - if (results) { - close(pfd[0]); /* close pipe */ - close(pfd[1]); - } - break; - } - return stat; -} - -/* - * Build argc and argv from a string - */ -static void build_argc_argv(char *cmd, int *bargc, char *bargv[], int max_argv) -{ - int i, quote; - char *p, *q; - int argc = 0; - - argc = 0; - for (i=0; iVolHdr.VolName, jcr->VolumeName) == 0)) { /* wrong tape mounted */ if (dev->num_writers != 0) { - Jmsg(jcr, M_FATAL, 0, _("Device %s is busy writing with another Volume.\n"), dev_name(dev)); + Jmsg(jcr, M_FATAL, 0, _("Device %s is busy writing on another Volume.\n"), dev_name(dev)); goto get_out; } /* Wrong tape mounted, release it, then fall through to get correct one */ diff --git a/bacula/src/stored/bsr.h b/bacula/src/stored/bsr.h index b28b39d36f..7a86787b89 100644 --- a/bacula/src/stored/bsr.h +++ b/bacula/src/stored/bsr.h @@ -40,6 +40,7 @@ struct s_vol_list { struct s_vol_list *next; char VolumeName[MAX_NAME_LENGTH]; int Slot; + int start_file; }; typedef struct s_vol_list VOL_LIST; @@ -67,34 +68,33 @@ typedef struct s_bsr_sessid { struct s_bsr_sessid *next; uint32_t sessid; uint32_t sessid2; - int found; + int done; /* local done */ } BSR_SESSID; typedef struct s_bsr_sesstime { struct s_bsr_sesstime *next; uint32_t sesstime; - int found; + int done; /* local done */ } BSR_SESSTIME; typedef struct s_bsr_volfile { struct s_bsr_volfile *next; uint32_t sfile; /* start file */ uint32_t efile; /* end file */ - int found; + int done; /* local done */ } BSR_VOLFILE; typedef struct s_bsr_findex { struct s_bsr_findex *next; int32_t findex; /* start file index */ int32_t findex2; /* end file index */ - int found; + int done; /* local done */ } BSR_FINDEX; typedef struct s_bsr_jobid { struct s_bsr_jobid *next; uint32_t JobId; uint32_t JobId2; - int found; } BSR_JOBID; typedef struct s_bsr_jobtype { @@ -110,13 +110,12 @@ typedef struct s_bsr_joblevel { typedef struct s_bsr_job { struct s_bsr_job *next; char Job[MAX_NAME_LENGTH]; - int found; + int done; } BSR_JOB; typedef struct s_bsr_stream { struct s_bsr_stream *next; int32_t stream; /* stream desired */ - int found; } BSR_STREAM; typedef struct s_bsr { @@ -124,7 +123,8 @@ typedef struct s_bsr { int done; /* set when everything found */ BSR_VOLUME *volume; int32_t Slot; /* Slot */ - int32_t count; /* count of files to restore this volume */ + uint32_t count; /* count of files to restore this bsr */ + uint32_t found; /* count of restored files this bsr */ BSR_VOLFILE *volfile; BSR_SESSTIME *sesstime; BSR_SESSID *sessid; diff --git a/bacula/src/stored/match_bsr.c b/bacula/src/stored/match_bsr.c index 000d6a3504..9798157601 100755 --- a/bacula/src/stored/match_bsr.c +++ b/bacula/src/stored/match_bsr.c @@ -33,74 +33,101 @@ #include /* Forward references */ -static int match_volume(BSR_VOLUME *volume, VOLUME_LABEL *volrec); -static int match_sesstime(BSR_SESSTIME *sesstime, DEV_RECORD *rec); -static int match_sessid(BSR_SESSID *sessid, DEV_RECORD *rec); -static int match_client(BSR_CLIENT *client, SESSION_LABEL *sessrec); -static int match_job(BSR_JOB *job, SESSION_LABEL *sessrec); -static int match_job_type(BSR_JOBTYPE *job_type, SESSION_LABEL *sessrec); -static int match_job_level(BSR_JOBLEVEL *job_level, SESSION_LABEL *sessrec); -static int match_jobid(BSR_JOBID *jobid, SESSION_LABEL *sessrec); -static int match_findex(BSR_FINDEX *findex, DEV_RECORD *rec); -static int match_volfile(BSR_VOLFILE *volfile, DEV_RECORD *rec); -static int match_stream(BSR_STREAM *stream, DEV_RECORD *rec); -static int match_one_bsr(BSR *bsr, DEV_RECORD *rec, VOLUME_LABEL *volrec, SESSION_LABEL *sessrec); +static int match_volume(BSR *bsr, BSR_VOLUME *volume, VOLUME_LABEL *volrec, int done); +static int match_sesstime(BSR *bsr, BSR_SESSTIME *sesstime, DEV_RECORD *rec, int done); +static int match_sessid(BSR *bsr, BSR_SESSID *sessid, DEV_RECORD *rec, int done); +static int match_client(BSR *bsr, BSR_CLIENT *client, SESSION_LABEL *sessrec, int done); +static int match_job(BSR *bsr, BSR_JOB *job, SESSION_LABEL *sessrec, int done); +static int match_job_type(BSR *bsr, BSR_JOBTYPE *job_type, SESSION_LABEL *sessrec, int done); +static int match_job_level(BSR *bsr, BSR_JOBLEVEL *job_level, SESSION_LABEL *sessrec, int done); +static int match_jobid(BSR *bsr, BSR_JOBID *jobid, SESSION_LABEL *sessrec, int done); +static int match_findex(BSR *bsr, BSR_FINDEX *findex, DEV_RECORD *rec, int done); +static int match_volfile(BSR *bsr, BSR_VOLFILE *volfile, DEV_RECORD *rec, int done); +static int match_stream(BSR *bsr, BSR_STREAM *stream, DEV_RECORD *rec, int done); +static int match_all(BSR *bsr, DEV_RECORD *rec, VOLUME_LABEL *volrec, SESSION_LABEL *sessrec, int done); /********************************************************************* * * Match Bootstrap records - * + * returns 1 on match + * returns 0 no match + * returns -1 no additional matches possible */ int match_bsr(BSR *bsr, DEV_RECORD *rec, VOLUME_LABEL *volrec, SESSION_LABEL *sessrec) { if (!bsr) { return 0; } - if (match_one_bsr(bsr, rec, volrec, sessrec)) { - return 1; - } - return match_bsr(bsr->next, rec, volrec, sessrec); + return match_all(bsr, rec, volrec, sessrec, 1); } -static int match_one_bsr(BSR *bsr, DEV_RECORD *rec, VOLUME_LABEL *volrec, SESSION_LABEL *sessrec) +/* + * Match all the components of current record + * returns 1 on match + * returns 0 no match + * returns -1 no additional matches possible + */ +static int match_all(BSR *bsr, DEV_RECORD *rec, VOLUME_LABEL *volrec, + SESSION_LABEL *sessrec, int done) { - if (!match_volume(bsr->volume, volrec)) { - return 0; + if (bsr->done) { + goto no_match; } - if (!match_volfile(bsr->volfile, rec)) { - return 0; + if (bsr->count && bsr->count <= bsr->found) { + bsr->done = 1; + goto no_match; } - if (!match_sesstime(bsr->sesstime, rec)) { - return 0; + if (!match_volume(bsr, bsr->volume, volrec, 1)) { + goto no_match; } - if (!match_sessid(bsr->sessid, rec)) { - return 0; + if (!match_volfile(bsr, bsr->volfile, rec, 1)) { + goto no_match; } - if (!match_jobid(bsr->JobId, sessrec)) { - return 0; + if (!match_sesstime(bsr, bsr->sesstime, rec, 1)) { + goto no_match; } - if (!match_job(bsr->job, sessrec)) { - return 0; + + /* NOTE!! This test MUST come after the sesstime test */ + if (!match_sessid(bsr, bsr->sessid, rec, 1)) { + goto no_match; } - if (!match_client(bsr->client, sessrec)) { - return 0; + + /* NOTE!! This test MUST come after sesstime and sessid tests */ + if (!match_findex(bsr, bsr->FileIndex, rec, 1)) { + goto no_match; } - if (!match_findex(bsr->FileIndex, rec)) { - return 0; + if (!match_jobid(bsr, bsr->JobId, sessrec, 1)) { + goto no_match; } - if (!match_job_type(bsr->JobType, sessrec)) { - return 0; + if (!match_job(bsr, bsr->job, sessrec, 1)) { + goto no_match; } - if (!match_job_level(bsr->JobLevel, sessrec)) { - return 0; + if (!match_client(bsr, bsr->client, sessrec, 1)) { + goto no_match; } - if (!match_stream(bsr->stream, rec)) { - return 0; + if (!match_job_type(bsr, bsr->JobType, sessrec, 1)) { + goto no_match; + } + if (!match_job_level(bsr, bsr->JobLevel, sessrec, 1)) { + goto no_match; } + if (!match_stream(bsr, bsr->stream, rec, 1)) { + goto no_match; + } + bsr->found++; return 1; + +no_match: + if (bsr->next) { + return match_all(bsr->next, rec, volrec, sessrec, bsr->done && done); + } + if (bsr->done && done) { + return -1; + } + return 0; } -static int match_volume(BSR_VOLUME *volume, VOLUME_LABEL *volrec) +static int match_volume(BSR *bsr, BSR_VOLUME *volume, VOLUME_LABEL *volrec, int done) { if (!volume) { return 0; /* Volume must match */ @@ -109,12 +136,12 @@ static int match_volume(BSR_VOLUME *volume, VOLUME_LABEL *volrec) return 1; } if (volume->next) { - return match_volume(volume->next, volrec); + return match_volume(bsr, volume->next, volrec, 1); } return 0; } -static int match_client(BSR_CLIENT *client, SESSION_LABEL *sessrec) +static int match_client(BSR *bsr, BSR_CLIENT *client, SESSION_LABEL *sessrec, int done) { if (!client) { return 1; /* no specification matches all */ @@ -123,28 +150,26 @@ static int match_client(BSR_CLIENT *client, SESSION_LABEL *sessrec) return 1; } if (client->next) { - return match_client(client->next, sessrec); + return match_client(bsr, client->next, sessrec, 1); } return 0; } -static int match_job(BSR_JOB *job, SESSION_LABEL *sessrec) +static int match_job(BSR *bsr, BSR_JOB *job, SESSION_LABEL *sessrec, int done) { if (!job) { return 1; /* no specification matches all */ } if (fnmatch(job->Job, sessrec->Job, 0) == 0) { - job->found++; return 1; } if (job->next) { - return match_job(job->next, sessrec); + return match_job(bsr, job->next, sessrec, 1); } return 0; } - -static int match_job_type(BSR_JOBTYPE *job_type, SESSION_LABEL *sessrec) +static int match_job_type(BSR *bsr, BSR_JOBTYPE *job_type, SESSION_LABEL *sessrec, int done) { if (!job_type) { return 1; /* no specification matches all */ @@ -153,12 +178,12 @@ static int match_job_type(BSR_JOBTYPE *job_type, SESSION_LABEL *sessrec) return 1; } if (job_type->next) { - return match_job_type(job_type->next, sessrec); + return match_job_type(bsr, job_type->next, sessrec, 1); } return 0; } -static int match_job_level(BSR_JOBLEVEL *job_level, SESSION_LABEL *sessrec) +static int match_job_level(BSR *bsr, BSR_JOBLEVEL *job_level, SESSION_LABEL *sessrec, int done) { if (!job_level) { return 1; /* no specification matches all */ @@ -167,43 +192,49 @@ static int match_job_level(BSR_JOBLEVEL *job_level, SESSION_LABEL *sessrec) return 1; } if (job_level->next) { - return match_job_level(job_level->next, sessrec); + return match_job_level(bsr, job_level->next, sessrec, 1); } return 0; } -static int match_jobid(BSR_JOBID *jobid, SESSION_LABEL *sessrec) +static int match_jobid(BSR *bsr, BSR_JOBID *jobid, SESSION_LABEL *sessrec, int done) { if (!jobid) { return 1; /* no specification matches all */ } if (jobid->JobId <= sessrec->JobId && jobid->JobId2 >= sessrec->JobId) { - jobid->found++; return 1; } if (jobid->next) { - return match_jobid(jobid->next, sessrec); + return match_jobid(bsr, jobid->next, sessrec, 1); } return 0; } - -static int match_volfile(BSR_VOLFILE *volfile, DEV_RECORD *rec) +static int match_volfile(BSR *bsr, BSR_VOLFILE *volfile, DEV_RECORD *rec, int done) { if (!volfile) { return 1; /* no specification matches all */ } if (volfile->sfile <= rec->File && volfile->efile >= rec->File) { - volfile->found++; return 1; } + /* Once we get past last efile, we are done */ + if (rec->File > volfile->efile) { + volfile->done = 1; /* set local done */ + } if (volfile->next) { - return match_volfile(volfile->next, rec); + return match_volfile(bsr, volfile->next, rec, volfile->done && done); + } + + /* If we are done and all prior matches are done, this bsr is finished */ + if (volfile->done && done) { + bsr->done = 1; } return 0; } -static int match_stream(BSR_STREAM *stream, DEV_RECORD *rec) +static int match_stream(BSR *bsr, BSR_STREAM *stream, DEV_RECORD *rec, int done) { if (!stream) { return 1; /* no specification matches all */ @@ -212,53 +243,67 @@ static int match_stream(BSR_STREAM *stream, DEV_RECORD *rec) return 1; } if (stream->next) { - return match_stream(stream->next, rec); + return match_stream(bsr, stream->next, rec, 1); } return 0; } -static int match_findex(BSR_FINDEX *findex, DEV_RECORD *rec) +static int match_sesstime(BSR *bsr, BSR_SESSTIME *sesstime, DEV_RECORD *rec, int done) { - if (!findex) { + if (!sesstime) { return 1; /* no specification matches all */ } - if (findex->findex <= rec->FileIndex && findex->findex2 >= rec->FileIndex) { - findex->found++; + if (sesstime->sesstime == rec->VolSessionTime) { return 1; } - if (findex->next) { - return match_findex(findex->next, rec); + if (rec->VolSessionTime > sesstime->sesstime) { + sesstime->done = 1; + } + if (sesstime->next) { + return match_sesstime(bsr, sesstime->next, rec, sesstime->done && done); + } + if (sesstime->done && done) { + bsr->done = 1; } return 0; } - -static int match_sessid(BSR_SESSID *sessid, DEV_RECORD *rec) +static int match_sessid(BSR *bsr, BSR_SESSID *sessid, DEV_RECORD *rec, int done) { if (!sessid) { return 1; /* no specification matches all */ } if (sessid->sessid <= rec->VolSessionId && sessid->sessid2 >= rec->VolSessionId) { - sessid->found++; return 1; } + if (rec->VolSessionId > sessid->sessid2) { + sessid->done = 1; + } if (sessid->next) { - return match_sessid(sessid->next, rec); + return match_sessid(bsr, sessid->next, rec, sessid->done && done); + } + if (sessid->done && done) { + bsr->done = 1; } return 0; } -static int match_sesstime(BSR_SESSTIME *sesstime, DEV_RECORD *rec) +static int match_findex(BSR *bsr, BSR_FINDEX *findex, DEV_RECORD *rec, int done) { - if (!sesstime) { + if (!findex) { return 1; /* no specification matches all */ } - if (sesstime->sesstime == rec->VolSessionTime) { - sesstime->found++; + if (findex->findex <= rec->FileIndex && findex->findex2 >= rec->FileIndex) { return 1; } - if (sesstime->next) { - return match_sesstime(sesstime->next, rec); + if (rec->FileIndex > findex->findex2) { + findex->done = 1; + } + if (findex->next) { + return match_findex(bsr, findex->next, rec, findex->done && done); + } + if (findex->done && done) { + bsr->done = 1; } return 0; } diff --git a/bacula/src/stored/parse_bsr.c b/bacula/src/stored/parse_bsr.c index 27fc007c8c..553854985f 100755 --- a/bacula/src/stored/parse_bsr.c +++ b/bacula/src/stored/parse_bsr.c @@ -690,10 +690,16 @@ int add_vol(JCR *jcr, VOL_LIST *vol) } else { for ( ; next->next; next=next->next) { if (strcmp(vol->VolumeName, next->VolumeName) == 0) { + if (vol->start_file < next->start_file) { + next->start_file = vol->start_file; + } return 0; /* already in list */ } } if (strcmp(vol->VolumeName, next->VolumeName) == 0) { + if (vol->start_file < next->start_file) { + next->start_file = vol->start_file; + } return 0; /* already in list */ } next->next = vol; /* add volume */ @@ -720,7 +726,7 @@ void create_vol_list(JCR *jcr) VOL_LIST *vol; /* - * Build a list of volume to be processed + * Build a list of volumes to be processed */ jcr->NumVolumes = 0; jcr->CurVolume = 1; @@ -731,10 +737,21 @@ void create_vol_list(JCR *jcr) } strcpy(jcr->VolumeName, bsr->volume->VolumeName); /* setup first volume */ for ( ; bsr; bsr=bsr->next) { - BSR_VOLUME *bsrvol = bsr->volume; - for ( ; bsrvol; bsrvol=bsrvol->next) { + BSR_VOLUME *bsrvol; + BSR_VOLFILE *volfile; + uint32_t sfile = 0; + + /* Find minimum start file so that we can forward space to it */ + for (volfile = bsr->volfile; volfile; volfile=volfile->next) { + if (volfile->sfile < sfile) { + sfile = volfile->sfile; + } + } + /* Now add volumes for this bsr */ + for (bsrvol = bsr->volume; bsrvol; bsrvol=bsrvol->next) { vol = new_vol(); strcpy(vol->VolumeName, bsrvol->VolumeName); + vol->start_file = sfile; if (add_vol(jcr, vol)) { jcr->NumVolumes++; Dmsg1(400, "Added volume %s\n", vol->VolumeName); @@ -742,6 +759,7 @@ void create_vol_list(JCR *jcr) Dmsg1(400, "Duplicate volume %s\n", vol->VolumeName); free((char *)vol); } + sfile = 0; /* start at beginning of second volume */ } } } else { diff --git a/bacula/src/stored/read.c b/bacula/src/stored/read.c index ac93ddf5a0..9a36388519 100644 --- a/bacula/src/stored/read.c +++ b/bacula/src/stored/read.c @@ -169,7 +169,11 @@ int do_read_data(JCR *jcr) /* Match BSR against current record */ if (jcr->bsr) { - if (!match_bsr(jcr->bsr, rec, &dev->VolHdr, &sessrec)) { + int stat = match_bsr(jcr->bsr, rec, &dev->VolHdr, &sessrec); + if (stat == -1) { /* no more possible matches */ + ok = FALSE; + break; + } else if (stat == 0) { /* no match */ Dmsg0(50, "BSR rejected record\n"); rec->remainder = 0; continue; diff --git a/bacula/src/stored/read_record.c b/bacula/src/stored/read_record.c index d8d281e816..ace357c879 100644 --- a/bacula/src/stored/read_record.c +++ b/bacula/src/stored/read_record.c @@ -155,14 +155,20 @@ next_record: /* * Apply BSR filter */ - if (jcr->bsr && !match_bsr(jcr->bsr, rec, &dev->VolHdr, &sessrec)) { - if (verbose) { - Dmsg5(10, "BSR no match rec=%d block=%d SessId=%d SessTime=%d FI=%d\n", - record, block->BlockNumber, rec->VolSessionId, rec->VolSessionTime, - rec->FileIndex); + if (jcr->bsr) { + int stat = match_bsr(jcr->bsr, rec, &dev->VolHdr, &sessrec); + if (stat == -1) { /* no more possible matches */ + ok = FALSE; + break; + } else if (stat == 0) { /* no match */ + if (verbose) { + Dmsg5(10, "BSR no match rec=%d block=%d SessId=%d SessTime=%d FI=%d\n", + record, block->BlockNumber, rec->VolSessionId, rec->VolSessionTime, + rec->FileIndex); + } + rec->remainder = 0; + continue; /* we don't want record, read next one */ } - rec->remainder = 0; - continue; /* we don't want record, read next one */ } if (is_partial_record(rec)) { Dmsg6(10, "Partial, break. recno=%d state=%s blk=%d SI=%d ST=%d FI=%d\n", record, diff --git a/bacula/src/stored/stored.c b/bacula/src/stored/stored.c index 04571249b4..ebe8cf2419 100644 --- a/bacula/src/stored/stored.c +++ b/bacula/src/stored/stored.c @@ -38,6 +38,7 @@ /* Forward referenced functions */ void terminate_stored(int sig); static void check_config(); +static void *device_allocation(void *arg); #define CONFIG_FILE "bacula-sd.conf" /* Default config file */ @@ -45,10 +46,6 @@ static void check_config(); /* Global variables exported */ -struct s_shm *shm; /* memory shared with children */ -BSHM bshm; /* shared memory control packet */ - - /* This is our own global resource */ STORES *me; @@ -84,10 +81,10 @@ static void usage() */ int main (int argc, char *argv[]) { - int ch, i; + int ch; int no_signals = FALSE; int test_config = FALSE; - DEVRES *device; + pthread_t thid; init_stack_dump(); my_name_is(argc, argv, "stored"); @@ -164,7 +161,6 @@ int main (int argc, char *argv[]) parse_config(configfile); check_config(); - bshm.size = 0; if (test_config) { terminate_stored(0); } @@ -176,16 +172,6 @@ int main (int argc, char *argv[]) create_pid_file(me->pid_directory, "bacula-sd", me->SDport); - /* ****FIXME**** clean this up */ - /* Create and attach to shared memory. This is a - * hold over from the days of child processes. - * Note, in reality all memory is shared. This - * is just a global buffer for the device packets. - */ - shm = (s_shm *) malloc(sizeof(struct s_shm)); - /* Zero shared memory */ - memset(shm, 0, sizeof(struct s_shm)); - /* Ensure that Volume Session Time and Id are both * set and are both non-zero. */ @@ -194,46 +180,22 @@ int main (int argc, char *argv[]) Emsg0(M_ABORT, 0, _("Volume Session Time is ZERO!\n")); } + /* Make sure on Solaris we can run concurrent, watch dog + servers + misc */ + set_thread_concurrency(me->max_concurrent_jobs * 2 + 4); + + /* + * Here we lock the resources then fire off the device allocation + * thread. That thread will release the resources when all the + * devices are allocated. This allows use to start the server + * right away, but any jobs will wait until the resources are + * unlocked. + */ LockRes(); - for (device=NULL,i=0; (device=(DEVRES *)GetNextRes(R_DEVICE, (RES *)device)); i++) { - if (i >= MAX_DEVICES) { - UnlockRes(); - Emsg1(M_ABORT, 0, _("Too many Device Resources. Max=%d\n"), MAX_DEVICES); - } - Dmsg1(90, "calling init_dev %s\n", device->device_name); - device->dev = init_dev(&shm->dev[i], device); - Dmsg1(10, "SD init done %s\n", device->device_name); - if (!device->dev) { - Emsg1(M_ERROR, 0, _("Could not initialize %s\n"), device->device_name); - } - if (device->cap_bits & CAP_ALWAYSOPEN) { - Dmsg1(20, "calling open_device %s\n", device->device_name); - if (!open_device(device->dev)) { - Emsg1(M_ERROR, 0, _("Could not open device %s\n"), device->device_name); - } - } - if (device->cap_bits & CAP_AUTOMOUNT && device->dev && - device->dev->state & ST_OPENED) { - DEV_BLOCK *block; - JCR *jcr; - block = new_block(device->dev); - jcr = new_jcr(sizeof(JCR), stored_free_jcr); - switch (read_dev_volume_label(jcr, device->dev, block)) { - case VOL_OK: - break; - default: - Emsg1(M_WARNING, 0, _("Could not mount device %s\n"), device->device_name); - break; - } - free_jcr(jcr); - free_block(block); - } - } - UnlockRes(); - device = NULL; + if (pthread_create(&thid, NULL, device_allocation, NULL) != 0) { + Emsg1(M_ABORT, 0, _("Unable to create thread. ERR=%s\n"), strerror(errno)); + } + - set_thread_concurrency(me->max_concurrent_jobs * 2 + - 4 /* watch dog + servers + misc */); start_watchdog(); /* start watchdog thread */ @@ -312,6 +274,53 @@ static void check_config() working_directory = me->working_directory; } +/* + * We are started as a separate thread. The + * resources are alread locked. + */ +static void *device_allocation(void *arg) +{ + int i; + DEVRES *device; + + pthread_detach(pthread_self()); + + /* LockRes() alread done */ + for (device=NULL,i=0; (device=(DEVRES *)GetNextRes(R_DEVICE, (RES *)device)); i++) { + Dmsg1(90, "calling init_dev %s\n", device->device_name); + device->dev = init_dev(NULL, device); + Dmsg1(10, "SD init done %s\n", device->device_name); + if (!device->dev) { + Emsg1(M_ERROR, 0, _("Could not initialize %s\n"), device->device_name); + } + if (device->cap_bits & CAP_ALWAYSOPEN) { + Dmsg1(20, "calling open_device %s\n", device->device_name); + if (!open_device(device->dev)) { + Emsg1(M_ERROR, 0, _("Could not open device %s\n"), device->device_name); + } + } + if (device->cap_bits & CAP_AUTOMOUNT && device->dev && + device->dev->state & ST_OPENED) { + DEV_BLOCK *block; + JCR *jcr; + block = new_block(device->dev); + jcr = new_jcr(sizeof(JCR), stored_free_jcr); + switch (read_dev_volume_label(jcr, device->dev, block)) { + case VOL_OK: + break; + default: + Emsg1(M_WARNING, 0, _("Could not mount device %s\n"), device->device_name); + break; + } + free_jcr(jcr); + free_block(block); + } + } + UnlockRes(); + return NULL; +} + + /* Clean up and then exit */ void terminate_stored(int sig) { @@ -346,10 +355,6 @@ void terminate_stored(int sig) term_msg(); close_memory_pool(); - if (shm) { - free(shm); - } - sm_dump(False); /* dump orphaned buffers */ exit(1); } diff --git a/bacula/src/stored/stored.h b/bacula/src/stored/stored.h index 022f3ee62f..28a1dcf859 100644 --- a/bacula/src/stored/stored.h +++ b/bacula/src/stored/stored.h @@ -40,19 +40,6 @@ #define uLongf uint32_t #endif -/* **** FIXME make this dynamic ****/ -#define MAX_DEVICES 20 - -/* - * Old shared memory buffer. Shared memory no longer used, - * so this just acts as a global. - */ -struct s_shm { - long VolSessionId; - long VolSessionTime; - DEVICE dev[MAX_DEVICES]; -}; - extern char errmsg[]; /* general error message */ #endif /* __STORED_H_ */ diff --git a/bacula/src/version.h b/bacula/src/version.h index 6855937499..031ebb90cd 100644 --- a/bacula/src/version.h +++ b/bacula/src/version.h @@ -1,8 +1,8 @@ /* */ -#define VERSION "1.27" +#define VERSION "1.28" #define VSTRING "1" -#define DATE "28 November 2002" -#define LSMDATE "28Nov02" +#define DATE "2 December 2002" +#define LSMDATE "02Dec02" /* Debug flags */ #define DEBUG 1