]> git.sur5r.net Git - bacula/bacula/blobdiff - regress/scripts/functions
Big backport from Enterprise
[bacula/bacula] / regress / scripts / functions
index 3a38be04af1ac8fde9911d1760d39f7a581cc4e9..2722e42d91274d466325a3c8253ee42c1f5ebcf5 100644 (file)
 #
+# Copyright (C) 2000-2017 Kern Sibbald
+# License: BSD 2-Clause; see file LICENSE-FOSS
+#
 # A set of useful functions to be sourced in each test
 #
 
+. ./config
+
+check_encoding()
+{
+   if ${bin}/bacula-dir -d50 -t -c ${conf}/bacula-dir.conf 2>&1 | grep 'Wanted SQL_ASCII, got UTF8' >/dev/null ; then
+       echo "Found database encoding problem, please modify the database encoding (SQL_ASCII)"
+       exit 1
+   fi
+}
 
 start_test()
 {
-   # Turn off email
-   outf="tmp/sed_tmp"
-   echo "s%  mail =%# mail = %g" >${outf}
-   echo "s%  operator =%# operator =%g" >>${outf}
-   cp bin/bacula-dir.conf tmp/1
-   sed -f ${outf} tmp/1 >bin/bacula-dir.conf
+   check_encoding
+   rm -rf ${working}/@*
+   variant_name=""
+   devicetype=0
+   # If no or dummy smtp_host, use dummy email handler
+   if [ x${SMTP_HOST} = x -o x${SMTP_HOST} = xdummy ]; then
+      cp scripts/dummy_bsmtp bin/bsmtp
+      chmod 755 bin/bsmtp
+   else
+      # Deactivate all email
+      outf="tmp/sed_tmp2"
+      echo "s%  mail =%# mail = %g" >${outf}
+      echo "s%  operator =%# operator =%g" >>${outf}
+      cp ${conf}/bacula-dir.conf ${tmp}/1
+      sed -f ${outf} ${tmp}/1 > ${conf}/bacula-dir.conf
+   fi
+   echo $TestName | grep aligned > /dev/null
+   if [ $? -eq 0 -o x$FORCE_ALIGNED = xyes ]; then
+      make -C build/src/stored install-aligned > /dev/null 2>&1
+      $bperl -e 'add_attribute("$conf/bacula-sd.conf", "Device Type", "Aligned", "Device")'
+      $bperl -e 'add_attribute("$conf/bacula-sd.conf", "Plugin Directory", "$plugins", "Storage")'
+      variant_name="Aligned"
+      devicetype=`expr $devicetype + 1`
+   fi
+   if [ x$FORCE_CLOUD = xyes ]; then
+      make -C build/src/stored install-cloud > /dev/null 2>&1
+      grep DummyCloud $conf/bacula-sd.conf > /dev/null
+      if [ $? != 0 ]; then
+         $bperl -e 'extract_resource("$rscripts/test-bacula-sd.conf", "Cloud", "DummyCloud")' >> $conf/bacula-sd.conf
+      fi
+      $bperl -e 'add_attribute("$conf/bacula-sd.conf", "Device Type", "Cloud", "Device")'
+      $bperl -e 'add_attribute("$conf/bacula-sd.conf", "Cloud", "DummyCloud", "Device")'
+      $bperl -e 'add_attribute("$conf/bacula-sd.conf", "Plugin Directory", "$plugins", "Storage")'
+      mkdir -p $tmp/cloud
+      if [ "$CLOUD_HOSTNAME" ]; then
+          $bperl -e "add_attribute('$conf/bacula-sd.conf', 'HostName', '\"$CLOUD_HOSTNAME\"', 'Cloud')"
+      fi
+      if [ "$CLOUD_BUCKETNAME" ]; then
+          $bperl -e "add_attribute('$conf/bacula-sd.conf', 'BucketName', '\"$CLOUD_BUCKETNAME\"', 'Cloud')"
+      fi
+      if [ "$CLOUD_ACCESSKEY" ]; then
+          $bperl -e "add_attribute('$conf/bacula-sd.conf', 'AccessKey', '\"$CLOUD_ACCESSKEY\"', 'Cloud')"
+      fi
+      if [ "$CLOUD_SECRETKEY" ]; then
+          $bperl -e "add_attribute('$conf/bacula-sd.conf', 'SecretKey', '\"$CLOUD_SECRETKEY\"', 'Cloud')"
+      fi
+      if [ "$CLOUD_REGION" ]; then
+          $bperl -e "add_attribute('$conf/bacula-sd.conf', 'Region', '\"$CLOUD_REGION\"', 'Cloud')"
+      fi
+      if [ "$CLOUD_PROTOCOL" ]; then
+          $bperl -e "add_attribute('$conf/bacula-sd.conf', 'Protocol', '$CLOUD_PROTOCOL', 'Cloud')"
+      fi
+      if [ "$CLOUD_URISTYLE" ]; then
+          $bperl -e "add_attribute('$conf/bacula-sd.conf', 'UriStyle', '$CLOUD_URISTYLE', 'Cloud')"
+      fi
+      if [ "$CLOUD_DRIVER" ]; then
+          driver=`echo $CLOUD_DRIVER | tr 'A-Z' 'a-z'`
+          # FakeS3 is useful but keeps big parts in memory. So we must limit parts.
+          if [ "$driver" = "fakes3" ]; then
+             if [ "$CLOUD_MAXIMUM_PART_SIZE" = "" ]; then
+                CLOUD_MAXIMUM_PART_SIZE=50MB
+             fi
+             CLOUD_DRIVER=S3
+          fi
+          $bperl -e "add_attribute('$conf/bacula-sd.conf', 'Driver', '\"$CLOUD_DRIVER\"', 'Cloud')"
+      fi
+      if [ "$CLOUD_MAXIMUM_PART_SIZE" ]; then
+          $bperl -e "add_attribute('$conf/bacula-sd.conf', 'MaximumPartSize', '$CLOUD_MAXIMUM_PART_SIZE', 'Device')"
+      fi
+
+      variant_name="Cloud"
+      devicetype=`expr $devicetype + 1`
+   fi
+   if [ x$FORCE_SDCALLS = xyes ]; then
+      $bperl -e 'add_attribute("$conf/bacula-dir.conf", "SD Calls Client", "yes", "Client")'
+      variant_name="${variant_name} SDcall"
+   fi
+   if [ $devicetype -gt 1 ]; then
+       echo "ERROR: Found multiple FORCE_XXX switches"
+       set | awk '/^FORCE_/ { print $0 }'
+       exit 1
+   fi
+   ./test_starttime
    echo " "
    echo " "
-   echo " === Starting $TestName at `date +%R:%S` ==="
-   echo " === Starting $TestName at `date +%R:%S` ===" >>working/log
+   echo " === Start $TestName ${variant_name} at `date +%R:%S` ==="
+   echo " === Start $TestName ${variant_name} at `date +%R:%S` ===" >> ${working}/log
    echo " "
+   export TestName
    export zstat
+   export estat
+   estat=0
+   zstat=0
+   bstat=0
+   rstat=0
+   vstat=0
+   dstat=0
+}
+
+# Change setup so that we run with shared storage plugin
+setup_shstore()
+{
+if test x${USE_SHSTORE} = xyes ; then
+   require_tape_drive
+   /bin/cp -f ${rscripts}/bacula-sd-lock-changer.conf ${scripts}/bacula-sd.conf
+   scripts/cleanup-tape
+
+   # install the shstore plugin
+   rm -f bin/plugins/shstore-sd.so
+   make -C build/src/plugins/sd install-shstore-plugin >/dev/null
+fi
+}
+
+require_cloud()
+{
+    if [ "$FORCE_CLOUD" != yes ]; then
+       echo "$TestName can run only with FORCE_CLOUD"
+       exit 0
+    fi
 }
 
 require_root()
@@ -34,6 +153,16 @@ fi
 
 require_tape_drive() 
 {
+#  tape not comptible with aligned
+if test x$FORCE_ALIGNED = xyes; then
+   echo "$TestName test not compatible with FORCE_ALIGNED."
+   exit 0
+fi
+#  tape not comptible with cloud
+if test x$FORCE_CLOUD = xyes; then
+   echo "$TestName test not compatible with FORCE_CLOUD."
+   exit 0
+fi
 if test x${TAPE_DRIVE} = x/dev/null ; then
    echo "$TestName test needs a tape drive, but has none."
    exit 1
@@ -58,17 +187,54 @@ fi
 
 require_vtape() 
 {
+#  vtape not comptible with aligned
+if test x$FORCE_ALIGNED = xyes; then
+   echo "$TestName test not compatible with FORCE_ALIGNED."
+   exit 0
+fi
+#  vtape not comptible with cloud
+if test x$FORCE_CLOUD = xyes; then
+   echo "$TestName test not compatible with FORCE_CLOUD."
+   exit 0
+fi
 if test x${USE_VTAPE} = x ; then
    echo "$TestName test needs the vtape driver."
    exit 0
 fi
 }
 
+require_disk() 
+{
+#  vtape not comptible with aligned
+if test x$FORCE_ALIGNED = xyes; then
+   echo "$TestName test not compatible with FORCE_ALIGNED."
+   exit 0
+fi
+#  vtape not comptible with cloud
+if test x$FORCE_CLOUD = xyes; then
+   echo "$TestName test not compatible with FORCE_CLOUD."
+   exit 0
+fi
+if test x${USE_VTAPE} = xyes ; then
+   echo "$TestName test not compatible with FORCE_VTAPE."
+   exit 0
+fi
+}
+
+require_changer() 
+{
+#  disk changer not comptible with cloud
+if test x$FORCE_CLOUD = xyes; then
+   echo "$TestName test not compatible with FORCE_CLOUD."
+   exit 0
+fi
+}
+
 require_linux()
 {
 os=`uname`
 if [ $os != 'Linux' ]; then
-   echo "This test $TestName runs only on Linux"
+   echo "$TestName test runs only on Linux"
    exit 0
 fi
 }
@@ -93,6 +259,10 @@ set_debug()
 
 print_debug()
 {
+   echo $* | grep ERROR > /dev/null
+   if test $? -eq 0; then
+     echo $* >> $tmp/err.log
+   fi
    if test "$debug" -eq 1 ; then
      echo $*
    fi
@@ -105,64 +275,84 @@ check_files_written()
     FILES=`awk '/FD Files Written:/ { last=$4 } END { print last }' $LOG`
 
     if [ "$NB" != "$FILES" ]; then
-        print_debug "Expect $NB files, get $FILES"
+        print_debug "ERROR: Expect $NB files, get $FILES"
         bstat=2
     fi
 }
 
-check_duration()
+################################################################
+# Get information from logs
+get_mig_info()
 {
-    LOG=$1
-    TIME=$2
-    T=`awk 'BEGIN {t["secs"]=1;t["sec"]=1;t["min"]=60;t["mins"]=60}; /Elapsed time:/ { last=$3*t[$4] } END { print last }' $LOG`
+    # Prev Backup JobId
+    JOBID=$1
+    LOG=$2
+    RET=`awk -F: "BEGIN { jobid=$JOBID } "'/Prev Backup JobId/ { cjid=$2 } /New Backup JobId/  { if (cjid == jobid) { print $2 } }' $LOG`
+}
 
-    if [ "$T" -gt $TIME ]; then
-        print_debug "Expect less than $TIME sec, get $T"
-        bstat=2
-    fi
+get_duration()
+{
+   LOG=$1
+   RET=`awk 'BEGIN {t["secs"]=1;t["sec"]=1;t["min"]=60;t["mins"]=60}; /Elapsed time:/ { last=$3*t[$4] } END { print last }' $LOG`
+}
+
+check_duration()
+{
+   LOG=$1
+   TIME=$2
+   OP=${3:-gt}
+
+   get_duration $LOG
+   if [ "$RET" -$OP "$TIME" ]; then
+       print_debug "Error, got $RET $OP $TIME sec on $LOG"
+       bstat=2
+   fi
 }
 
 run_bacula()
 {
    debug_wait
    zstat=0
+   estat=0
    if test "$debug" -eq 1 ; then
-     bin/bacula-ctl-sd start
-     bin/bacula-ctl-fd start $1
-     bin/bacula-ctl-dir start
-     cat tmp/bconcmds | bin/bconsole -c bin/bconsole.conf
+
+     ${scripts}/bacula-ctl-sd start -m
+     ${scripts}/bacula-ctl-fd start -m $1
+     ${scripts}/bacula-ctl-dir start -m
+     cat ${tmp}/bconcmds | ${bin}/bconsole -c ${conf}/bconsole.conf
      return $?
    else
-     bin/bacula start 2>&1 >/dev/null
-     cat tmp/bconcmds | bin/bconsole -c bin/bconsole.conf  2>&1 >/dev/null
+     ${scripts}/bacula start >/dev/null 2>&1
+     cat ${tmp}/bconcmds | ${bin}/bconsole -c ${conf}/bconsole.conf >/dev/null 2>&1
      return $?
    fi
 }
 
 run_bconsole()
 {
+   bconsole_file=${1:-${tmp}/bconcmds}
    if test "$debug" -eq 1 ; then
-     cat tmp/bconcmds | bin/bconsole -c bin/bconsole.conf
+     cat $bconsole_file | ${bin}/bconsole -c ${conf}/bconsole.conf
    else
-     cat tmp/bconcmds | bin/bconsole -c bin/bconsole.conf  2>&1 >/dev/null
+     cat $bconsole_file | ${bin}/bconsole -c ${conf}/bconsole.conf  2>&1 >/dev/null
    fi
 }
 
 run_btape()
 {
    if test "$debug" -eq 1 ; then
-     cat tmp/bconcmds | bin/btape -c bin/bacula-sd.conf DDS-4 | tee tmp/log1.out
+     cat ${tmp}/bconcmds | ${bin}/btape -c ${conf}/bacula-sd.conf tape | tee ${tmp}/log1.out
    else
-     cat tmp/bconcmds | bin/btape -c bin/bacula-sd.conf DDS-4 2>&1 >tmp/log1.out
+     cat ${tmp}/bconcmds | ${bin}/btape -c ${conf}/bacula-sd.conf tape >${tmp}/log1.out 2>&1
    fi
 }
 
 run_bscan()
 {
    if test "$debug" -eq 1 ; then
-      bin/bscan $* | tee tmp/log.out
+      ${bin}/bscan $* | tee ${tmp}/log.out
    else
-      bin/bscan $* 2>&1 >/dev/null
+      ${bin}/bscan $* 2>&1 >/dev/null
    fi
 }
 
@@ -184,12 +374,49 @@ bscan_libdbi()
 
 stop_bacula()
 {
-   bin/bacula stop 2>&1 >/dev/null
+   if [ "$CHECK_JOBMEDIA" != 0 ]; then
+      $bperl -e 'check_jobmedia()'
+      if [ $? -ne 0 ]; then
+         echo " "
+         echo "  !!!!! $TestName ${variant_name} failed!!! `date +%R:%S` !!!!! "
+         echo "     JobMedia is corrupted"
+         echo " " >>test.out
+         echo " " >>test.out
+         echo "  !!!!! $TestName ${variant_name} failed!!! `date +%R:%S` !!!!! " >>test.out
+         echo "     JobMedia is corrupted" >>test.out
+         echo " "
+         exit 1
+      fi
+   fi
+   if [ "$FORCE_CLOUD" = yes ]; then
+      $bperl -e 'check_parts()'
+      if [ $? -ne 0 ]; then
+         estat=1
+      fi
+   fi
+   if test "$debug" -eq 1 ; then
+      ${scripts}/bacula stop
+   else
+      ${scripts}/bacula stop 2>&1 >/dev/null
+   fi
+   # Any File/Dir with @ as the first character is a lock file
+   ls ${working}/@* 2>/dev/null 1>/dev/null
+   if test $? -eq 0 ; then
+      echo " "
+      echo "  !!!!! $TestName ${variant_name} failed!!! `date +%R:%S` !!!!! "
+      echo "     SCSI lock still set"
+      echo " " >>test.out
+      echo " " >>test.out
+      echo "  !!!!! $TestName ${variant_name} failed!!! `date +%R:%S` !!!!! " >>test.out
+      echo "     SCSI lock still set" >>test.out
+      echo " "
+      exit 1
+   fi
 }
 
 check_for_zombie_jobs()
 {
-   scripts/check_for_zombie_jobs $*
+   ${rscripts}/check_for_zombie_jobs $*
 }
 
 change_jobname()
@@ -201,102 +428,279 @@ change_jobname()
       oldname=$1
       newname=$2
    fi
-   rm -f bin/1
-   mv bin/bacula-dir.conf bin/1
-   echo "s%${oldname}%${newname}%g" >tmp/1
-   sed -f tmp/1 bin/1 >bin/bacula-dir.conf
+   rm -f $tmp/1 $tmp/2
+   mv ${conf}/bacula-dir.conf $tmp/1
+   echo "s%${oldname}%${newname}%g" >$tmp/2
+   sed -f $tmp/2 $tmp/1 >$conf/bacula-dir.conf
 #  echo "Job ${oldname} changed to ${newname}"
 }
 
 check_two_logs()
 {
-   grep "^  Termination: *Backup OK" tmp/log1.out 2>&1 >/dev/null
-   bstat=${bstat-$?}
-   grep "^  Termination: .*Backup Error" tmp/log1.out 2>&1 >/dev/null
+   bstat=${bstat:-99}   # We must find at least one job in log1.out
+   grep "^  Termination: *Backup OK" ${tmp}/log1.out 2>&1 >/dev/null
+   if test $? -ne 0; then
+      bstat=2
+   fi
+   grep "^  Termination: .*Backup Error" ${tmp}/log1.out 2>&1 >/dev/null
    if test $? -eq 0; then
       bstat=2
    fi
-   grep "^  Termination: *Restore OK" tmp/log2.out 2>&1 >/dev/null
-   rstat=${rstat-$?}
-   grep "^  Termination: .*Restore Error" tmp/log2.out 2>&1 >/dev/null
+   # Do not check for restart jobs in restart tests
+   echo "$TestName" | grep restart.*-test > /dev/null
+   if [ $? -ne 0 ]; then
+      grep "^  Termination: .*Backup failed" ${tmp}/log1.out 2>&1 >/dev/null
+      if test $? -eq 0; then
+         bstat=3
+      fi
+   fi
+   grep "^  Termination: *Restore OK" ${tmp}/log2.out 2>&1 >/dev/null
+   rstat=${rstat:-$?}
+   grep "^  Termination: .*Restore Error" ${tmp}/log2.out 2>&1 >/dev/null
    if test $? -eq 0; then
       rstat=2
    fi
-   grep "^  Termination: *Restore OK -- warning file count mismatch" tmp/log2.out 2>&1 >/dev/null
+   grep "^  Termination: *Restore OK -- warning file count mismatch" ${tmp}/log2.out 2>&1 >/dev/null
    if test $? -eq 0; then
       rstat=3
    fi
-   grep "^  Termination: .*Verify Differences" tmp/log2.out 2>&1 >/dev/null
+   grep "^  Termination: .*Verify Differences" ${tmp}/log2.out 2>&1 >/dev/null
+   if test $? -eq 0; then
+      vstat=4
+   fi
+   grep "Encoding error for database" ${tmp}/log1.out > /dev/null
    if test $? -eq 0; then
-      rstat=4
+      print_debug "Found database encoding error"
+      bstat=2
+   fi
+   grep "Orphaned buffer" ${tmp}/log1.out ${tmp}/log2.out
+   if test $? -eq 0; then
+      print_debug "Found orphaned buffers"
+      estat=1
+   fi
+   if [ x$REGRESS_CHECK_CORRUPTION = xyes ]; then
+       perl -Mscripts::functions -e "check_volumes('$tmp/log1.out', '$tmp/log2.out')" 2>&1 >/dev/null
+       if test $? -ne 0; then
+           print_debug "Found volume corruption"
+           estat=1
+       fi
+   fi
+}
+
+die_test()
+{
+   code=$1
+   msg=$2
+
+   print_debug $msg
+   stop_bacula
+
+   if test "$code" -gt "$estat" ; then
+      estat=$code
+   fi
+   end_test
+   exit 1
+}
+
+dtitle()
+{
+   if test "$debug" -eq 1 ; then
+      echo "============================================================"
+      echo $*
+      echo "============================================================"
    fi
 }
 
+dmsg()
+{
+   test "$debug" -eq 1 && echo $*
+}
+
+check_size_interval()
+{
+   # if mi == -1 then then ignore mi (idem for ma)
+   mi=$1        
+   value=$2
+   ma=$3
+   
+   test \( -1 -eq $mi -o $mi -le $value \) -a \( -1 -eq $ma -o $value -le $ma \) 
+}
+
 check_restore_diff()
 {
    if test "$debug" -eq 1 ; then
-      diff -ur build tmp/bacula-restores${cwd}/build
+      $rscripts/diff.pl -notop -s ${src} -d ${tmp}/bacula-restores${src} 2>&1 >/tmp/d$$
+      if test $? -ne 0; then
+         dstat=1
+         cat /tmp/d$$
+         ls -Rl ${src}
+         ls -Rl ${tmp}/bacula-restores${src}
+      fi
+      rm -f /tmp/d$$
+      diff -ur ${src} ${tmp}/bacula-restores${src}
    else 
-      diff -ur build tmp/bacula-restores${cwd}/build 2>&1 >/dev/null
+      $rscripts/diff.pl -notop -s ${src} -d ${tmp}/bacula-restores${src} 2>&1 >/dev/null
+      if test $? -ne 0; then
+         dstat=1
+      fi
+      diff -ur ${src} ${tmp}/bacula-restores${src} 2>&1 >/dev/null
+   fi
+   if test $? -ne 0; then
+     dstat=1
    fi
-   dstat=$?
 }
 
+check_restore_bin_diff()
+{
+   if test "$debug" -eq 1 ; then
+      $rscripts/diff.pl -notop -s ${bin} -d ${tmp}/bacula-restores${bin} 2>&1 >/tmp/d$$
+      if test $? -ne 0; then
+         dstat=1
+         cat /tmp/d$$
+         ls -Rl ${src}
+         ls -Rl ${tmp}/bacula-restores${src}
+      fi
+      rm -f /tmp/d$$
+      diff -ur ${bin} ${tmp}/bacula-restores${bin}
+   else 
+      $rscripts/diff.pl -notop -s ${bin} -d ${tmp}/bacula-restores${bin} 2>&1 >/dev/null
+      if test $? -ne 0; then
+         dstat=1
+      fi
+      diff -ur ${bin} ${tmp}/bacula-restores${bin} 2>&1 >/dev/null
+   fi
+   if test $? -ne 0; then
+      dstat=1
+   fi
+}
+
+
 check_restore_tmp_build_diff()
 {
    if test "$debug" -eq 1 ; then
-      diff -ur tmp/build tmp/bacula-restores${cwd}/tmp/build
+      $rscripts/diff.pl -notop -s ${tmpsrc} -d ${tmp}/bacula-restores${tmpsrc} 2>&1 >/tmp/d$$
+      if test $? -ne 0; then
+         dstat=1
+         cat /tmp/d$$
+         ls -Rl ${src}
+         ls -Rl ${tmp}/bacula-restores${src}
+      fi
+      rm -f /tmp/d$$
+      diff -ur ${tmpsrc} ${tmp}/bacula-restores${tmpsrc}
    else
-      diff -ur tmp/build tmp/bacula-restores${cwd}/tmp/build 2>&1 >/dev/null
+      $rscripts/diff.pl -notop -s ${tmpsrc} -d ${tmp}/bacula-restores${tmpsrc} 2>&1 >/dev/null
+      if test $? -ne 0; then
+         dstat=1
+      fi
+      diff -ur ${tmpsrc} ${tmp}/bacula-restores${tmpsrc} 2>&1 >/dev/null
+   fi
+   if test $? -ne 0; then
+      dstat=1
    fi
-   dstat=$?
 }
 
+# bstat is backup error
+# dstat is diff difference
+# estat is special error status (shown by print_debug message)
+# rstat is restore status
+# zstat is zombie job(s)
+# vstat is verify status
+#
 end_test()
 {
-   if [ $zstat != 0 ] ; then
+   if [ x$notracedump != xyes ]; then 
+      cat ${working}/bacula.*.traceback 2>/dev/null
+      cp -f  ${working}/bacula.*.traceback ${dumps} 2>/dev/null
+   fi
+   if [ -f $tmp/err.log ]; then
+      cat $tmp/err.log
+   fi
+   d=`./test_duration`
+   t=`date +%R:%S`
+   if [ $estat != 0 ] ; then
       echo " "
-      echo "  !!!!! $TestName failed!!! `date +%R:%S` !!!!! "
-      echo "  !!!!! $TestName failed!!! `date +%R:%S` !!!!! " >>test.out
-      echo "   Status: zombie=$zstat backup=$bstat restore=$rstat diff=$dstat" >>test.out
+      echo "  !!!!! $TestName ${variant_name} failed!!! $t $d !!!!! "
+      echo "     Status: estat=$estat zombie=$zstat backup=$bstat restore=$rstat diff=$dstat verify=$vstat"
+      echo " " >>test.out
+      echo "  !!!!! $TestName ${variant_name} failed!!! $t $d !!!!! " >>test.out
+      echo "     Status: estat=$estat zombie=$zstat backup=$bstat restore=$rstat diff=$dstat verify=$vstat" >>test.out
       echo " "
       exit 1
    fi
-   if [ $dstat != 0 -o $bstat != 0 -o $rstat != 0 ] ; then
+   if [ $zstat != 0 ] ; then
       echo " "
+      echo "  !!!!! $TestName ${variant_name} failed!!! $t $d !!!!! "
+      echo "     Status: zombie=$zstat backup=$bstat restore=$rstat diff=$dstat verify=$vstat"
+      echo " " >>test.out
+      echo "  !!!!! $TestName ${variant_name} failed!!! $t $d !!!!! " >>test.out
+      echo "     Status: zombie=$zstat backup=$bstat restore=$rstat diff=$dstat verify=$vstat" >>test.out
       echo " "
-      echo "  !!!!! $TestName failed!!! `date +%R:%S` !!!!! "
-      echo "  !!!!! $TestName failed!!! `date +%R:%S` !!!!! " >>test.out
-      if [ $bstat != 0 -o $rstat != 0 ] ; then
-         echo "  !!!!! Bad termination status       !!!!! "
-         echo "  !!!!! Bad termination status       !!!!! " >>test.out
+      exit 1
+   fi
+   if [ $dstat != 0 -o $bstat != 0 -o $rstat != 0 -o $vstat != 0 ] ; then
+      echo " "
+      echo "  !!!!! $TestName ${variant_name} failed!!! $t $d !!!!! "
+      echo "     Status: zombie=$zstat backup=$bstat restore=$rstat diff=$dstat verify=$vstat"
+      echo " " >>test.out
+      echo "  !!!!! $TestName ${variant_name} failed!!! $t $d !!!!! " >>test.out
+      echo "     Status: zombie=$zstat backup=$bstat restore=$rstat diff=$dstat verify=$vstat" >>test.out
+      if [ $bstat != 0 -o $rstat != 0 -o $vstat != 0 ] ; then
+         echo "     !!! Bad termination status       !!! "
+         echo "     !!! Bad termination status       !!! " >>test.out
       else
-         echo "  !!!!! Restored files differ          !!!!! "
-         echo "  !!!!! Restored files differ          !!!!! " >>test.out
+         echo "     !!! Restored files differ        !!! "
+         echo "     !!! Restored files differ        !!! " >>test.out
       fi
-      echo "   Status: backup=$bstat restore=$rstat diff=$dstat"
-      echo "   Status: backup=$bstat restore=$rstat diff=$dstat" >>test.out
+      echo "     Status: backup=$bstat restore=$rstat diff=$dstat verify=$vstat"
+      echo "     Status: backup=$bstat restore=$rstat diff=$dstat verify=$vstat" >>test.out
+      echo "     Test owner of $SITE_NAME is $EMAIL"
+      echo "     Test owner of $SITE_NAME is $EMAIL" >>test.out
+      echo " " >>test.out
       echo " "
       exit 1
    else
-      echo "  ===== $TestName OK `date +%R:%S` ===== "
-      echo "  ===== $TestName OK `date +%R:%S` ===== " >>test.out
+      # KES -- remove variant to make line fit in 80 chars
+      echo "  ===== End $TestName OK $t $d ===== "
+      echo "  ===== End $TestName OK $t $d ===== " >>test.out
       if test "$debug" -eq 0 ; then
-         scripts/cleanup
+         ${rscripts}/cleanup
       fi
    fi
 }
 
 copy_tape_confs()
 {
-   scripts/copy-tape-confs
-   scripts/cleanup-tape
+   ${rscripts}/copy-tape-confs
+   ${rscripts}/cleanup-tape
 }
 
 copy_test_confs()
 {
-   scripts/copy-test-confs
-   scripts/cleanup
+   ${rscripts}/copy-test-confs
+   ${rscripts}/cleanup
+}
+
+disable_plugins()
+{
+   for i in ${conf}/bacula-fd.conf; do
+      sed 's/Plugin/#Plugin/' $i > $tmp/1
+      cp -f $tmp/1 $i
+   done
+}
+
+update_win32()
+{
+   if [ -d $cwd/build/src/win32/release32   \
+     -o -d $cwd/build/src/win32/release64 ] \
+   || [ -d $cwd/release32 -o -d $cwd/release64 ] \
+   || [ -d $cwd/../bacula/src/win32/release32 \
+     -o -d $cwd/../bacula/src/win32/release64 ]
+   then
+       echo "Try to upgrade the FileDaemon:\t"
+       wget -qO - "$WIN32_ADDR:8091/install"
+   else 
+       echo "Windows binaries not found, skiping upgrade"
+   fi
 }
 
 debug_wait()
@@ -314,6 +718,9 @@ init_slot()
    if test -n "$DRIVE" -a -n "$SLOT"; then
       if test ! -c $DRIVE -a ! -b $DRIVE -a x$USE_VTAPE != x ; then
          dir=`dirname $DRIVE`
+         if [ ! -d "$dir" ]; then
+            mkdir -p "$dir"
+         fi
          touch $dir/slot$SLOT
       fi
    fi
@@ -339,10 +746,10 @@ rewind_drive()
 load_slot1() 
 {
 # Get a tape from slot1
-slot=`bin/$MTX ${AUTOCHANGER} loaded 0 ${TAPE_DRIVE} $DRIVE1`
+slot=`${scripts}/$MTX ${AUTOCHANGER} loaded 0 ${TAPE_DRIVE} $DRIVE1`
 case $slot in
  0)
-    bin/$MTX ${AUTOCHANGER} load $SLOT1 ${TAPE_DRIVE} $DRIVE1
+    ${scripts}/$MTX ${AUTOCHANGER} load $SLOT1 ${TAPE_DRIVE} $DRIVE1
     slot=$SLOT1
     ;;
  $SLOT1)
@@ -350,8 +757,8 @@ case $slot in
     ;;
  *)
     rewind_drive ${TAPE_DRIVE}
-    bin/$MTX ${AUTOCHANGER} unload $slot  ${TAPE_DRIVE} $DRIVE1
-    bin/$MTX ${AUTOCHANGER} load   $SLOT1 ${TAPE_DRIVE} $DRIVE1
+    ${scripts}/$MTX ${AUTOCHANGER} unload $slot  ${TAPE_DRIVE} $DRIVE1
+    ${scripts}/$MTX ${AUTOCHANGER} load   $SLOT1 ${TAPE_DRIVE} $DRIVE1
     slot=$SLOT1
     ;;
 esac
@@ -366,17 +773,17 @@ load_other_slot()
 rewind_drive ${TAPE_DRIVE}
 case $1 in
  0)
-    bin/${AUTOCHANGER_SCRIPT} ${AUTOCHANGER} load $SLOT1 ${TAPE_DRIVE} $DRIVE1
+    ${scripts}/${AUTOCHANGER_SCRIPT} ${AUTOCHANGER} load $SLOT1 ${TAPE_DRIVE} $DRIVE1
     slot=1
     ;;
  $SLOT1)
-    bin/${AUTOCHANGER_SCRIPT} ${AUTOCHANGER} unload $1 ${TAPE_DRIVE} $DRIVE1
-    bin/${AUTOCHANGER_SCRIPT} ${AUTOCHANGER} load $SLOT2 ${TAPE_DRIVE} $DRIVE1
+    ${scripts}/${AUTOCHANGER_SCRIPT} ${AUTOCHANGER} unload $1 ${TAPE_DRIVE} $DRIVE1
+    ${scripts}/${AUTOCHANGER_SCRIPT} ${AUTOCHANGER} load $SLOT2 ${TAPE_DRIVE} $DRIVE1
     slot=2
     ;;
  $SLOT2)
-    bin/${AUTOCHANGER_SCRIPT} ${AUTOCHANGER} unload $1 ${TAPE_DRIVE} $DRIVE1
-    bin/${AUTOCHANGER_SCRIPT} ${AUTOCHANGER} load $SLOT1 ${TAPE_DRIVE} $DRIVE1
+    ${scripts}/${AUTOCHANGER_SCRIPT} ${AUTOCHANGER} unload $1 ${TAPE_DRIVE} $DRIVE1
+    ${scripts}/${AUTOCHANGER_SCRIPT} ${AUTOCHANGER} load $SLOT1 ${TAPE_DRIVE} $DRIVE1
     slot=1
     ;;
  *)
@@ -401,23 +808,66 @@ fi
 db_name=${db_name:-"regress"}
 db_user=${db_user:-"regress"}
 db_password=${db_password:-""}
+working=${working:-"$cwd/working"}
+dumps=${dumps:-"$cwd/dumps"}
+bin=${bin:-"$cwd/bin"}
+
+# Bacula scripts
+scripts=${scripts:-"$cwd/bin"}
+
+# Bacula conf files
+conf=${conf:-"$cwd/bin"}
+
+# Regress scripts
+rscripts=${rscripts:-"$cwd/scripts"}
+
+tmp=${tmp:-"$cwd/tmp"}
+
+# Bacula source directory when copied here
+#  also build directory
+src=${src:-"$cwd/build"}
+
+# Temp source directory so we don't mess up $src
+tmpsrc=${tmpsrc:-"$cwd/tmp/build"}
+
+export bin
+export conf
+export working
+export dumps
+export scripts
+export rscripts
+export tmp
+export src
+export tmpsrc
+
+bperl="perl -Mscripts::functions"
+export bperl
+
+mkdir -p ${tmp}
+touch ${tmp}/dir.out ${tmp}/fd.out ${tmp}/sd.out
 
 CLIENT=${HOST}-fd
 
 if [ x$USE_VTAPE = xyes ]; then
-   mkdir -p $cwd/working/ach
+   mkdir -p $working/ach
    SLOT1=1
    SLOT2=2
-   TAPE_DRIVE=$cwd/working/ach/drive1
-   TAPE_DRIVE1=$cwd/working/ach/drive1
-   AUTOCHANGER=$cwd/working/ach/config
+   TAPE_DRIVE=$working/ach/drive0
+   TAPE_DRIVE1=$working/ach/drive1
+   AUTOCHANGER=$working/ach/config
    AUTOCHANGER_SCRIPT=disk-changer
    DRIVE1=0
    DRIVE2=1
+   cp /dev/null $working/ach/slot$SLOT1
+   cp /dev/null $working/ach/slot$SLOT2
    cat > $AUTOCHANGER <<EOF
-maxdrive=2
-maxslot=10
+maxdrive=8
+maxslot=80
 EOF
 fi
 
 AUTOCHANGER_SCRIPT=${AUTOCHANGER_SCRIPT:-mtx-changer}
+LD_LIBRARY_PATH=$bin:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH
+
+trap "{ estat=999; end_test; }" TERM