1 ################################################################
6 Bacula® - The Network Backup Solution
8 Copyright (C) 2000-2014 Free Software Foundation Europe e.V.
10 The main author of Bacula is Kern Sibbald, with contributions from
11 many others, a complete list can be found in the file AUTHORS.
13 You may use this file and others of this release according to the
14 license defined in the LICENSE file, which includes the Affero General
15 Public License, v3.0 ("AGPLv3") and some additional permissions and
16 terms pursuant to its AGPLv3 Section 7.
18 Bacula® is a registered trademark of Kern Sibbald.
22 package scripts::functions;
23 # Export all functions needed to be used by a simple
24 # perl -Mscripts::functions -e '' script
26 our @ISA = qw(Exporter);
28 our @EXPORT = qw(update_some_files create_many_files check_multiple_copies
29 update_client $HOST $BASEPORT add_to_backup_list
30 check_volume_size create_many_dirs cleanup start_bacula
31 stop_bacula get_resource set_maximum_concurrent_jobs get_time
32 add_attribute check_prune_list check_min_volume_size
33 check_max_volume_size $estat $bstat $rstat $zstat $cwd $bin
34 $scripts $conf $rscripts $tmp $working $dstat extract_resource
35 $db_name $db_user $db_password $src $tmpsrc
36 remote_init remote_config remote_stop remote_diff );
39 use File::Copy qw/copy/;
41 our ($cwd, $bin, $scripts, $conf, $rscripts, $tmp, $working, $estat, $dstat,
42 $bstat, $zstat, $rstat, $debug,
43 $REMOTE_CLIENT, $REMOTE_ADDR, $REMOTE_FILE, $REMOTE_PORT, $REMOTE_PASSWORD,
44 $REMOTE_STORE_ADDR, $REGRESS_DEBUG,
45 $db_name, $db_user, $db_password, $src, $tmpsrc, $HOST, $BASEPORT);
48 if ($estat || $rstat || $zstat || $bstat || $dstat) {
54 # start by loading the ./config file
56 if (! -f "./config") {
57 die "Could not find ./config file\n";
59 # load the ./config file in a subshell doesn't allow to use "env" to display all variable
60 open(IN, ". ./config; set |") or die "Could not run shell: $!\n";
61 while ( my $l = <IN> ) {
63 if ($l =~ /^([\w\d]+)='?([^']+)'?/) {
64 next if ($1 eq 'SHELLOPTS'); # is in read-only
65 ($envar,$enval) = ($1, $2);
66 $ENV{$envar} = $enval;
73 # set internal variable name and update environment variable
74 $ENV{db_name} = $db_name = $ENV{db_name} || 'regress';
75 $ENV{db_user} = $db_user = $ENV{db_user} || 'regress';
76 $ENV{db_password} = $db_password = $ENV{db_password} || '';
78 $ENV{bin} = $bin = $ENV{bin} || "$cwd/bin";
79 $ENV{tmp} = $tmp = $ENV{tmp} || "$cwd/tmp";
80 $ENV{src} = $src = $ENV{src} || "$cwd/src";
81 $ENV{conf} = $conf = $ENV{conf} || $bin;
82 $ENV{scripts} = $scripts = $ENV{scripts} || $bin;
83 $ENV{tmpsrc} = $tmpsrc = $ENV{tmpsrc} || "$cwd/tmp/build";
84 $ENV{working} = $working = $ENV{working} || "$cwd/working";
85 $ENV{rscripts} = $rscripts = $ENV{rscripts} || "$cwd/scripts";
86 $ENV{HOST} = $HOST = $ENV{HOST} || "localhost";
87 $ENV{BASEPORT} = $BASEPORT = $ENV{BASEPORT} || "8101";
88 $ENV{REGRESS_DEBUG} = $debug = $ENV{REGRESS_DEBUG} || 0;
89 $ENV{REMOTE_CLIENT} = $REMOTE_CLIENT = $ENV{REMOTE_CLIENT} || 'remote-fd';
90 $ENV{REMOTE_ADDR} = $REMOTE_ADDR = $ENV{REMOTE_ADDR} || undef;
91 $ENV{REMOTE_FILE} = $REMOTE_FILE = $ENV{REMOTE_FILE} || "/tmp";
92 $ENV{REMOTE_PORT} = $REMOTE_PORT = $ENV{REMOTE_PORT} || 9102;
93 $ENV{REMOTE_PASSWORD} = $REMOTE_PASSWORD = $ENV{REMOTE_PASSWORD} || "xxx";
94 $ENV{REMOTE_STORE_ADDR}=$REMOTE_STORE_ADDR=$ENV{REMOTE_STORE_ADDR} || undef;
96 $estat = $rstat = $bstat = $zstat = 0;
101 system("$rscripts/cleanup");
109 system("$bin/bacula start");
111 open(FP, ">$tmp/bcmd");
112 print FP "sql\ntruncate client_group;\ntruncate client_group_member;\nupdate Media set LocationId=0;\ntruncate location;\n\n";
114 system("cat $tmp/bcmd | $bin/bconsole >/dev/null");
121 system("$bin/bacula stop");
127 my ($file, $type, $name) = @_;
129 open(FP, $file) or die "Can't open $file";
130 my $content = join("", <FP>);
132 if ($content =~ m/(^$type {[^}]+?Name\s*=\s*"?$name"?[^}]+?^})/ms) {
142 my $ret = get_resource(@_);
148 sub check_min_volume_size
150 my ($size, @vol) = @_;
153 foreach my $v (@vol) {
154 if (! -f "$tmp/$v") {
155 print "ERR: $tmp/$v not accessible\n";
159 if (-s "$tmp/$v" < $size) {
160 print "ERR: $tmp/$v too small\n";
168 sub check_max_volume_size
170 my ($size, @vol) = @_;
173 foreach my $v (@vol) {
174 if (! -f "$tmp/$v") {
175 print "ERR: $tmp/$v not accessible\n";
179 if (-s "$tmp/$v" > $size) {
180 print "ERR: $tmp/$v too big\n";
188 sub add_to_backup_list
190 open(FP, ">>$tmp/file-list") or die "Can't open $tmp/file-list for update $!";
191 print FP join("\n", @_);
195 # update client definition for the current test
196 # it permits to test remote client
199 my ($new_passwd, $new_address, $new_port) = @_;
202 open(FP, "$conf/bacula-dir.conf") or die "can't open source $!";
203 open(NEW, ">$tmp/bacula-dir.conf.$$") or die "can't open dest $!";
204 while (my $l = <FP>) {
205 if (!$in_client && $l =~ /^Client {/) {
209 if ($in_client && $l =~ /Address/i) {
210 $l = "Address = $new_address\n";
213 if ($in_client && $l =~ /FDPort/i) {
214 $l = "FDPort = $new_port\n";
217 if ($in_client && $l =~ /Password/i) {
218 $l = "Password = \"$new_passwd\"\n";
221 if ($in_client && $l =~ /^}/) {
228 my $ret = copy("$tmp/bacula-dir.conf.$$", "$conf/bacula-dir.conf");
229 unlink("$tmp/bacula-dir.conf.$$");
233 # open a directory and update all files
234 sub update_some_files
240 print "Update files in $dest\n";
241 opendir(DIR, $dest) || die "$!";
245 open(FP, ">$f") or die "$f $!";
246 print FP "$t update $f\n";
252 print "$nb files updated\n";
255 # create big number of files in a given directory
256 # Inputs: dest destination directory
257 # nb number of file to create
259 # perl -Mscripts::functions -e 'create_many_files("$cwd/files", 100000)'
260 sub create_many_files
262 my ($dest, $nb) = @_;
267 $base = chr($nb % 26 + 65); # We use a base directory A-Z
270 if (-f "$dest/$base/a${base}a${nb}aaa${base}") {
271 print "Files already created\n";
275 # auto flush stdout for dots
277 print "Create $nb files into $dest\n";
278 for(my $i=0; $i < 26; $i++) {
279 $base = chr($i + 65);
280 mkdir("$dest/$base") if (! -d "$dest/$base");
282 for(my $i=0; $i<=$nb; $i++) {
283 $base = chr($i % 26 + 65);
284 open(FP, ">$dest/$base/a${base}a${i}aaa$base") or die "$dest/$base $!";
288 open(FP, ">$dir/b${base}a${i}csq$base") or die "$dir $!";
289 print FP "$base $i\n";
293 $dir = "$dest/$base/$base$i$base";
296 print "." if (!($i % 10000));
301 # create big number of dirs in a given directory
302 # Inputs: dest destination directory
303 # nb number of dirs to create
305 # perl -Mscripts::functions -e 'create_many_dirs("$cwd/files", 100000)'
308 my ($dest, $nb) = @_;
313 $base = chr($nb % 26 + 65); # We use a base directory A-Z
314 $base2 = chr(($nb+10) % 26 + 65);
316 if (-d "$dest/$base/$base2/$base/a${base}a${nb}aaa${base}") {
317 print "Files already created\n";
321 # auto flush stdout for dots
323 print "Create $nb dirs into $dest\n";
324 for(my $i=0; $i < 26; $i++) {
325 $base = chr($i + 65);
326 $base2 = chr(($i+10) % 26 + 65);
327 mkdir("$dest/$base");
328 mkdir("$dest/$base/$base2");
329 mkdir("$dest/$base/$base2/$base$base2");
330 mkdir("$dest/$base/$base2/$base$base2/$base$base2");
331 mkdir("$dest/$base/$base2/$base$base2/$base$base2/$base2$base");
333 for(my $i=0; $i<=$nb; $i++) {
334 $base = chr($i % 26 + 65);
335 $base2 = chr(($i+10) % 26 + 65);
336 mkdir("$dest/$base/$base2/$base$base2/$base$base2/$base2$base/a${base}a${i}aaa$base");
337 print "." if (!($i % 10000));
344 if (grep {/Wanted SQL_ASCII, got UTF8/}
345 `${bin}/bacula-dir -d50 -t -c ${conf}/bacula-dir.conf 2>&1`)
347 print "Found database encoding problem, please modify the ",
348 "database encoding (SQL_ASCII)\n";
353 # You can change the maximum concurrent jobs for any config file
354 # If specified, you can change only one Resource or one type of
355 # resource at the time (optional)
356 # set_maximum_concurrent_jobs('$conf/bacula-dir.conf', 100);
357 # set_maximum_concurrent_jobs('$conf/bacula-dir.conf', 100, 'Director');
358 # set_maximum_concurrent_jobs('$conf/bacula-dir.conf', 100, 'Device', 'Drive-0');
359 sub set_maximum_concurrent_jobs
361 my ($file, $nb, $obj, $name) = @_;
363 die "Can't get new maximumconcurrentjobs"
366 add_attribute($file, "Maximum Concurrent Jobs", $nb, $obj, $name);
369 # You can add option to a resource
370 # add_attribute('$conf/bacula-dir.conf', 'FDTimeout', 1600, 'Director');
371 # add_attribute('$conf/bacula-dir.conf', 'FDTimeout', 1600, 'Storage', 'FileStorage');
374 my ($file, $attr, $value, $obj, $name) = @_;
375 my ($cur_obj, $cur_name, $done);
377 open(FP, ">$tmp/1.$$") or die "Can't write to $tmp/1.$$";
378 open(SRC, $file) or die "Can't open $file";
379 while (my $l = <SRC>)
386 if ($l =~ /^(\w+) {/) {
391 if ($l =~ /^\s*\Q$attr\E/i) {
392 if (!$obj || $cur_obj eq $obj) {
393 if (!$name || $cur_name eq $name) {
394 $l =~ s/\Q$attr\E\s*=\s*.+/$attr = $value/ig;
400 if ($l =~ /^\s*Name\s*=\s*"?([\w\d\.-]+)"?/i) {
406 if ($cur_obj eq $obj) {
407 if (!$name || $cur_name eq $name) {
408 $l = " $attr = $value\n$l";
412 $cur_name = $cur_obj = undef;
418 copy("$tmp/1.$$", $file) or die "Can't copy $tmp/1.$$ to $file";
421 # This test the list jobs output to check differences
422 # Input: read file argument
423 # check if all jobids in argument are present in the first
424 # 'list jobs' and not present in the second
425 # Output: exit(1) if something goes wrong and print error
429 my %to_check = map { $_ => 1} @_;
434 open(FP, $f) or die "Can't open $f $!";
435 while (my $l = <FP>) # read all files to check
437 if ($l =~ /list jobs/) {
441 if ($nb_list_job == 2) {
442 foreach my $jobid (keys %to_check) {
443 if (!$seen{$jobid}) {
444 print "ERROR: in $f, can't find JobId=$jobid in first 'list jobs'\n";
451 if ($nb_list_job == 0) {
454 if ($l =~ /Pruned (\d+) Job for client/) {
456 print "ERROR: in $f, Prune command returns $1 jobs, want $nb\n";
461 if ($l =~ /No Jobs found to prune/) {
463 print "ERROR: in $f, Prune command returns 0 job, want $nb\n";
469 # | 1 | NightlySave | 2010-06-16 22:43:05 | B | F | 27 | 4173577 | T |
470 if ($l =~ /^\|\s+(\d+)/) {
471 if ($nb_list_job == 1) {
479 foreach my $jobid (keys %to_check) {
480 if (!$seen{$jobid}) {
481 print "******* listing of $f *********\n";
483 print "******* end listing of $f *********\n";
484 print "ERROR: in $f, JobId=$jobid should not but is still present in the 2nd 'list jobs'\n";
488 if ($nb_list_job != 2) {
489 print "ERROR: in $f, not enough 'list jobs'\n";
495 # This test ensure that 'list copies' displays only each copy one time
497 # Input: read stream from stdin or with file list argument
498 # check the number of copies with the ARGV[1]
499 # Output: exit(1) if something goes wrong and print error
500 sub check_multiple_copies
502 my ($nb_to_found) = @_;
504 my $in_list_copies=0; # are we or not in a list copies block
505 my $nb_found=0; # count the number of copies found
509 while (my $l = <>) # read all files to check
511 if ($l =~ /list copies/) {
517 # not in a list copies anymore
518 if ($in_list_copies && $l =~ /^ /) {
524 # | 3 | Backup.2009-09-28 | 9 | DiskChangerMedia |
525 if ($in_list_copies && $l =~ /^\|\s+\d+/) {
526 my (undef, $jobid, undef, $copyid, undef) = split(/\s*\|\s*/, $l);
527 if (exists $seen{$jobid}) {
528 print "ERROR: $jobid/$copyid already known as $seen{$jobid}\n";
531 $seen{$jobid}=$copyid;
537 # test the number of copies against the given arg
538 if ($nb_to_found && ($nb_to_found != $nb_found)) {
539 print "ERROR: Found wrong number of copies ",
540 "($nb_to_found != $nb_found)\n";
547 use POSIX qw/strftime/;
551 print strftime('%F %T', localtime(time+$sec)), "\n";
557 print join("\n", @_), "\n";
563 open(FP, ">$REMOTE_FILE/bacula-fd.conf") or
564 die "ERROR: Can't open $REMOTE_FILE/bacula-fd.conf $!";
568 Password = \"$REMOTE_PASSWORD\"
572 FDport = $REMOTE_PORT
573 WorkingDirectory = $REMOTE_FILE/working
574 Pid Directory = $REMOTE_FILE/working
575 Maximum Concurrent Jobs = 20
579 director = $HOST-dir = all, !skipped, !restored
583 system("mkdir -p '$REMOTE_FILE/working' '$REMOTE_FILE/save'");
584 system("rm -rf '$REMOTE_FILE/restore'");
587 close(STDIN); open(STDIN, "/dev/null");
588 close(STDOUT); open(STDOUT, ">/dev/null");
589 close(STDERR); open(STDERR, ">/dev/null");
590 exec("/opt/bacula/bin/bacula-fd -c $REMOTE_FILE/bacula-fd.conf");
594 $pid = `cat $REMOTE_FILE/working/bacula-fd.$REMOTE_PORT.pid`;
597 # create files and tweak rights
598 create_many_files("$REMOTE_FILE/save", 5000);
599 chdir("$REMOTE_FILE/save");
602 for my $g ( split(' ', $( )) {
607 # create a simple script to execute
608 open(FP, ">test.sh") or die "Can't open test.sh $!";
609 print FP "#!/bin/sh\n";
610 print FP "echo this is a script";
612 chmod 0755, "test.sh";
615 link("test.sh", "link-test.sh");
617 # create long filename
618 mkdir("b" x 255) or print "can't create long dir $!\n";
619 copy("test.sh", ("b" x 255) . '/' . ("a" x 255)) or print "can't create long dir $!\n";
621 # play with some symlinks
622 symlink("test.sh", "sym-test.sh");
623 symlink("$REMOTE_FILE/save/test.sh", "sym-abs-test.sh");
635 debug("Doing diff between save and restore");
636 system("ssh $REMOTE_ADDR " .
637 "$REMOTE_FILE/scripts/diff.pl -s $REMOTE_FILE/save -d $REMOTE_FILE/restore/$REMOTE_FILE/save");
643 debug("Kill remote bacula-fd");
644 system("ssh $REMOTE_ADDR " .
645 "'test -f $REMOTE_FILE/working/bacula-fd.$REMOTE_PORT.pid && " .
646 "kill `cat $REMOTE_FILE/working/bacula-fd.$REMOTE_PORT.pid`'");
651 system("ssh $REMOTE_ADDR mkdir -p '$REMOTE_FILE/scripts/'");
652 system("scp -q scripts/functions.pm scripts/diff.pl $REMOTE_ADDR:$REMOTE_FILE/scripts/");
653 system("scp -q config $REMOTE_ADDR:$REMOTE_FILE/");
654 debug("INFO: Configuring remote client");
655 system("ssh $REMOTE_ADDR 'cd $REMOTE_FILE && perl -Mscripts::functions -e remote_config'");