X-Git-Url: https://git.sur5r.net/?a=blobdiff_plain;f=testcases%2Fcomplete-run.pl;h=7ca89016f6ea45443372cba02cd50ac6f9c93d22;hb=fa41ad7e218d1497a749eb59983f3dd162ddbe29;hp=247f15b0a880df079241e09367e05007bf1c456d;hpb=1c72e8b69e2be37689518835b06140498ab6b767;p=i3%2Fi3 diff --git a/testcases/complete-run.pl b/testcases/complete-run.pl index 247f15b0..7ca89016 100755 --- a/testcases/complete-run.pl +++ b/testcases/complete-run.pl @@ -1,10 +1,11 @@ #!/usr/bin/env perl # vim:ts=4:sw=4:expandtab -# © 2010-2011 Michael Stapelberg and contributors +# © 2010-2012 Michael Stapelberg and contributors package complete_run; use strict; use warnings; use v5.10; +use utf8; # the following are modules which ship with Perl (>= 5.10): use Pod::Usage; use Cwd qw(abs_path); @@ -15,6 +16,7 @@ use TAP::Harness; use TAP::Parser; use TAP::Parser::Aggregator; use Time::HiRes qw(time); +use IO::Handle; # these are shipped with the testsuite use lib qw(lib); use StartXDummy; @@ -28,6 +30,9 @@ use AnyEvent::I3 qw(:all); use X11::XCB::Connection; use JSON::XS; # AnyEvent::I3 depends on it, too. +binmode STDOUT, ':utf8'; +binmode STDERR, ':utf8'; + # Close superfluous file descriptors which were passed by running in a VIM # subshell or situations like that. AnyEvent::Util::close_all_fds_except(0, 1, 2); @@ -37,20 +42,27 @@ my $log; sub Log { say $log "@_" } my %timings; -my $coverage_testing = 0; -my $valgrind = 0; -my $strace = 0; my $help = 0; # Number of tests to run in parallel. Important to know how many Xdummy # instances we need to start (unless @displays are given). Defaults to # num_cores * 2. my $parallel = undef; my @displays = (); +my %options = ( + valgrind => 0, + strace => 0, + xtrace => 0, + coverage => 0, + restart => 0, +); +my $keep_xdummy_output = 0; my $result = GetOptions( - "coverage-testing" => \$coverage_testing, - "valgrind" => \$valgrind, - "strace" => \$strace, + "coverage-testing" => \$options{coverage}, + "keep-xdummy-output" => \$keep_xdummy_output, + "valgrind" => \$options{valgrind}, + "strace" => \$options{strace}, + "xtrace" => \$options{xtrace}, "display=s" => \@displays, "parallel=i" => \$parallel, "help|?" => \$help, @@ -58,11 +70,37 @@ my $result = GetOptions( pod2usage(-verbose => 2, -exitcode => 0) if $help; +# Check for missing executables +my @binaries = qw( + ../i3 + ../i3bar/i3bar + ../i3-config-wizard/i3-config-wizard + ../i3-dump-log/i3-dump-log + ../i3-input/i3-input + ../i3-msg/i3-msg + ../i3-nagbar/i3-nagbar + ); + +foreach my $binary (@binaries) { + die "$binary executable not found, did you run “make”?" unless -e $binary; + die "$binary is not an executable" unless -x $binary; +} + @displays = split(/,/, join(',', @displays)); @displays = map { s/ //g; $_ } @displays; +# 2: get a list of all testcases +my @testfiles = @ARGV; + +# if no files were passed on command line, run all tests from t/ +@testfiles = if @testfiles == 0; + +my $numtests = scalar @testfiles; + # No displays specified, let’s start some Xdummy instances. -@displays = start_xdummy($parallel) if @displays == 0; +if (@displays == 0) { + @displays = start_xdummy($parallel, $numtests, $keep_xdummy_output); +} # 1: create an output directory for this test-run my $outdir = "testsuite-"; @@ -70,7 +108,7 @@ $outdir .= POSIX::strftime("%Y-%m-%d-%H-%M-%S-", localtime()); $outdir .= `git describe --tags`; chomp($outdir); mkdir($outdir) or die "Could not create $outdir"; -unlink("latest") if -e "latest"; +unlink("latest") if -l "latest"; symlink("$outdir", "latest") or die "Could not symlink latest to $outdir"; @@ -79,7 +117,7 @@ symlink("$outdir", "latest") or die "Could not symlink latest to $outdir"; # 2: keep the connection open so that i3 is not the only client. this prevents # the X server from exiting (Xdummy will restart it, but not quick enough # sometimes) -my @worker; +my @single_worker; for my $display (@displays) { my $screen; my $x = X11::XCB::Connection->new(display => $display); @@ -87,7 +125,7 @@ for my $display (@displays) { die "Could not connect to display $display\n"; } else { # start a TestWorker for each display - push @worker, worker($display, $x, $outdir); + push @single_worker, worker($display, $x, $outdir, \%options); } } @@ -96,12 +134,6 @@ for my $display (@displays) { my $timingsjson = StartXDummy::slurp('.last_run_timings.json'); %timings = %{decode_json($timingsjson)} if length($timingsjson) > 0; -# 2: get a list of all testcases -my @testfiles = @ARGV; - -# if no files were passed on command line, run all tests from t/ -@testfiles = if @testfiles == 0; - # Re-order the files so that those which took the longest time in the previous # run will be started at the beginning to not delay the whole run longer than # necessary. @@ -109,6 +141,12 @@ my @testfiles = @ARGV; sort { $b->[1] <=> $a->[1] } map { [$_, $timings{$_} // 999] } @testfiles; +# Run 000-load-deps.t first to bail out early when dependencies are missing. +my $loadtest = "t/000-load-deps.t"; +if ((scalar grep { $_ eq $loadtest } @testfiles) > 0) { + @testfiles = ($loadtest, grep { $_ ne $loadtest } @testfiles); +} + printf("\nRough time estimate for this run: %.2f seconds\n\n", $timings{GLOBAL}) if exists($timings{GLOBAL}); @@ -118,6 +156,7 @@ printf("\nRough time estimate for this run: %.2f seconds\n\n", $timings{GLOBAL}) my $logfile = "$outdir/complete-run.log"; open $log, '>', $logfile or die "Could not create '$logfile': $!"; +$log->autoflush(1); say "Writing logfile to '$logfile'..."; # 3: run all tests @@ -130,13 +169,16 @@ $aggregator->start(); status_init(displays => \@displays, tests => $num); -my $cv = AE::cv; +my $single_cv = AE::cv; # We start tests concurrently: For each display, one test gets started. Every # test starts another test after completing. -for (@worker) { $cv->begin; take_job($_) } +for (@single_worker) { + $single_cv->begin; + take_job($_, $single_cv, \@testfiles); +} -$cv->recv; +$single_cv->recv; $aggregator->stop(); @@ -170,7 +212,16 @@ my @slowest = map { $_->[0] } grep { !/^GLOBAL$/ } keys %timings; say ''; say 'The slowest tests are:'; -printf("\t%s with %.2f seconds\n", $_, $timings{$_}) for @slowest[0..4]; +printf("\t%s with %.2f seconds\n", $_, $timings{$_}) + for @slowest[0..($#slowest > 4 ? 4 : $#slowest)]; + +# When we are running precisely one test, print the output. Makes developing +# with a single testcase easier. +if ($numtests == 1) { + say ''; + say 'Test output:'; + say StartXDummy::slurp($logfile); +} END { cleanup() } @@ -187,9 +238,9 @@ exit 0; # triggered to finish testing. # sub take_job { - my ($worker) = @_; + my ($worker, $cv, $tests) = @_; - my $test = shift @testfiles + my $test = shift @$tests or return $cv->end; my $display = $worker->{display}; @@ -242,9 +293,16 @@ sub take_job { for (1 .. $lines) { my $result = $parser->next; - if (defined($result) and $result->is_test) { + next unless defined($result); + if ($result->is_test) { $tests_completed++; status($display, "$test: [$tests_completed/??] "); + } elsif ($result->is_bailout) { + Log status($display, "$test: BAILOUT"); + status_completed(scalar @done); + say ""; + say "test $test bailed out: " . $result->explanation; + exit 1; } } @@ -258,7 +316,7 @@ sub take_job { push @done, [ $test, $output ]; undef $w; - take_job($worker); + take_job($worker, $cv, $tests); } ); } @@ -323,6 +381,11 @@ C. Runs i3 under strace to trace system calls. The output will be available in C. +=item B<--xtrace> + +Runs i3 under xtrace to trace X11 requests/replies. The output will be +available in C. + =item B<--coverage-testing> Exits i3 cleanly (instead of kill -9) to make coverage testing work properly.