mirror of https://github.com/moparisthebest/curl
test: offer "automake" output and check for perl better
runtests.pl -am now uses the "PASS/FAIL: [desc]" output for each executed test. You can run 'make test-am' in the root build directory to invoke that. The reason for this output style is to better allow generic test suite parsers to also grok our test output. The test Makefile now also tests that perl was indeed found and that the PERL variable points to an executable before it tries to run the main test perl script runtests.pl,
This commit is contained in:
Daniel Stenberg
parent
ef6f040355
commit
14dc679ce0
|
|
@ -81,6 +81,9 @@ test-full:
|
|||
test-torture:
|
||||
@(cd tests; $(MAKE) all torture-test)
|
||||
|
||||
test-am:
|
||||
@(cd tests; $(MAKE) all am-test)
|
||||
|
||||
endif
|
||||
|
||||
examples:
|
||||
|
|
|
|||
|
|
@ -55,26 +55,31 @@ curl:
|
|||
|
||||
if CROSSCOMPILING
|
||||
TEST = @echo "NOTICE: we can't run the tests when cross-compiling!"
|
||||
TEST_Q =
|
||||
TEST_F =
|
||||
TEST_T =
|
||||
else # if not cross-compiling:
|
||||
TEST = srcdir=$(srcdir) $(PERL) $(PERLFLAGS) $(srcdir)/runtests.pl
|
||||
TEST_Q = -a -s
|
||||
TEST_AM = -a -am
|
||||
TEST_F = -a -p -r
|
||||
TEST_T = -a -t
|
||||
endif
|
||||
|
||||
test: all
|
||||
# make sure that PERL is pointing to an executable
|
||||
perlcheck:
|
||||
@if ! test -x "$(PERL)"; then echo "No perl!"; exit 2; fi
|
||||
|
||||
test: perhcheck all
|
||||
$(TEST)
|
||||
|
||||
quiet-test: all
|
||||
quiet-test: perlcheck all
|
||||
$(TEST) $(TEST_Q)
|
||||
|
||||
full-test: all
|
||||
am-test: perlcheck all
|
||||
$(TEST) $(TEST_AM)
|
||||
|
||||
full-test: perlcheck all
|
||||
$(TEST) $(TEST_F)
|
||||
|
||||
torture-test: all
|
||||
torture-test: perlcheck all
|
||||
$(TEST) $(TEST_T)
|
||||
|
||||
.1.html:
|
||||
|
|
|
|||
|
|
@ -262,6 +262,7 @@ my %oldenv;
|
|||
#
|
||||
|
||||
my $short;
|
||||
my $automakestyle;
|
||||
my $verbose;
|
||||
my $debugprotocol;
|
||||
my $anyway;
|
||||
|
|
@ -2148,18 +2149,25 @@ sub filteroff {
|
|||
#
|
||||
sub compare {
|
||||
# filter off patterns _before_ this comparison!
|
||||
my ($subject, $firstref, $secondref)=@_;
|
||||
my ($testnum, $testname, $subject, $firstref, $secondref)=@_;
|
||||
|
||||
my $result = compareparts($firstref, $secondref);
|
||||
|
||||
if($result) {
|
||||
# timestamp test result verification end
|
||||
$timevrfyend{$testnum} = Time::HiRes::time() if($timestats);
|
||||
|
||||
if(!$short) {
|
||||
logmsg "\n $subject FAILED:\n";
|
||||
logmsg "\n $testnum: $subject FAILED:\n";
|
||||
logmsg showdiff($LOGDIR, $firstref, $secondref);
|
||||
}
|
||||
else {
|
||||
elsif(!$automakestyle) {
|
||||
logmsg "FAILED\n";
|
||||
}
|
||||
else {
|
||||
# automakestyle
|
||||
logmsg "FAIL: $testnum - $testname - $subject\n";
|
||||
}
|
||||
}
|
||||
return $result;
|
||||
}
|
||||
|
|
@ -2874,7 +2882,7 @@ sub singletest {
|
|||
timestampskippedevents($testnum);
|
||||
return -1;
|
||||
}
|
||||
logmsg sprintf("test %03d...", $testnum);
|
||||
logmsg sprintf("test %03d...", $testnum) if(!$automakestyle);
|
||||
|
||||
# extract the reply data
|
||||
my @reply = getpart("reply", "data");
|
||||
|
|
@ -2916,12 +2924,9 @@ sub singletest {
|
|||
|
||||
# name of the test
|
||||
my @testname= getpart("client", "name");
|
||||
|
||||
if(!$short) {
|
||||
my $name = $testname[0];
|
||||
$name =~ s/\n//g;
|
||||
logmsg "[$name]\n";
|
||||
}
|
||||
my $testname = $testname[0];
|
||||
$testname =~ s/\n//g;
|
||||
logmsg "[$testname]\n" if(!$short);
|
||||
|
||||
if($listonly) {
|
||||
timestampskippedevents($testnum);
|
||||
|
|
@ -3333,10 +3338,8 @@ sub singletest {
|
|||
chomp($validstdout[$#validstdout]);
|
||||
}
|
||||
|
||||
$res = compare("stdout", \@actual, \@validstdout);
|
||||
$res = compare($testnum, $testname, "stdout", \@actual, \@validstdout);
|
||||
if($res) {
|
||||
# timestamp test result verification end
|
||||
$timevrfyend{$testnum} = Time::HiRes::time() if($timestats);
|
||||
return 1;
|
||||
}
|
||||
$ok .= "s";
|
||||
|
|
@ -3357,10 +3360,8 @@ sub singletest {
|
|||
map s/\r\n/\n/g, @out;
|
||||
}
|
||||
|
||||
$res = compare("data", \@out, \@reply);
|
||||
$res = compare($testnum, $testname, "data", \@out, \@reply);
|
||||
if ($res) {
|
||||
# timestamp test result verification end
|
||||
$timevrfyend{$testnum} = Time::HiRes::time() if($timestats);
|
||||
return 1;
|
||||
}
|
||||
$ok .= "d";
|
||||
|
|
@ -3372,10 +3373,8 @@ sub singletest {
|
|||
if(@upload) {
|
||||
# verify uploaded data
|
||||
my @out = loadarray("$LOGDIR/upload.$testnum");
|
||||
$res = compare("upload", \@out, \@upload);
|
||||
$res = compare($testnum, $testname, "upload", \@out, \@upload);
|
||||
if ($res) {
|
||||
# timestamp test result verification end
|
||||
$timevrfyend{$testnum} = Time::HiRes::time() if($timestats);
|
||||
return 1;
|
||||
}
|
||||
$ok .= "u";
|
||||
|
|
@ -3419,10 +3418,8 @@ sub singletest {
|
|||
}
|
||||
}
|
||||
|
||||
$res = compare("protocol", \@out, \@protstrip);
|
||||
$res = compare($testnum, $testname, "protocol", \@out, \@protstrip);
|
||||
if($res) {
|
||||
# timestamp test result verification end
|
||||
$timevrfyend{$testnum} = Time::HiRes::time() if($timestats);
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
|
@ -3469,10 +3466,8 @@ sub singletest {
|
|||
}
|
||||
}
|
||||
|
||||
$res = compare("proxy", \@out, \@protstrip);
|
||||
$res = compare($testnum, $testname, "proxy", \@out, \@protstrip);
|
||||
if($res) {
|
||||
# timestamp test result verification end
|
||||
$timevrfyend{$testnum} = Time::HiRes::time() if($timestats);
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
|
@ -3521,10 +3516,9 @@ sub singletest {
|
|||
|
||||
@outfile = fixarray(@outfile);
|
||||
|
||||
$res = compare("output ($filename)", \@generated, \@outfile);
|
||||
$res = compare($testnum, $testname, "output ($filename)",
|
||||
\@generated, \@outfile);
|
||||
if($res) {
|
||||
# timestamp test result verification end
|
||||
$timevrfyend{$testnum} = Time::HiRes::time() if($timestats);
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
|
@ -3614,8 +3608,13 @@ sub singletest {
|
|||
}
|
||||
my @e = valgrindparse($srcdir, $feature{'SSL'}, "$LOGDIR/$vgfile");
|
||||
if(@e && $e[0]) {
|
||||
logmsg " valgrind ERROR ";
|
||||
logmsg @e;
|
||||
if($automakestyle) {
|
||||
logmsg "FAIL: $testnum - $testname - valgrind\n";
|
||||
}
|
||||
else {
|
||||
logmsg " valgrind ERROR ";
|
||||
logmsg @e;
|
||||
}
|
||||
# timestamp test result verification end
|
||||
$timevrfyend{$testnum} = Time::HiRes::time() if($timestats);
|
||||
return 1;
|
||||
|
|
@ -3641,7 +3640,13 @@ sub singletest {
|
|||
my $left=sprintf("remaining: %02d:%02d",
|
||||
$estleft/60,
|
||||
$estleft%60);
|
||||
logmsg sprintf("OK (%-3d out of %-3d, %s)\n", $count, $total, $left);
|
||||
|
||||
if(!$automakestyle) {
|
||||
logmsg sprintf("OK (%-3d out of %-3d, %s)\n", $count, $total, $left);
|
||||
}
|
||||
else {
|
||||
logmsg "PASS: $testnum - $testname\n";
|
||||
}
|
||||
|
||||
# the test succeeded, remove all log files
|
||||
if(!$keepoutfiles) {
|
||||
|
|
@ -4284,6 +4289,11 @@ while(@ARGV) {
|
|||
# short output
|
||||
$short=1;
|
||||
}
|
||||
elsif($ARGV[0] eq "-am") {
|
||||
# automake-style output
|
||||
$short=1;
|
||||
$automakestyle=1;
|
||||
}
|
||||
elsif($ARGV[0] eq "-n") {
|
||||
# no valgrind
|
||||
undef $valgrind;
|
||||
|
|
@ -4360,6 +4370,7 @@ Usage: runtests.pl [options] [test selection(s)]
|
|||
-r run time statistics
|
||||
-rf full run time statistics
|
||||
-s short output
|
||||
-am automake style output PASS/FAIL: [number] [name]
|
||||
-t[N] torture (simulate memory alloc failures); N means fail Nth alloc
|
||||
-v verbose output
|
||||
[num] like "5 6 9" or " 5 to 22 " to run those tests only
|
||||
|
|
|
|||
Loading…
Reference in New Issue