From 3f0bef2b5303fec1a4b483e65d767205efd481b7 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 17 Nov 2012 12:12:42 +0100 Subject: [PATCH] runtests.pl: add an -E option to specify an exclude file It can contain additional restraints for test numbers, keywords and tools. The idea is to let third parties like the Privoxy project distribute an exclude file with their tarballs that specifies which curl tests are not expected to work when using Privoxy as a proxy, without having to fork the whole curl test suite. The syntax could be changed to be extendable and maybe more closely reflect the "curl test" syntax. Currently it's a bunch of lines like these: test:$TESTNUMBER:Reason why this test with number $TESTNUMBER should be skipped keyword:$KEYWORD:Reason why tests whose keywords contain the $KEYWORD should be skipped tool:$TOOL:Reason why tests with tools that contain $TOOL should be skipped To specify multiple $TESTNUMBERs, $KEYWORDs and $TOOLs on a single line, split them with commas. --- tests/data/DISABLED | 4 ++++ tests/data/Makefile.inc | 2 +- tests/data/test1182 | 36 ++++++++++++++++++++++++++++++ tests/runtests.pl | 49 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 tests/data/test1182 diff --git a/tests/data/DISABLED b/tests/data/DISABLED index 1806a5dbe..2ff0fc49a 100644 --- a/tests/data/DISABLED +++ b/tests/data/DISABLED @@ -11,6 +11,10 @@ 836 882 938 +# test 1182 kills the test servers as a side effect +# of running runtests.pl as a child of itself sharing +# some of the directories. +1182 1209 1211 # fnmatch differences are just too common to make testing them sensible diff --git a/tests/data/Makefile.inc b/tests/data/Makefile.inc index 485fd72b6..5b60d1606 100644 --- a/tests/data/Makefile.inc +++ b/tests/data/Makefile.inc @@ -137,7 +137,7 @@ test1144 test1145 test1146 test1147 test1148 test1149 test1150 test1151 \ test1152 test1153 test1154 test1155 test1156 test1157 test1158 test1159 \ test1160 test1161 test1162 test1163 test1164 test1165 test1166 test1167 \ test1168 test1169 test1170 test1171 test1172 test1173 test1174 test1175 \ -test1176 test1177 test1178 test1179 test1180 test1181 \ +test1176 test1177 test1178 test1179 test1180 test1181 test1182 \ \ test1188 \ \ diff --git a/tests/data/test1182 b/tests/data/test1182 new file mode 100644 index 000000000..0e5a672d2 --- /dev/null +++ b/tests/data/test1182 @@ -0,0 +1,36 @@ + + + +runtests.pl + + + +# +# Client-side + + +none + + +Verify that runtests.pl accepts an exclude file with the -E option + + + +%SRCDIR/runtests.pl -o TESTDIR=%SRCDIR/log/data -o LOGDIR=%SRCDIR/log/log -E %SRCDIR/log/test1182-exclude-file 1 + + +mkdir %SRCDIR/log/data ; mkdir %SRCDIR/log/log; cp %SRCDIR/data/test1 %SRCDIR/log/data; echo 'test:1:Test should not run for unit test 1182' > %SRCDIR/log/test1182-exclude-file + + +grep -q "Test should not run for unit test 1182" %SRCDIR/log/stdout1182 + + + + + + +1 + + + + diff --git a/tests/runtests.pl b/tests/runtests.pl index 27800d817..8af6bfa97 100755 --- a/tests/runtests.pl +++ b/tests/runtests.pl @@ -165,6 +165,7 @@ my $HTTPUNIXPATH; # HTTP server Unix domain socket path my $use_external_proxy = 0; my $proxy_address; +my %custom_skip_reasons; my $SSHSRVMD5 = "[uninitialized]"; # MD5 of ssh server public key my $VERSION; # curl's reported version number @@ -3597,6 +3598,31 @@ sub singletest { } } + if (!$why && defined $custom_skip_reasons{test}{$testnum}) { + $why = $custom_skip_reasons{test}{$testnum}; + } + + if (!$why && defined $custom_skip_reasons{tool}) { + foreach my $tool (getpart("client", "tool")) { + foreach my $tool_skip_pattern (keys %{$custom_skip_reasons{tool}}) { + if ($tool =~ /$tool_skip_pattern/i) { + $why = $custom_skip_reasons{tool}{$tool_skip_pattern}; + } + } + } + } + + if (!$why && defined $custom_skip_reasons{keyword}) { + foreach my $keyword (getpart("info", "keywords")) { + foreach my $keyword_skip_pattern (keys %{$custom_skip_reasons{keyword}}) { + if ($keyword =~ /$keyword_skip_pattern/i) { + $why = $custom_skip_reasons{keyword}{$keyword_skip_pattern}; + } + } + } + } + + # test definition may instruct to (un)set environment vars # this is done this early, so that the precheck can use environment # variables and still bail out fine on errors @@ -5328,6 +5354,28 @@ while(@ARGV) { # run the tests cases event based if possible $run_event_based=1; } + elsif($ARGV[0] eq "-E") { + # load additional reasons to skip tests + shift @ARGV; + my $exclude_file = $ARGV[0]; + open(my $fd, "<", $exclude_file) or die "Couldn't open '$exclude_file': $!"; + while(my $line = <$fd>) { + next if ($line =~ /^#/); + chomp $line; + my ($type, $patterns, $skip_reason) = split(/\s*:\s*/, $line, 3); + + die "Unsupported type: $type\n" if($type !~ /^keyword|test|tool$/); + + foreach my $pattern (split(/,/, $patterns)) { + if($type =~ /^test$/) { + # Strip leading zeros in the test number + $pattern = int($pattern); + } + $custom_skip_reasons{$type}{$pattern} = $skip_reason; + } + } + close($fd); + } elsif ($ARGV[0] eq "-g") { # run this test with gdb $gdbthis=1; @@ -5439,6 +5487,7 @@ Usage: runtests.pl [options] [test selection(s)] -c path use this curl executable -d display server debug info -e event-based execution + -E file load the specified file to exclude certain tests -g run the test case with gdb -gw run the test case with gdb as a windowed application -h this help text