mirror of
https://github.com/moparisthebest/curl
synced 2024-12-23 16:48:49 -05:00
ci/tests: Make it possible to still run but ignore failing tests
This enables the development of a solution for the failing tests by running them on CI while ignoring their result for the overall status. Closes #4994
This commit is contained in:
Marc Hoersken
parent
86ceb9b006
commit
119ea453f9
@ -219,7 +219,7 @@ stages:
|
||||
env:
|
||||
MSYSTEM: MINGW32
|
||||
MSYS2_PATH_TYPE: inherit
|
||||
TFLAGS: "!323 !1056 !1299"
|
||||
TFLAGS: "~323 ~1056 ~1299"
|
||||
steps:
|
||||
- script: C:\msys64\usr\bin\sh -l -c "cd $(echo '%cd%') && ./buildconf && ./configure --host=i686-w64-mingw32 --build=i686-w64-mingw32 --enable-debug --enable-werror"
|
||||
displayName: 'configure debug'
|
||||
@ -242,7 +242,7 @@ stages:
|
||||
env:
|
||||
MSYSTEM: MINGW64
|
||||
MSYS2_PATH_TYPE: inherit
|
||||
TFLAGS: "!323 !1056 !1299"
|
||||
TFLAGS: "~323 ~1056 ~1299"
|
||||
steps:
|
||||
- script: C:\msys64\usr\bin\sh -l -c "cd $(echo '%cd%') && ./buildconf && ./configure --host=x86_64-w64-mingw32 --build=x86_64-w64-mingw32 --enable-debug --enable-werror"
|
||||
displayName: 'configure debug'
|
||||
@ -263,7 +263,7 @@ stages:
|
||||
container:
|
||||
image: mback2k/curl-docker-winbuildenv-msys1-mingw:ltsc2019
|
||||
env:
|
||||
TFLAGS: "!203 !1056 !1143"
|
||||
TFLAGS: "~203 ~1056 ~1143"
|
||||
steps:
|
||||
- script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && ./buildconf && ./configure --host=i686-pc-mingw32 --build=i686-pc-mingw32 --prefix=/mingw --enable-debug"
|
||||
displayName: 'configure debug'
|
||||
@ -284,7 +284,7 @@ stages:
|
||||
container:
|
||||
image: mback2k/curl-docker-winbuildenv-msys1-mingw32:ltsc2019
|
||||
env:
|
||||
TFLAGS: "!203 !1056 !1143 !1299"
|
||||
TFLAGS: "~203 ~1056 ~1143 ~1299"
|
||||
steps:
|
||||
- script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && ./buildconf && ./configure --host=i686-w64-mingw32 --build=i686-w64-mingw32 --prefix=/mingw32 --enable-debug --enable-werror --without-zlib"
|
||||
displayName: 'configure debug without zlib'
|
||||
@ -305,7 +305,7 @@ stages:
|
||||
container:
|
||||
image: mback2k/curl-docker-winbuildenv-msys1-mingw64:ltsc2019
|
||||
env:
|
||||
TFLAGS: "!203 !1056 !1143 !1299"
|
||||
TFLAGS: "~203 ~1056 ~1143 ~1299"
|
||||
steps:
|
||||
- script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && ./buildconf && ./configure --host=x86_64-w64-mingw32 --build=x86_64-w64-mingw32 --prefix=/mingw64 --enable-debug --enable-werror --without-zlib"
|
||||
displayName: 'configure debug without zlib'
|
||||
@ -328,7 +328,7 @@ stages:
|
||||
env:
|
||||
MSYSTEM: MINGW32
|
||||
MSYS2_PATH_TYPE: inherit
|
||||
TFLAGS: "!165 !310 !1013 !1056 !1299 !1448 !2034 !2037 !2041 !2046 !2047 !3000 !3001"
|
||||
TFLAGS: "~165 ~310 ~1013 ~1056 ~1299 ~1448 ~2034 ~2037 ~2041 ~2046 ~2047 ~3000 ~3001"
|
||||
steps:
|
||||
- script: C:\msys64\usr\bin\sh -l -c "cd $(echo '%cd%') && ./buildconf && ./configure --host=i686-w64-mingw32 --build=i686-w64-mingw32 --enable-debug --enable-werror --enable-sspi --without-ssl --with-schannel --with-winidn"
|
||||
displayName: 'configure debug with sspi/schannel/winidn'
|
||||
@ -351,7 +351,7 @@ stages:
|
||||
env:
|
||||
MSYSTEM: MINGW64
|
||||
MSYS2_PATH_TYPE: inherit
|
||||
TFLAGS: "!165 !310 !1013 !1056 !1299 !1448 !2034 !2037 !2041 !2046 !2047 !3000 !3001"
|
||||
TFLAGS: "~165 ~310 ~1013 ~1056 ~1299 ~1448 ~2034 ~2037 ~2041 ~2046 ~2047 ~3000 ~3001"
|
||||
steps:
|
||||
- script: C:\msys64\usr\bin\sh -l -c "cd $(echo '%cd%') && ./buildconf && ./configure --host=x86_64-w64-mingw32 --build=x86_64-w64-mingw32 --enable-debug --enable-werror --enable-sspi --without-ssl --with-schannel --with-winidn"
|
||||
displayName: 'configure debug with sspi/schannel/winidn'
|
||||
@ -372,7 +372,7 @@ stages:
|
||||
container:
|
||||
image: mback2k/curl-docker-winbuildenv-msys1-mingw:ltsc2019
|
||||
env:
|
||||
TFLAGS: "!203 !305 !310 !311 !312 !313 !404 !1013 !1056 !1143 !2034 !2035 !2037 !2038 !2041 !2042 !2048 !3000 !3001"
|
||||
TFLAGS: "~203 ~305 ~310 ~311 ~312 ~313 ~404 ~1013 ~1056 ~1143 ~2034 ~2035 ~2037 ~2038 ~2041 ~2042 ~2048 ~3000 ~3001"
|
||||
steps:
|
||||
- script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && ./buildconf && ./configure --host=i686-pc-mingw32 --build=i686-pc-mingw32 --prefix=/mingw --enable-debug --enable-sspi --without-ssl --with-schannel --with-winidn"
|
||||
displayName: 'configure debug with sspi/schannel/winidn'
|
||||
@ -393,7 +393,7 @@ stages:
|
||||
container:
|
||||
image: mback2k/curl-docker-winbuildenv-msys1-mingw32:ltsc2019
|
||||
env:
|
||||
TFLAGS: "!203 !310 !1013 !1056 !1143 !1299 !2034 !2037 !2041 !3000 !3001"
|
||||
TFLAGS: "~203 ~310 ~1013 ~1056 ~1143 ~1299 ~2034 ~2037 ~2041 ~3000 ~3001"
|
||||
steps:
|
||||
- script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && ./buildconf && ./configure --host=i686-w64-mingw32 --build=i686-w64-mingw32 --prefix=/mingw32 --enable-debug --enable-werror --enable-sspi --without-ssl --with-schannel --with-winidn --without-zlib"
|
||||
displayName: 'configure debug with sspi/schannel/winidn without zlib'
|
||||
@ -414,7 +414,7 @@ stages:
|
||||
container:
|
||||
image: mback2k/curl-docker-winbuildenv-msys1-mingw64:ltsc2019
|
||||
env:
|
||||
TFLAGS: "!203 !310 !1013 !1056 !1143 !1299 !2034 !2037 !2041 !3000 !3001"
|
||||
TFLAGS: "~203 ~310 ~1013 ~1056 ~1143 ~1299 ~2034 ~2037 ~2041 ~3000 ~3001"
|
||||
steps:
|
||||
- script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && ./buildconf && ./configure --host=x86_64-w64-mingw32 --build=x86_64-w64-mingw32 --prefix=/mingw64 --enable-debug --enable-werror --enable-sspi --without-ssl --with-schannel --with-winidn --without-zlib"
|
||||
displayName: 'configure debug with sspi/schannel/winidn without zlib'
|
||||
|
||||
@ -63,7 +63,11 @@ sub appveyor_update_test_result {
|
||||
my $appveyor_duration = sprintf("%.0f", ($stop-$start)*1000);
|
||||
my $appveyor_outcome;
|
||||
my $appveyor_category;
|
||||
if($error < 0) {
|
||||
if($error == 2) {
|
||||
$appveyor_outcome = 'Ignored';
|
||||
$appveyor_category = 'Warning';
|
||||
}
|
||||
elsif($error < 0) {
|
||||
$appveyor_outcome = 'NotRunnable';
|
||||
$appveyor_category = 'Warning';
|
||||
}
|
||||
|
||||
@ -88,7 +88,10 @@ sub azure_update_test_result {
|
||||
my $azure_complete = strftime "%Y-%m-%dT%H:%M:%SZ", gmtime $stop;
|
||||
my $azure_duration = sprintf("%.0f", ($stop-$start)*1000);
|
||||
my $azure_outcome;
|
||||
if($error < 0) {
|
||||
if($error == 2) {
|
||||
$azure_outcome = 'Not applicable';
|
||||
}
|
||||
elsif($error < 0) {
|
||||
$azure_outcome = 'Not executed';
|
||||
}
|
||||
elsif(!$error) {
|
||||
|
||||
@ -276,8 +276,10 @@ my $skipped=0; # number of tests skipped; reported in main loop
|
||||
my %skipped; # skipped{reason}=counter, reasons for skip
|
||||
my @teststat; # teststat[testnum]=reason, reasons for skip
|
||||
my %disabled_keywords; # key words of tests to skip
|
||||
my %ignored_keywords; # key words of tests to ignore results
|
||||
my %enabled_keywords; # key words of tests to run
|
||||
my %disabled; # disabled test cases
|
||||
my %ignored; # ignored results of test cases
|
||||
|
||||
my $sshdid; # for socks server, ssh daemon version id
|
||||
my $sshdvernum; # for socks server, ssh daemon version number
|
||||
@ -3250,6 +3252,7 @@ sub singletest {
|
||||
my $why;
|
||||
my $cmd;
|
||||
my $disablevalgrind;
|
||||
my $errorreturncode = 1; # 1 means normal error, 2 means ignored error
|
||||
|
||||
# fist, remove all lingering log files
|
||||
cleardir($LOGDIR);
|
||||
@ -3267,6 +3270,10 @@ sub singletest {
|
||||
if($disabled{$testnum}) {
|
||||
logmsg "Warning: test$testnum is explicitly disabled\n";
|
||||
}
|
||||
if($ignored{$testnum}) {
|
||||
logmsg "Warning: test$testnum result is ignored\n";
|
||||
$errorreturncode = 2;
|
||||
}
|
||||
|
||||
# load the test case file definition
|
||||
if(loadtest("${TESTDIR}/test${testnum}")) {
|
||||
@ -3334,6 +3341,10 @@ sub singletest {
|
||||
} elsif ($enabled_keywords{lc($k)}) {
|
||||
$match = 1;
|
||||
}
|
||||
if ($ignored_keywords{lc($k)}) {
|
||||
logmsg "Warning: test$testnum result is ignored due to $k\n";
|
||||
$errorreturncode = 2;
|
||||
}
|
||||
|
||||
$keywords{$k} = 1;
|
||||
}
|
||||
@ -3918,7 +3929,7 @@ sub singletest {
|
||||
logmsg " postcheck FAILED\n";
|
||||
# timestamp test result verification end
|
||||
$timevrfyend{$testnum} = Time::HiRes::time();
|
||||
return 1;
|
||||
return $errorreturncode;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -3990,7 +4001,7 @@ sub singletest {
|
||||
|
||||
$res = compare($testnum, $testname, "stdout", \@actual, \@validstdout);
|
||||
if($res) {
|
||||
return 1;
|
||||
return $errorreturncode;
|
||||
}
|
||||
$ok .= "s";
|
||||
}
|
||||
@ -4041,7 +4052,7 @@ sub singletest {
|
||||
|
||||
$res = compare($testnum, $testname, "stderr", \@actual, \@validstderr);
|
||||
if($res) {
|
||||
return 1;
|
||||
return $errorreturncode;
|
||||
}
|
||||
$ok .= "r";
|
||||
}
|
||||
@ -4087,7 +4098,7 @@ sub singletest {
|
||||
|
||||
$res = compare($testnum, $testname, "protocol", \@out, \@protstrip);
|
||||
if($res) {
|
||||
return 1;
|
||||
return $errorreturncode;
|
||||
}
|
||||
|
||||
$ok .= "p";
|
||||
@ -4102,7 +4113,7 @@ sub singletest {
|
||||
my @out = loadarray($CURLOUT);
|
||||
$res = compare($testnum, $testname, "data", \@out, \@reply);
|
||||
if ($res) {
|
||||
return 1;
|
||||
return $errorreturncode;
|
||||
}
|
||||
$ok .= "d";
|
||||
}
|
||||
@ -4126,7 +4137,7 @@ sub singletest {
|
||||
|
||||
$res = compare($testnum, $testname, "upload", \@out, \@upload);
|
||||
if ($res) {
|
||||
return 1;
|
||||
return $errorreturncode;
|
||||
}
|
||||
$ok .= "u";
|
||||
}
|
||||
@ -4172,7 +4183,7 @@ sub singletest {
|
||||
|
||||
$res = compare($testnum, $testname, "proxy", \@out, \@protstrip);
|
||||
if($res) {
|
||||
return 1;
|
||||
return $errorreturncode;
|
||||
}
|
||||
|
||||
$ok .= "P";
|
||||
@ -4230,7 +4241,7 @@ sub singletest {
|
||||
$res = compare($testnum, $testname, "output ($filename)",
|
||||
\@generated, \@outfile);
|
||||
if($res) {
|
||||
return 1;
|
||||
return $errorreturncode;
|
||||
}
|
||||
|
||||
$outputok = 1; # output checked
|
||||
@ -4260,7 +4271,7 @@ sub singletest {
|
||||
logmsg " exit FAILED\n";
|
||||
# timestamp test result verification end
|
||||
$timevrfyend{$testnum} = Time::HiRes::time();
|
||||
return 1;
|
||||
return $errorreturncode;
|
||||
}
|
||||
|
||||
if($has_memory_tracking) {
|
||||
@ -4283,7 +4294,7 @@ sub singletest {
|
||||
logmsg @memdata;
|
||||
# timestamp test result verification end
|
||||
$timevrfyend{$testnum} = Time::HiRes::time();
|
||||
return 1;
|
||||
return $errorreturncode;
|
||||
}
|
||||
else {
|
||||
$ok .= "m";
|
||||
@ -4300,7 +4311,7 @@ sub singletest {
|
||||
logmsg "ERROR: unable to read $LOGDIR\n";
|
||||
# timestamp test result verification end
|
||||
$timevrfyend{$testnum} = Time::HiRes::time();
|
||||
return 1;
|
||||
return $errorreturncode;
|
||||
}
|
||||
my @files = readdir(DIR);
|
||||
closedir(DIR);
|
||||
@ -4315,7 +4326,7 @@ sub singletest {
|
||||
logmsg "ERROR: valgrind log file missing for test $testnum\n";
|
||||
# timestamp test result verification end
|
||||
$timevrfyend{$testnum} = Time::HiRes::time();
|
||||
return 1;
|
||||
return $errorreturncode;
|
||||
}
|
||||
my @e = valgrindparse("$LOGDIR/$vgfile");
|
||||
if(@e && $e[0]) {
|
||||
@ -4328,7 +4339,7 @@ sub singletest {
|
||||
}
|
||||
# timestamp test result verification end
|
||||
$timevrfyend{$testnum} = Time::HiRes::time();
|
||||
return 1;
|
||||
return $errorreturncode;
|
||||
}
|
||||
$ok .= "v";
|
||||
}
|
||||
@ -4367,6 +4378,10 @@ sub singletest {
|
||||
logmsg "PASS: $testnum - $testname\n";
|
||||
}
|
||||
|
||||
if($errorreturncode==2) {
|
||||
logmsg "Warning: test$testnum result is ignored, but passed!\n";
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -5164,8 +5179,10 @@ Usage: runtests.pl [options] [test selection(s)]
|
||||
-vc path use this curl only to verify the existing servers
|
||||
[num] like "5 6 9" or " 5 to 22 " to run those tests only
|
||||
[!num] like "!5 !6 !9" to disable those tests
|
||||
[~num] like "~5 ~6 ~9" to ignore the result of those tests
|
||||
[keyword] like "IPv6" to select only tests containing the key word
|
||||
[!keyword] like "!cookies" to disable any tests containing the key word
|
||||
[~keyword] like "~cookies" to ignore results of tests containing key word
|
||||
EOHELP
|
||||
;
|
||||
exit;
|
||||
@ -5198,9 +5215,16 @@ EOHELP
|
||||
$fromnum = -1;
|
||||
$disabled{$1}=$1;
|
||||
}
|
||||
elsif($ARGV[0] =~ /^~(\d+)/) {
|
||||
$fromnum = -1;
|
||||
$ignored{$1}=$1;
|
||||
}
|
||||
elsif($ARGV[0] =~ /^!(.+)/) {
|
||||
$disabled_keywords{lc($1)}=$1;
|
||||
}
|
||||
elsif($ARGV[0] =~ /^~(.+)/) {
|
||||
$ignored_keywords{lc($1)}=$1;
|
||||
}
|
||||
elsif($ARGV[0] =~ /^([-[{a-zA-Z].*)/) {
|
||||
$enabled_keywords{lc($1)}=$1;
|
||||
}
|
||||
@ -5532,6 +5556,7 @@ if(azure_check_environment()) {
|
||||
my $failed;
|
||||
my $testnum;
|
||||
my $ok=0;
|
||||
my $ign=0;
|
||||
my $total=0;
|
||||
my $lasttest=0;
|
||||
my @at = split(" ", $TESTCASES);
|
||||
@ -5568,7 +5593,10 @@ foreach $testnum (@at) {
|
||||
# display all files in log/ in a nice way
|
||||
displaylogs($testnum);
|
||||
}
|
||||
if(!$anyway) {
|
||||
if($error==2) {
|
||||
$ign++; # ignored test result counter
|
||||
}
|
||||
elsif(!$anyway) {
|
||||
# a test failed, abort
|
||||
logmsg "\n - abort tests\n";
|
||||
last;
|
||||
@ -5644,6 +5672,6 @@ if($skipped && !$short) {
|
||||
}
|
||||
}
|
||||
|
||||
if($total && ($ok != $total)) {
|
||||
if($total && (($ok+$ign) != $total)) {
|
||||
exit 1;
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user