mirror of
https://github.com/moparisthebest/curl
synced 2024-11-04 16:45:06 -05:00
ci/tests: Send test results to Azure DevOps for reporting
This commit is contained in:
Marc Hoersken
parent
fa0216b294
commit
ada581f2cc
@ -29,6 +29,8 @@ stages:
|
||||
|
||||
- script: make test-nonflaky
|
||||
displayName: 'test'
|
||||
env:
|
||||
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
|
||||
|
||||
- job: disable_ipv6
|
||||
displayName: ubuntu w/o IPv6
|
||||
@ -43,6 +45,8 @@ stages:
|
||||
|
||||
- script: make test-nonflaky
|
||||
displayName: 'test'
|
||||
env:
|
||||
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
|
||||
|
||||
- job: disable_http_smtp_imap
|
||||
displayName: ubuntu w/o HTTP/SMTP/IMAP
|
||||
@ -57,6 +61,8 @@ stages:
|
||||
|
||||
- script: make test-nonflaky
|
||||
displayName: 'test'
|
||||
env:
|
||||
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
|
||||
|
||||
- job: disable_thredres
|
||||
displayName: ubuntu sync resolver
|
||||
@ -71,6 +77,8 @@ stages:
|
||||
|
||||
- script: make test-nonflaky
|
||||
displayName: 'test'
|
||||
env:
|
||||
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
|
||||
|
||||
- job: http_only
|
||||
displayName: ubuntu HTTP only
|
||||
@ -85,6 +93,8 @@ stages:
|
||||
|
||||
- script: make test-nonflaky
|
||||
displayName: 'test'
|
||||
env:
|
||||
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
|
||||
|
||||
- stage: linux_torture
|
||||
dependsOn: linux
|
||||
@ -129,6 +139,8 @@ stages:
|
||||
|
||||
- script: make test-nonflaky
|
||||
displayName: 'test'
|
||||
env:
|
||||
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
|
||||
|
||||
- job: macos_libssh2
|
||||
displayName: macos libssh2
|
||||
@ -146,6 +158,8 @@ stages:
|
||||
|
||||
- script: make test-nonflaky
|
||||
displayName: 'test'
|
||||
env:
|
||||
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
|
||||
|
||||
- job: macos_cmake
|
||||
displayName: macos cmake openssl
|
||||
@ -205,6 +219,8 @@ stages:
|
||||
|
||||
- script: C:\msys64\usr\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky"
|
||||
displayName: 'test'
|
||||
env:
|
||||
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
|
||||
|
||||
- job: windows_msys2_mingw64_debug_openssl
|
||||
displayName: msys2 mingw64 debug openssl
|
||||
@ -226,6 +242,8 @@ stages:
|
||||
|
||||
- script: C:\msys64\usr\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky"
|
||||
displayName: 'test'
|
||||
env:
|
||||
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
|
||||
|
||||
- job: windows_msys1_mingw_debug_openssl
|
||||
displayName: msys1 mingw debug openssl
|
||||
@ -245,6 +263,8 @@ stages:
|
||||
|
||||
- script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky"
|
||||
displayName: 'test'
|
||||
env:
|
||||
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
|
||||
|
||||
- job: windows_msys1_mingw32_debug_openssl
|
||||
displayName: msys1 mingw32 debug openssl
|
||||
@ -264,6 +284,8 @@ stages:
|
||||
|
||||
- script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky"
|
||||
displayName: 'test'
|
||||
env:
|
||||
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
|
||||
|
||||
- job: windows_msys1_mingw64_debug_openssl
|
||||
displayName: msys1 mingw64 debug openssl
|
||||
@ -283,6 +305,8 @@ stages:
|
||||
|
||||
- script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky"
|
||||
displayName: 'test'
|
||||
env:
|
||||
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
|
||||
|
||||
- job: windows_msys2_mingw32_debug_schannel
|
||||
displayName: msys2 mingw32 debug schannel
|
||||
@ -304,6 +328,8 @@ stages:
|
||||
|
||||
- script: C:\msys64\usr\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky"
|
||||
displayName: 'test'
|
||||
env:
|
||||
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
|
||||
|
||||
- job: windows_msys2_mingw64_debug_schannel
|
||||
displayName: msys2 mingw64 debug schannel
|
||||
@ -325,6 +351,8 @@ stages:
|
||||
|
||||
- script: C:\msys64\usr\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky"
|
||||
displayName: 'test'
|
||||
env:
|
||||
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
|
||||
|
||||
- job: windows_msys1_mingw_debug_schannel
|
||||
displayName: msys1 mingw debug schannel
|
||||
@ -344,6 +372,8 @@ stages:
|
||||
|
||||
- script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky"
|
||||
displayName: 'test'
|
||||
env:
|
||||
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
|
||||
|
||||
- job: windows_msys1_mingw32_debug_schannel
|
||||
displayName: msys1 mingw32 debug schannel
|
||||
@ -363,6 +393,8 @@ stages:
|
||||
|
||||
- script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky"
|
||||
displayName: 'test'
|
||||
env:
|
||||
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
|
||||
|
||||
- job: windows_msys1_mingw64_debug_schannel
|
||||
displayName: msys1 mingw64 debug schannel
|
||||
@ -382,3 +414,5 @@ stages:
|
||||
|
||||
- script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky"
|
||||
displayName: 'test'
|
||||
env:
|
||||
AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
|
||||
|
||||
140
tests/azure.pm
Normal file
140
tests/azure.pm
Normal file
@ -0,0 +1,140 @@
|
||||
#***************************************************************************
|
||||
# _ _ ____ _
|
||||
# Project ___| | | | _ \| |
|
||||
# / __| | | | |_) | |
|
||||
# | (__| |_| | _ <| |___
|
||||
# \___|\___/|_| \_\_____|
|
||||
#
|
||||
# Copyright (C) 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
|
||||
# Copyright (C) 2020, Marc Hoersken, <info@marc-hoersken.de>
|
||||
#
|
||||
# This software is licensed as described in the file COPYING, which
|
||||
# you should have received as part of this distribution. The terms
|
||||
# are also available at https://curl.haxx.se/docs/copyright.html.
|
||||
#
|
||||
# You may opt to use, copy, modify, merge, publish, distribute and/or sell
|
||||
# copies of the Software, and permit persons to whom the Software is
|
||||
# furnished to do so, under the terms of the COPYING file.
|
||||
#
|
||||
# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
|
||||
# KIND, either express or implied.
|
||||
#
|
||||
###########################################################################
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
|
||||
use POSIX qw(strftime);
|
||||
|
||||
sub azure_check_environment {
|
||||
if(defined $ENV{'AZURE_ACCESS_TOKEN'} &&
|
||||
defined $ENV{'AGENT_JOBNAME'} && $ENV{'BUILD_BUILDID'} &&
|
||||
defined $ENV{'SYSTEM_TEAMFOUNDATIONCOLLECTIONURI'} &&
|
||||
defined $ENV{'SYSTEM_TEAMPROJECTID'}) {
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
sub azure_create_test_run {
|
||||
my $azure_baseurl="$ENV{'SYSTEM_TEAMFOUNDATIONCOLLECTIONURI'}$ENV{'SYSTEM_TEAMPROJECTID'}";
|
||||
my $azure_run=`curl --silent \\
|
||||
--header "Authorization: Bearer $ENV{'AZURE_ACCESS_TOKEN'}" \\
|
||||
--header "Content-Type: application/json" \\
|
||||
--data "
|
||||
{
|
||||
'name': '$ENV{'AGENT_JOBNAME'}',
|
||||
'automated': true,
|
||||
'build': {'id': '$ENV{'BUILD_BUILDID'}'}
|
||||
}
|
||||
" \\
|
||||
"$azure_baseurl/_apis/test/runs?api-version=5.0"`;
|
||||
if($azure_run =~ /"id":(\d+)/) {
|
||||
return $1;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
sub azure_create_test_result {
|
||||
my ($azure_run_id, $testnum, $testname)=@_;
|
||||
my $azure_baseurl="$ENV{'SYSTEM_TEAMFOUNDATIONCOLLECTIONURI'}$ENV{'SYSTEM_TEAMPROJECTID'}";
|
||||
my $azure_result=`curl --silent \\
|
||||
--header "Authorization: Bearer $ENV{'AZURE_ACCESS_TOKEN'}" \\
|
||||
--header "Content-Type: application/json" \\
|
||||
--data "
|
||||
[
|
||||
{
|
||||
'build': {'id': '$ENV{'BUILD_BUILDID'}'},
|
||||
'testCase': {'id': $testnum},
|
||||
'testCaseTitle': '$testname',
|
||||
'automatedTestName': 'curl.tests.$testnum',
|
||||
'outcome': 'InProgress'
|
||||
}
|
||||
]
|
||||
" \\
|
||||
"$azure_baseurl/_apis/test/runs/$azure_run_id/results?api-version=5.0"`;
|
||||
if($azure_result =~ /\[\{"id":(\d+)/) {
|
||||
return $1;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
sub azure_update_test_result {
|
||||
my ($azure_run_id, $azure_result_id, $testnum, $error, $start, $stop)=@_;
|
||||
if(!defined $stop) {
|
||||
$stop = $start;
|
||||
}
|
||||
my $azure_start = strftime "%Y-%m-%dT%H:%M:%SZ", gmtime $start;
|
||||
my $azure_complete = strftime "%Y-%m-%dT%H:%M:%SZ", gmtime $stop;
|
||||
my $azure_duration = sprintf("%.0f", ($stop-$start)*1000);
|
||||
my $azure_outcome;
|
||||
if($error < 0) {
|
||||
$azure_outcome = 'Not executed';
|
||||
}
|
||||
elsif(!$error) {
|
||||
$azure_outcome = 'Passed';
|
||||
}
|
||||
else {
|
||||
$azure_outcome = 'Failed';
|
||||
}
|
||||
my $azure_baseurl="$ENV{'SYSTEM_TEAMFOUNDATIONCOLLECTIONURI'}$ENV{'SYSTEM_TEAMPROJECTID'}";
|
||||
my $azure_result=`curl --silent --request PATCH \\
|
||||
--header "Authorization: Bearer $ENV{'AZURE_ACCESS_TOKEN'}" \\
|
||||
--header "Content-Type: application/json" \\
|
||||
--data "
|
||||
[
|
||||
{
|
||||
'id': $azure_result_id,
|
||||
'outcome': '$azure_outcome',
|
||||
'startedDate': '$azure_start',
|
||||
'completedDate': '$azure_complete',
|
||||
'durationInMs': $azure_duration
|
||||
}
|
||||
]
|
||||
" \\
|
||||
"$azure_baseurl/_apis/test/runs/$azure_run_id/results?api-version=5.0"`;
|
||||
if($azure_result =~ /\[\{"id":(\d+)/) {
|
||||
return $1;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
sub azure_update_test_run {
|
||||
my ($azure_run_id)=@_;
|
||||
my $azure_baseurl="$ENV{'SYSTEM_TEAMFOUNDATIONCOLLECTIONURI'}$ENV{'SYSTEM_TEAMPROJECTID'}";
|
||||
my $azure_run=`curl --silent --request PATCH \\
|
||||
--header "Authorization: Bearer $ENV{'AZURE_ACCESS_TOKEN'}" \\
|
||||
--header "Content-Type: application/json" \\
|
||||
--data "
|
||||
{
|
||||
'state': 'Completed'
|
||||
}
|
||||
" \\
|
||||
"$azure_baseurl/_apis/test/runs/$azure_run_id?api-version=5.0"`;
|
||||
if($azure_run =~ /"id":(\d+)/) {
|
||||
return $1;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
1;
|
||||
@ -111,6 +111,7 @@ use pathhelp;
|
||||
require "getpart.pm"; # array functions
|
||||
require "valgrind.pm"; # valgrind report parser
|
||||
require "ftp.pm";
|
||||
require "azure.pm";
|
||||
|
||||
my $HOSTIP="127.0.0.1"; # address on which the test server listens
|
||||
my $HOST6IP="[::1]"; # address on which the test server listens
|
||||
@ -327,6 +328,10 @@ my $tortalloc;
|
||||
my $shallow;
|
||||
my $randseed = 0;
|
||||
|
||||
# Azure Pipelines specific variables
|
||||
my $AZURE_RUN_ID = 0;
|
||||
my $AZURE_RESULT_ID = 0;
|
||||
|
||||
#######################################################################
|
||||
# logmsg is our general message logging subroutine.
|
||||
#
|
||||
@ -3732,6 +3737,10 @@ sub singletest {
|
||||
close(GDBCMD);
|
||||
}
|
||||
|
||||
if(azure_check_environment() && $AZURE_RUN_ID) {
|
||||
$AZURE_RESULT_ID = azure_create_test_result($AZURE_RUN_ID, $testnum, $testname);
|
||||
}
|
||||
|
||||
# timestamp starting of test command
|
||||
$timetoolini{$testnum} = Time::HiRes::time();
|
||||
|
||||
@ -4346,7 +4355,6 @@ sub singletest {
|
||||
logmsg "PASS: $testnum - $testname\n";
|
||||
}
|
||||
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -5504,6 +5512,15 @@ sub displaylogs {
|
||||
}
|
||||
}
|
||||
|
||||
#######################################################################
|
||||
# Setup Azure Pipelines Test Run (if running in Azure DevOps)
|
||||
#
|
||||
|
||||
if(azure_check_environment()) {
|
||||
$AZURE_RUN_ID = azure_create_test_run();
|
||||
logmsg "Azure Run ID: $AZURE_RUN_ID\n" if ($verbose);
|
||||
}
|
||||
|
||||
#######################################################################
|
||||
# The main test-loop
|
||||
#
|
||||
@ -5524,6 +5541,12 @@ foreach $testnum (@at) {
|
||||
$count++;
|
||||
|
||||
my $error = singletest($run_event_based, $testnum, $count, scalar(@at));
|
||||
|
||||
if(azure_check_environment() && $AZURE_RUN_ID && $AZURE_RESULT_ID) {
|
||||
$AZURE_RESULT_ID = azure_update_test_result($AZURE_RUN_ID, $AZURE_RESULT_ID, $testnum, $error,
|
||||
$timeprepini{$testnum}, $timevrfyend{$testnum});
|
||||
}
|
||||
|
||||
if($error < 0) {
|
||||
# not a test we can run
|
||||
next;
|
||||
@ -5552,6 +5575,14 @@ foreach $testnum (@at) {
|
||||
|
||||
my $sofar = time() - $start;
|
||||
|
||||
#######################################################################
|
||||
# Finish Azure Pipelines Test Run (if running in Azure DevOps)
|
||||
#
|
||||
|
||||
if(azure_check_environment() && $AZURE_RUN_ID) {
|
||||
$AZURE_RUN_ID = azure_update_test_run($AZURE_RUN_ID);
|
||||
}
|
||||
|
||||
#######################################################################
|
||||
# Close command log
|
||||
#
|
||||
|
||||
Loading…
Reference in New Issue
Block a user