tests: add https-proxy support to the test suite

Initial test 1630 added with basic HTTPS-proxy use. HTTPS-proxy is like
HTTP proxy but with a full TLS connection to the proxy.

Closes #5399

tests/FILEFORMAT.md

@@ -45,6 +45,7 @@ Available substitute variables include:
 - `%HTTP6PORT` - IPv6 port number of the HTTP server
 - `%HTTPPORT` - Port number of the HTTP server
 - `%HTTPSPORT` - Port number of the HTTPS server
+- `%HTTPSPROXYPORT` - Port number of the HTTPS-proxy
 - `%HTTPTLS6PORT` - IPv6 port number of the HTTP TLS server
 - `%HTTPTLSPORT` - Port number of the HTTP TLS server
 - `%HTTPUNIXPATH` - Path to the Unix socket of the HTTP server

tests/data/Makefile.inc

@@ -194,6 +194,8 @@ test1608 test1609 test1610 test1611 test1612 \
 \
 test1620 test1621 \
 \
+test1630 \
+\
 test1650 test1651 test1652 test1653 test1654 test1655 \
 \
 test1700 test1701 test1702 \

tests/data/test1630

@@ -0,0 +1,56 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+HTTPS-proxy
+HTTPS-proxy Basic auth
+</keywords>
+</info>
+
+# Server-side
+<reply>
+<data>
+HTTP/1.1 200 OK
+Date: Thu, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Content-Type: text/html
+Funny-head: yesyes
+Content-Length: 22
+
+the content goes here
+</data>
+</reply>
+
+# Client-side
+<client>
+<server>
+http
+https-proxy
+</server>
+ <name>
+HTTPS-proxy with Basic auth to HTTP without auth
+ </name>
+ <command>
+ -U fake@user:loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong -x https://%HOSTIP:%HTTPSPROXYPORT http://we.want.that.site.com/1630 --proxy-insecure
+</command>
+<features>
+proxy
+</features>
+</client>
+
+# Verify data after the test has been "shot"
+<verify>
+<strip>
+^User-Agent:.*
+</strip>
+<proxy>
+GET http://we.want.that.site.com/1630 HTTP/1.1
+Host: we.want.that.site.com
+Proxy-Authorization: Basic ZmFrZUB1c2VyOmxvb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29vb29uZw==
+Accept: */*
+Proxy-Connection: Keep-Alive
+
+</proxy>
+</verify>
+</testcase>

tests/runtests.pl

@@ -137,6 +137,7 @@ my $MQTTPORT=$noport;    # MQTT server port
 my $HTTPPORT=$noport;    # HTTP server port
 my $HTTP6PORT=$noport;   # HTTP IPv6 server port
 my $HTTPSPORT=$noport;   # HTTPS (stunnel) server port
+my $HTTPSPROXYPORT = $noport; # HTTPS-proxy (stunnel) port
 my $FTPPORT=$noport;     # FTP server port
 my $FTPSPORT=$noport;    # FTPS (stunnel) server port
 my $FTP6PORT=$noport;    # FTP IPv6 server port
@@ -250,6 +251,7 @@ my $has_charconv;   # set if libcurl is built with CharConv support
 my $has_tls_srp;    # set if libcurl is built with TLS-SRP support
 my $has_metalink;   # set if curl is built with Metalink support
 my $has_http2;      # set if libcurl is built with HTTP2 support
+my $has_httpsproxy; # set if libcurl is built with HTTPS-proxy support
 my $has_crypto;     # set if libcurl is built with cryptographic support
 my $has_cares;      # set if built with c-ares
 my $has_threadedres;# set if built with threaded resolver
@@ -1588,7 +1590,7 @@ sub runhttpserver {
 # start the https stunnel based server
 #
 sub runhttpsserver {
-    my ($verbose, $ipv6, $certfile) = @_;
+    my ($verbose, $ipv6, $proxy, $certfile) = @_;
     my $proto = 'https';
     my $ip = ($ipv6 && ($ipv6 =~ /6$/)) ? "$HOST6IP" : "$HOSTIP";
     my $ipvnum = ($ipv6 && ($ipv6 =~ /6$/)) ? 6 : 4;
@@ -1599,6 +1601,11 @@ sub runhttpsserver {
     my $logfile;
     my $flags = "";
 
+    if($proxy eq "proxy") {
+        # the https-proxy runs as https2
+        $idnum = 2;
+    }
+
     if(!$stunnel) {
         return (0,0);
     }
@@ -1630,7 +1637,13 @@ sub runhttpsserver {
     $flags .= "--ipv$ipvnum --proto $proto ";
     $flags .= "--certfile \"$certfile\" " if($certfile ne 'stunnel.pem');
     $flags .= "--stunnel \"$stunnel\" --srcdir \"$srcdir\" ";
-    $flags .= "--connect $HTTPPORT";
+    if(!$proxy) {
+        $flags .= "--connect $HTTPPORT";
+    }
+    else {
+        # for HTTPS-proxy we connect to the HTTP proxy
+        $flags .= "--connect $HTTPPROXYPORT";
+    }
 
     my $pid2;
     my $pid3;
@@ -2780,43 +2793,44 @@ sub compare {
 }
 
 sub setupfeatures {
-    $feature{"SSL"} = $has_ssl;
+    $feature{"alt-svc"} = $has_altsvc;
-    $feature{"MultiSSL"} = $has_multissl;
+    $feature{"brotli"} = $has_brotli;
-    $feature{"SSLpinning"} = $has_sslpinning;
+    $feature{"crypto"} = $has_crypto;
-    $feature{"OpenSSL"} = $has_openssl;
-    $feature{"GnuTLS"} = $has_gnutls;
-    $feature{"NSS"} = $has_nss;
-    $feature{"WinSSL"} = $has_winssl;
-    $feature{"Schannel"} = $has_winssl; # alias
-    $feature{"sectransp"} = $has_darwinssl;
     $feature{"DarwinSSL"} = $has_darwinssl; # alias
-    $feature{"ld_preload"} = ($has_ldpreload && !$debug_build);
-    $feature{"unittest"} = $debug_build;
     $feature{"debug"} = $debug_build;
-    $feature{"TrackMemory"} = $has_memory_tracking;
+    $feature{"getrlimit"} = $has_getrlimit;
-    $feature{"large_file"} = $has_largefile;
+    $feature{"GnuTLS"} = $has_gnutls;
+    $feature{"GSS-API"} = $has_gssapi;
+    $feature{"http/2"} = $has_http2;
+    $feature{"https-proxy"} = $has_httpsproxy;
     $feature{"idn"} = $has_idn;
     $feature{"ipv6"} = $has_ipv6;
+    $feature{"Kerberos"} = $has_kerberos;
+    $feature{"large_file"} = $has_largefile;
+    $feature{"ld_preload"} = ($has_ldpreload && !$debug_build);
     $feature{"libz"} = $has_libz;
-    $feature{"brotli"} = $has_brotli;
+    $feature{"manual"} = $has_manual;
+    $feature{"Metalink"} = $has_metalink;
+    $feature{"MinGW"} = $has_mingw;
+    $feature{"MultiSSL"} = $has_multissl;
+    $feature{"NSS"} = $has_nss;
     $feature{"NTLM"} = $has_ntlm;
     $feature{"NTLM_WB"} = $has_ntlm_wb;
-    $feature{"SSPI"} = $has_sspi;
+    $feature{"OpenSSL"} = $has_openssl;
-    $feature{"GSS-API"} = $has_gssapi;
-    $feature{"Kerberos"} = $has_kerberos;
-    $feature{"SPNEGO"} = $has_spnego;
-    $feature{"getrlimit"} = $has_getrlimit;
-    $feature{"crypto"} = $has_crypto;
-    $feature{"TLS-SRP"} = $has_tls_srp;
-    $feature{"Metalink"} = $has_metalink;
-    $feature{"http/2"} = $has_http2;
-    $feature{"threaded-resolver"} = $has_threadedres;
     $feature{"PSL"} = $has_psl;
-    $feature{"alt-svc"} = $has_altsvc;
+    $feature{"Schannel"} = $has_winssl; # alias
-    $feature{"manual"} = $has_manual;
+    $feature{"sectransp"} = $has_darwinssl;
+    $feature{"SPNEGO"} = $has_spnego;
+    $feature{"SSL"} = $has_ssl;
+    $feature{"SSLpinning"} = $has_sslpinning;
+    $feature{"SSPI"} = $has_sspi;
+    $feature{"threaded-resolver"} = $has_threadedres;
+    $feature{"TLS-SRP"} = $has_tls_srp;
+    $feature{"TrackMemory"} = $has_memory_tracking;
+    $feature{"unittest"} = $debug_build;
     $feature{"unix-sockets"} = $has_unix;
     $feature{"win32"} = $has_win32;
-    $feature{"MinGW"} = $has_mingw;
+    $feature{"WinSSL"} = $has_winssl;
 
     # make each protocol an enabled "feature"
     for my $p (@protocols) {
@@ -3063,6 +3077,12 @@ sub checksystem {
 
                 push @protocols, 'http/2';
             }
+            if($feat =~ /HTTPS-proxy/) {
+                $has_httpsproxy=1;
+
+                # 'https-proxy' is used as "server" so consider it a protocol
+                push @protocols, 'https-proxy';
+            }
         }
         #
         # Test harness currently uses a non-stunnel server in order to
@@ -3244,6 +3264,7 @@ sub subVariables {
     $$thing =~ s/${prefix}HTTPTLSPORT/$HTTPTLSPORT/g;
     $$thing =~ s/${prefix}HTTP6PORT/$HTTP6PORT/g;
     $$thing =~ s/${prefix}HTTPSPORT/$HTTPSPORT/g;
+    $$thing =~ s/${prefix}HTTPSPROXYPORT/$HTTPSPROXYPORT/g;
     $$thing =~ s/${prefix}HTTP2PORT/$HTTP2PORT/g;
     $$thing =~ s/${prefix}HTTPPORT/$HTTPPORT/g;
     $$thing =~ s/${prefix}PROXYPORT/$HTTPPROXYPORT/g;
@@ -4769,7 +4790,7 @@ sub startservers {
             }
             if(!$run{'https'}) {
                 ($pid, $pid2, $HTTPSPORT) =
-                    runhttpsserver($verbose, "", $certfile);
+                    runhttpsserver($verbose, "", "", $certfile);
                 if($pid <= 0) {
                     return "failed starting HTTPS server (stunnel)";
                 }
@@ -4778,6 +4799,35 @@ sub startservers {
                 $run{'https'}="$pid $pid2";
             }
         }
+        elsif($what eq "https-proxy") {
+            if(!$stunnel) {
+                # we can't run https-proxy tests without stunnel
+                return "no stunnel";
+            }
+            if($runcert{'https-proxy'} &&
+               ($runcert{'https-proxy'} ne $certfile)) {
+                # stop server when running and using a different cert
+                stopserver('https-proxy');
+            }
+
+            # we front the http-proxy with stunnel so we need to make sure the
+            # proxy runs as well
+            my $f = startservers("http-proxy");
+            if($f) {
+                return $f;1
+            }
+
+            if(!$run{'https-proxy'}) {
+                ($pid, $pid2, $HTTPSPROXYPORT) =
+                    runhttpsserver($verbose, "", "proxy", $certfile);
+                if($pid <= 0) {
+                    return "failed starting HTTPS-proxy (stunnel)";
+                }
+                logmsg sprintf("* pid https-proxy => %d %d\n", $pid, $pid2)
+                    if($verbose);
+                $run{'https-proxy'}="$pid $pid2";
+            }
+        }
         elsif($what eq "httptls") {
             if(!$httptlssrv) {
                 # for now, we can't run http TLS-EXT tests without gnutls-serv