url: add option CURLOPT_SUPPRESS_CONNECT_HEADERS

- Add new option CURLOPT_SUPPRESS_CONNECT_HEADERS to allow suppressing
  proxy CONNECT response headers from the user callback functions
  CURLOPT_HEADERFUNCTION and CURLOPT_WRITEFUNCTION.

- Add new tool option --suppress-connect-headers to expose
  CURLOPT_SUPPRESS_CONNECT_HEADERS and allow suppressing proxy CONNECT
  response headers from --dump-header and --include.

Assisted-by: Jay Satiro
Assisted-by: CarloCannas@users.noreply.github.com
Closes https://github.com/curl/curl/pull/783

docs/cmdline-opts/Makefile.am

@@ -59,7 +59,8 @@ DPAGES = abstract-unix-socket.d anyauth.d append.d basic.d cacert.d capath.d cer
   service-name.d show-error.d silent.d socks4a.d socks4.d socks5.d	\
   socks5-gssapi-nec.d socks5-gssapi-service.d socks5-hostname.d		\
   speed-limit.d speed-time.d ssl-allow-beast.d ssl.d ssl-no-revoke.d	\
-  ssl-reqd.d sslv2.d sslv3.d stderr.d tcp-fastopen.d tcp-nodelay.d	\
+  ssl-reqd.d sslv2.d sslv3.d stderr.d suppress-connect-headers.d	\
+  tcp-fastopen.d tcp-nodelay.d						\
   telnet-option.d tftp-blksize.d tftp-no-options.d time-cond.d		\
   tls-max.d \
   tlsauthtype.d tlspassword.d tlsuser.d tlsv1.0.d tlsv1.1.d tlsv1.2.d	\

docs/cmdline-opts/proxytunnel.d

@@ -8,3 +8,6 @@ to attempt to tunnel through the proxy instead of merely using it to do
 HTTP-like operations. The tunnel approach is made with the HTTP proxy CONNECT
 request and requires that the proxy allows direct connect to the remote port
 number curl wants to tunnel through to.
+
+To suppress proxy CONNECT response headers when curl is set to output headers
+use --suppress-connect-headers.

docs/cmdline-opts/suppress-connect-headers.d

@@ -0,0 +1,8 @@
+Long: suppress-connect-headers
+Help: Suppress proxy CONNECT response headers
+See-also: dump-header include proxytunnel
+---
+When --proxytunnel is used and a CONNECT request is made don't output proxy
+CONNECT response headers. This option is meant to be used with --dump-header or
+--include which are used to show protocol headers in the output. It has no
+effect on debug options such as --verbose or --trace, or any statistics.

docs/libcurl/curl_easy_setopt.3

@@ -137,6 +137,8 @@ Data pointer to pass to the chunk callbacks. See \fICURLOPT_CHUNK_DATA(3)\fP
 Callback for wildcard matching. See \fICURLOPT_FNMATCH_FUNCTION(3)\fP
 .IP CURLOPT_FNMATCH_DATA
 Data pointer to pass to the wildcard matching callback. See \fICURLOPT_FNMATCH_DATA(3)\fP
+.IP CURLOPT_SUPPRESS_CONNECT_HEADERS
+Suppress proxy CONNECT response headers from user callbacks. See \fICURLOPT_SUPPRESS_CONNECT_HEADERS(3)\fP
 .SH ERROR OPTIONS
 .IP CURLOPT_ERRORBUFFER
 Error message buffer. See \fICURLOPT_ERRORBUFFER(3)\fP

docs/libcurl/opts/CURLINFO_HEADER_SIZE.3

@@ -30,6 +30,9 @@ CURLcode curl_easy_getinfo(CURL *handle, CURLINFO_HEADER_SIZE, long *sizep);
 .SH DESCRIPTION
 Pass a pointer to a long to receive the total size of all the headers
 received. Measured in number of bytes.
+
+The total includes the size of any received headers suppressed by
+\fICURLOPT_SUPPRESS_CONNECT_HEADERS(3)\fP.
 .SH PROTOCOLS
 All
 .SH EXAMPLE

docs/libcurl/opts/CURLOPT_HTTPPROXYTUNNEL.3

@@ -40,6 +40,9 @@ it allows CONNECT requests to and often only port 80 and 443 are allowed.
 
 When using this, it only makes sense to use \fICURLOPT_PROXYTYPE(3)\fP set to
 a HTTP proxy.
+
+To suppress proxy CONNECT response headers from user callbacks use
+\fICURLOPT_SUPPRESS_CONNECT_HEADERS(3)\fP.
 .SH DEFAULT
 0
 .SH PROTOCOLS

docs/libcurl/opts/CURLOPT_SUPPRESS_CONNECT_HEADERS.3

@@ -0,0 +1,95 @@
+.\" **************************************************************************
+.\" *                                  _   _ ____  _
+.\" *  Project                     ___| | | |  _ \| |
+.\" *                             / __| | | | |_) | |
+.\" *                            | (__| |_| |  _ <| |___
+.\" *                             \___|\___/|_| \_\_____|
+.\" *
+.\" * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al.
+.\" *
+.\" * This software is licensed as described in the file COPYING, which
+.\" * you should have received as part of this distribution. The terms
+.\" * are also available at https://curl.haxx.se/docs/copyright.html.
+.\" *
+.\" * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+.\" * copies of the Software, and permit persons to whom the Software is
+.\" * furnished to do so, under the terms of the COPYING file.
+.\" *
+.\" * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+.\" * KIND, either express or implied.
+.\" *
+.\" **************************************************************************
+.\"
+.TH CURLOPT_SUPPRESS_CONNECT_HEADERS 3 "13 February 2017" "libcurl 7.54.0" "curl_easy_setopt options"
+.SH NAME
+CURLOPT_SUPPRESS_CONNECT_HEADERS \- Suppress proxy CONNECT response headers from user callbacks
+.SH SYNOPSIS
+.nf
+#include <curl/curl.h>
+
+CURLcode curl_easy_setopt(CURL *handle, CURLOPT_SUPPRESS_CONNECT_HEADERS, long onoff);
+.fi
+.SH DESCRIPTION
+When \fICURLOPT_HTTPPROXYTUNNEL(3)\fP is used and a CONNECT request is made,
+suppress proxy CONNECT response headers from the user callback functions
+\fICURLOPT_HEADERFUNCTION(3)\fP and \fICURLOPT_WRITEFUNCTION(3)\fP.
+
+Proxy CONNECT response headers can complicate header processing since it's
+essentially a separate set of headers. You can enable this option to suppress
+those headers.
+
+For example let's assume an HTTPS URL is to be retrieved via CONNECT. On
+success there would normally be two sets of headers, and each header line sent
+to the header function and/or the write function. The data given to the
+callbacks would look like this:
+
+.nf
+HTTP/1.1 200 Connection established
+{headers}...
+
+HTTP/1.1 200 OK
+Content-Type: application/json
+{headers}...
+
+{body}...
+.fi
+
+However by enabling this option the CONNECT response headers are suppressed, so
+the data given to the callbacks would look like this:
+
+.nf
+HTTP/1.1 200 OK
+Content-Type: application/json
+{headers}...
+
+{body}...
+.fi
+
+.SH DEFAULT
+0
+.SH PROTOCOLS
+All
+.SH EXAMPLE
+.nf
+CURL *curl = curl_easy_init();
+if(curl) {
+  curl_easy_setopt(curl, CURLOPT_URL, "https://example.com");
+
+  curl_easy_setopt(curl, CURLOPT_HEADER, 1L);
+  curl_easy_setopt(curl, CURLOPT_PROXY, "http://foo:3128");
+  curl_easy_setopt(curl, CURLOPT_HTTPPROXYTUNNEL, 1L);
+  curl_easy_setopt(curl, CURLOPT_SUPPRESS_CONNECT_HEADERS, 1L);
+
+  curl_easy_perform(curl);
+
+  /* always cleanup */
+  curl_easy_cleanup(curl);
+}
+.fi
+.SH AVAILABILITY
+Added in 7.54.0
+.SH RETURN VALUE
+CURLE_OK or an error such as CURLE_UNKNOWN_OPTION.
+.SH "SEE ALSO"
+.BR CURLOPT_HEADER "(3), " CURLOPT_PROXY "(3), "
+.BR CURLOPT_HTTPPROXYTUNNEL "(3), "

docs/libcurl/opts/Makefile.am

@@ -300,6 +300,7 @@ man_MANS =                                      \
  CURLOPT_STREAM_DEPENDS.3                       \
  CURLOPT_STREAM_DEPENDS_E.3                     \
  CURLOPT_STREAM_WEIGHT.3                        \
+ CURLOPT_SUPPRESS_CONNECT_HEADERS.3             \
  CURLOPT_TCP_FASTOPEN.3                         \
  CURLOPT_TCP_KEEPALIVE.3                        \
  CURLOPT_TCP_KEEPIDLE.3                         \

docs/libcurl/symbols-in-versions

@@ -559,6 +559,7 @@ CURLOPT_STDERR                  7.1
 CURLOPT_STREAM_DEPENDS          7.46.0
 CURLOPT_STREAM_DEPENDS_E        7.46.0
 CURLOPT_STREAM_WEIGHT           7.46.0
+CURLOPT_SUPPRESS_CONNECT_HEADERS 7.54.0
 CURLOPT_TCP_KEEPALIVE           7.25.0
 CURLOPT_TCP_KEEPIDLE            7.25.0
 CURLOPT_TCP_KEEPINTVL           7.25.0

include/curl/curl.h

@@ -1778,6 +1778,9 @@ typedef enum {
   /* Path to an abstract Unix domain socket */
   CINIT(ABSTRACT_UNIX_SOCKET, STRINGPOINT, 264),
 
+  /* Suppress proxy CONNECT response headers from user callbacks */
+  CINIT(SUPPRESS_CONNECT_HEADERS, LONG, 265),
+
   CURLOPT_LASTENTRY /* the last unused */
 } CURLoption;
 

lib/http_proxy.c

@@ -316,8 +316,6 @@ CURLcode Curl_proxyCONNECT(struct connectdata *conn,
       perline = 0;
 
       while(nread < BUFSIZE && keepon && !error) {
-        int writetype;
-
         if(Curl_pgrsUpdate(conn))
           return CURLE_ABORTED_BY_CALLBACK;
 
@@ -419,19 +417,20 @@ CURLcode Curl_proxyCONNECT(struct connectdata *conn,
           Curl_debug(data, CURLINFO_HEADER_IN,
                      line_start, (size_t)perline, conn);
 
-        /* send the header to the callback */
+        if(!data->set.suppress_connect_headers) {
-        writetype = CLIENTWRITE_HEADER;
+          /* send the header to the callback */
-        if(data->set.include_header)
+          int writetype = CLIENTWRITE_HEADER;
-          writetype |= CLIENTWRITE_BODY;
+          if(data->set.include_header)
+            writetype |= CLIENTWRITE_BODY;
 
-        result = Curl_client_write(conn, writetype, line_start, perline);
+          result = Curl_client_write(conn, writetype, line_start, perline);
+          if(result)
+            return result;
+        }
 
         data->info.header_size += (long)perline;
         data->req.headerbytecount += (long)perline;
 
-        if(result)
-          return result;
-
         /* Newlines are CRLF, so the CR is ignored as the line isn't
            really terminated until the LF comes. Treat a following CR
            as end-of-headers as well.*/

lib/url.c

@@ -2894,6 +2894,9 @@ CURLcode Curl_setopt(struct Curl_easy *data, CURLoption option,
   case CURLOPT_CONNECT_TO:
     data->set.connect_to = va_arg(param, struct curl_slist *);
     break;
+  case CURLOPT_SUPPRESS_CONNECT_HEADERS:
+    data->set.suppress_connect_headers = (0 != va_arg(param, long))?TRUE:FALSE;
+    break;
   default:
     /* unknown tag and its companion, just ignore: */
     result = CURLE_UNKNOWN_OPTION;

lib/urldata.h

@@ -1756,6 +1756,8 @@ struct UserDefined {
   bool pipewait;        /* wait for pipe/multiplex status before starting a
                            new connection */
   long expect_100_timeout; /* in milliseconds */
+  bool suppress_connect_headers;  /* suppress proxy CONNECT response headers
+                                     from user callbacks */
 
   struct Curl_easy *stream_depends_on;
   bool stream_depends_e; /* set or don't set the Exclusive bit */

packages/OS400/curl.inc.in

@@ -1310,6 +1310,8 @@
      d                 c                   10263
      d  CURLOPT_ABSTRACT_UNIX_SOCKET...
      d                 c                   10264
+     d  CURLOPT_SUPPRESS_CONNECT_HEADERS...
+     d                 c                   00265
       *
       /if not defined(CURL_NO_OLDIES)
      d  CURLOPT_FILE   c                   10001

src/tool_cfgable.h

@@ -235,6 +235,8 @@ struct OperationConfig {
   bool falsestart;
   bool path_as_is;
   double expect100timeout;
+  bool suppress_connect_headers;  /* suppress proxy CONNECT response headers
+                                     from user callbacks */
   struct GlobalConfig *global;
   struct OperationConfig *prev;
   struct OperationConfig *next;   /* Always last in the struct */

src/tool_getparam.c

@@ -185,6 +185,7 @@ static const struct LongShort aliases[]= {
   {"$U", "connect-to",               TRUE},
   {"$W", "abstract-unix-socket",     TRUE},
   {"$X", "tls-max",                  TRUE},
+  {"$Y", "suppress-connect-headers", FALSE},
   {"0",   "http1.0",                 FALSE},
   {"01",  "http1.1",                 FALSE},
   {"02",  "http2",                   FALSE},
@@ -1066,6 +1067,9 @@ ParameterError getparameter(char *flag,    /* f or -long-flag */
         if(err)
           return err;
         break;
+      case 'Y': /* --suppress-connect-headers */
+        config->suppress_connect_headers = toggle;
+        break;
       }
       break;
     case '#': /* --progress-bar */

src/tool_help.c

@@ -249,6 +249,7 @@ static const char *const helptext[] = {
   "     --ssl-allow-beast  Allow security flaw to improve interop (SSL)",
   "     --ssl-no-revoke    Disable cert revocation checks (WinSSL)",
   "     --stderr FILE   Where to redirect stderr (use \"-\" for stdout)",
+  "     --suppress-connect-headers  Suppress proxy CONNECT response headers",
   "     --tcp-nodelay   Use the TCP_NODELAY option",
   "     --tcp-fastopen  Use TCP Fast Open",
   " -t, --telnet-option OPT=VAL  Set telnet option",

src/tool_operate.c

@@ -886,8 +886,11 @@ static CURLcode operate_do(struct GlobalConfig *global,
 
           /* new in libcurl 7.19.4 */
           my_setopt_str(curl, CURLOPT_NOPROXY, config->noproxy);
+
+          my_setopt(curl, CURLOPT_SUPPRESS_CONNECT_HEADERS,
+                    config->suppress_connect_headers?1L:0L);
         }
-#endif
+#endif /* !CURL_DISABLE_PROXY */
 
         my_setopt(curl, CURLOPT_FAILONERROR, config->failonerror?1L:0L);
         my_setopt(curl, CURLOPT_UPLOAD, uploadfile?1L:0L);

tests/data/Makefile.inc

@@ -132,6 +132,7 @@ test1252 test1253 test1254 test1255 test1256 test1257 test1258 test1259 \
 test1260 \
 \
 test1280 test1281 test1282 test1283 test1284 test1285 test1286 test1287 \
+test1288 \
 \
 test1300 test1301 test1302 test1303 test1304 test1305 test1306 test1307 \
 test1308 test1309 test1310 test1311 test1312 test1313 test1314 test1315 \

tests/data/test1288

@@ -0,0 +1,96 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+HTTP CONNECT
+HTTP proxy
+proxytunnel
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<connect>
+HTTP/1.1 200 Mighty fine indeed
+Server: test tunnel 2000
+
+</connect>
+
+<data nocheck="yes">
+HTTP/1.1 200 OK
+Date: Thu, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Content-Type: text/html
+Funny-head: yesyes
+Content-Length: 9
+Connection: keep-alive
+
+contents
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+http-proxy
+</server>
+<name>
+Suppress proxy CONNECT response headers
+</name>
+<command>
+--proxytunnel --suppress-connect-headers --dump-header - --include --write-out "\nCONNECT CODE: %{http_connect}\nRECEIVED HEADER BYTE TOTAL: %{size_header}\n" --proxy %HOSTIP:%PROXYPORT http://%HOSTIP.1288:%HTTPPORT/we/want/that/page/1288
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<strip>
+^User-Agent:.*
+</strip>
+<proxy>
+CONNECT %HOSTIP.1288:%HTTPPORT HTTP/1.1
+Host: %HOSTIP.1288:%HTTPPORT
+Proxy-Connection: Keep-Alive
+
+</proxy>
+<protocol>
+GET /we/want/that/page/1288 HTTP/1.1
+Host: %HOSTIP.1288:%HTTPPORT
+Accept: */*
+
+</protocol>
+
+# This test is structured to test all the expectations of
+# --suppress-connect-headers, which are:
+# Must suppress in --include and --dump-header
+# Must not suppress in --verbose and --trace
+# Must not suppress in statistics (eg received header byte total)
+<stdout>
+HTTP/1.1 200 OK
+HTTP/1.1 200 OK
+Date: Thu, 09 Nov 2010 14:49:00 GMT
+Date: Thu, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Server: test-server/fake
+Content-Type: text/html
+Content-Type: text/html
+Funny-head: yesyes
+Funny-head: yesyes
+Content-Length: 9
+Content-Length: 9
+Connection: keep-alive
+Connection: keep-alive
+
+
+contents
+
+CONNECT CODE: 200
+RECEIVED HEADER BYTE TOTAL: 231
+</stdout>
+</verify>
+</testcase>