transfer-encoding: added new option and cmdline

Added CURLOPT_TRANSFER_ENCODING as the option to set to request Transfer
Encoding in HTTP requests (if built zlib enabled). I also renamed
CURLOPT_ENCODING to CURLOPT_ACCEPT_ENCODING (while keeping the old name
around) to reduce the confusion when we have to encoding options for
HTTP.

--tr-encoding is now the new command line option for curl to request
this, and thus I updated the test cases accordingly.

docs/libcurl/symbols-in-versions

@@ -282,6 +282,7 @@ CURLOPTTYPE_FUNCTIONPOINT       7.1
 CURLOPTTYPE_LONG                7.1
 CURLOPTTYPE_OBJECTPOINT         7.1
 CURLOPTTYPE_OFF_T               7.11.0
+CURLOPT_ACCEPT_ENCODING         7.21.6
 CURLOPT_ADDRESS_SCOPE           7.19.0
 CURLOPT_APPEND                  7.17.0
 CURLOPT_AUTOREFERER             7.1
@@ -485,6 +486,7 @@ CURLOPT_TLSAUTH_PASSWORD        7.21.4
 CURLOPT_TLSAUTH_TYPE            7.21.4
 CURLOPT_TLSAUTH_USERNAME        7.21.4
 CURLOPT_TRANSFERTEXT            7.1.1
+CURLOPT_TRANSFER_ENCODING       7.21.6
 CURLOPT_UNRESTRICTED_AUTH       7.10.4
 CURLOPT_UPLOAD                  7.1
 CURLOPT_URL                     7.1

include/curl/curl.h

@@ -517,7 +517,8 @@ typedef enum {
 #ifndef CURL_NO_OLDIES /* define this to test if your app builds with all
                           the obsolete stuff removed! */
 
-/* Backwards compatibility with older names */
+/*  compatibility with older names */
+#define CURLOPT_ENCODING CURLOPT_ACCEPT_ENCODING
 
 /* The following were added in 7.21.5, April 2011 */
 #define CURLE_UNKNOWN_TELNET_OPTION CURLE_UNKNOWN_OPTION
@@ -1103,8 +1104,9 @@ typedef enum {
   CINIT(PROXYTYPE, LONG, 101),
 
   /* Set the Accept-Encoding string. Use this to tell a server you would like
-     the response to be compressed. */
+     the response to be compressed. Before 7.21.6, this was known as
-  CINIT(ENCODING, OBJECTPOINT, 102),
+     CURLOPT_ENCODING */
+  CINIT(ACCEPT_ENCODING, OBJECTPOINT, 102),
 
   /* Set pointer to private data */
   CINIT(PRIVATE, OBJECTPOINT, 103),
@@ -1462,6 +1464,18 @@ typedef enum {
   /* Set authentication type for authenticated TLS */
   CINIT(TLSAUTH_TYPE, OBJECTPOINT, 206),
 
+  /* Set to 1 to enable the "TE:" header in HTTP requests to ask for
+     compressed transfer-encoded responses. Set to 0 to disable the use of TE:
+     in outgoing requests. The current default is 0, but it might change in a
+     future libcurl release.
+
+     libcurl will ask for the compressed methods it knows of, and if that
+     isn't any, it will not ask for transfer-encoding at all even if this
+     option is set to 1.
+
+  */
+  CINIT(TRANSFER_ENCODING, LONG, 207),
+
   CURLOPT_LASTENTRY /* the last unused */
 } CURLoption;
 

include/curl/typecheck-gcc.h

@@ -7,7 +7,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2010, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2011, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -224,7 +224,7 @@ _CURL_WARNING(_curl_easy_getinfo_err_curl_slist,
    (option) == CURLOPT_PROXYUSERNAME ||                                       \
    (option) == CURLOPT_PROXYPASSWORD ||                                       \
    (option) == CURLOPT_NOPROXY ||                                             \
-   (option) == CURLOPT_ENCODING ||                                            \
+   (option) == CURLOPT_ACCEPT_ENCODING ||                                     \
    (option) == CURLOPT_REFERER ||                                             \
    (option) == CURLOPT_USERAGENT ||                                           \
    (option) == CURLOPT_COOKIE ||                                              \

lib/http.c

@@ -1733,21 +1733,28 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
       return CURLE_OUT_OF_MEMORY;
   }
 
-  if(Curl_checkheaders(data, "TE:")) {
+#ifdef HAVE_LIBZ
-    /* When we insert a TE: header in the request, we must also insert TE in a
+  /* we only consider transfer-encoding magic if libz support is built-in */
-       Connection: header, so we need to merge the custom provided Connection:
+
-       header and prevent the original to get sent */
+  if(!Curl_checkheaders(data, "TE:") && data->set.http_transfer_encoding) {
+    /* When we are to insert a TE: header in the request, we must also insert
+       TE in a Connection: header, so we need to merge the custom provided
+       Connection: header and prevent the original to get sent. Note that if
+       the user has inserted his/hers own TE: header we don't do this magic
+       but then assume that the user will handle it all! */
     char *cptr = Curl_checkheaders(data, "Connection:");
+#define TE_HEADER "TE: gzip\r\n"
 
     Curl_safefree(conn->allocptr.te);
 
     /* Create the (updated) Connection: header */
-    conn->allocptr.te = cptr? aprintf("%s, TE\r\n", cptr):
+    conn->allocptr.te = cptr? aprintf("%s, TE\r\n" TE_HEADER, cptr):
-      strdup("Connection: TE\r\n");
+      strdup("Connection: TE\r\n" TE_HEADER);
 
     if(!conn->allocptr.te)
       return CURLE_OUT_OF_MEMORY;
   }
+#endif
 
   ptr = Curl_checkheaders(data, "Transfer-Encoding:");
   if(ptr) {

lib/url.c

@@ -1076,7 +1076,7 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option,
     data->set.http_auto_referer = (bool)(0 != va_arg(param, long));
     break;
 
-  case CURLOPT_ENCODING:
+  case CURLOPT_ACCEPT_ENCODING:
     /*
      * String to use at the value of Accept-Encoding header.
      *
@@ -1092,6 +1092,10 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option,
                        (char *) ALL_CONTENT_ENCODINGS: argptr);
     break;
 
+  case CURLOPT_TRANSFER_ENCODING:
+    data->set.http_transfer_encoding = (bool)(0 != va_arg(param, long));
+    break;
+
   case CURLOPT_FOLLOWLOCATION:
     /*
      * Follow Location: header hints on a HTTP-server.

lib/urldata.h

@@ -1456,6 +1456,7 @@ struct UserDefined {
   bool hide_progress;    /* don't use the progress meter */
   bool http_fail_on_error;  /* fail on HTTP error codes >= 300 */
   bool http_follow_location; /* follow HTTP redirects */
+  bool http_transfer_encoding; /* request compressed HTTP transfer-encoding */
   bool http_disable_hostname_check_before_authentication;
   bool include_header;   /* include received protocol headers in data output */
   bool http_set_referer; /* is a custom referer used */

src/main.c

@@ -474,6 +474,7 @@ struct Configurable {
   char *cookiefile; /* read from this file */
   bool cookiesession; /* new session? */
   bool encoding;    /* Accept-Encoding please */
+  bool tr_encoding; /* Transfer-Encoding please */
   long authtype;    /* auth bitmask */
   bool use_resume;
   bool resume_from_current;
@@ -904,6 +905,7 @@ static void help(void)
     "    --trace <file>  Write a debug trace to the given file",
     "    --trace-ascii <file> Like --trace but without the hex output",
     "    --trace-time    Add time stamps to trace/verbose output",
+    "    --tr-encoding   Request compressed transfer encoding (H)",
     " -T/--upload-file <file> Transfer <file> to remote site",
     "    --url <URL>     Set URL to work with",
     " -B/--use-ascii     Use ASCII/text transfer",
@@ -1823,7 +1825,8 @@ static ParameterError getparameter(char *flag, /* f or -long-flag */
     {"*g", "trace",      TRUE},
     {"*h", "trace-ascii", TRUE},
     {"*i", "limit-rate", TRUE},
-    {"*j", "compressed",  FALSE}, /* might take an arg someday */
+    {"*j", "compressed",  FALSE},
+    {"*J", "tr-encoding",  FALSE},
     {"*k", "digest",     FALSE},
     {"*l", "negotiate",  FALSE},
     {"*m", "ntlm",       FALSE},
@@ -2148,6 +2151,10 @@ static ParameterError getparameter(char *flag, /* f or -long-flag */
         config->encoding = toggle;
         break;
 
+      case 'J': /* --tr-encoding */
+        config->tr_encoding = toggle;
+        break;
+
       case 'k': /* --digest */
         if(toggle)
           config->authtype |= CURLAUTH_DIGEST;
@@ -2765,13 +2772,13 @@ static ParameterError getparameter(char *flag, /* f or -long-flag */
           GetStr(&config->tls_username, nextarg);
         else
           return PARAM_LIBCURL_DOESNT_SUPPORT;
-	break;
+        break;
       case 'l': /* TLS password */
         if(curlinfo->features & CURL_VERSION_TLSAUTH_SRP)
           GetStr(&config->tls_password, nextarg);
         else
           return PARAM_LIBCURL_DOESNT_SUPPORT;
-	break;
+        break;
       case 'm': /* TLS authentication type */
         if(curlinfo->features & CURL_VERSION_TLSAUTH_SRP) {
           GetStr(&config->tls_authtype, nextarg);
@@ -2780,7 +2787,7 @@ static ParameterError getparameter(char *flag, /* f or -long-flag */
         }
         else
           return PARAM_LIBCURL_DOESNT_SUPPORT;
-	break;
+        break;
       default: /* certificate file */
       {
         char *ptr = strchr(nextarg, ':');
@@ -5381,9 +5388,11 @@ operate(struct Configurable *config, int argc, argv_item_t argv[])
         if(res != CURLE_OK)
           goto show_error;
 
-        /* new in curl 7.10 */
+        if(config->encoding)
-        my_setopt_str(curl, CURLOPT_ENCODING,
+          my_setopt_str(curl, CURLOPT_ACCEPT_ENCODING, "");
-                      (config->encoding) ? "" : NULL);
+
+        if(config->tr_encoding)
+          my_setopt(curl, CURLOPT_TRANSFER_ENCODING, 1);
 
         /* new in curl 7.10.7, extended in 7.19.4 but this only sets 0 or 1 */
         my_setopt(curl, CURLOPT_FTP_CREATE_MISSING_DIRS,

tests/data/test1122

@@ -48,7 +48,7 @@ http
 HTTP GET gzip transfer-encoded content
  </name>
  <command>
-http://%HOSTIP:%HTTPPORT/1122 -H "TE: gzip"
+http://%HOSTIP:%HTTPPORT/1122 --tr-encoding
 </command>
 </client>
 

tests/data/test1123

@@ -178,7 +178,7 @@ http
 HTTP GET deflate transfer-encoded content
  </name>
  <command>
-http://%HOSTIP:%HTTPPORT/1123 -H "TE: deflate, gzip"
+http://%HOSTIP:%HTTPPORT/1123 --tr-encoding
 </command>
 </client>
 
@@ -193,7 +193,7 @@ GET /1123 HTTP/1.1
 Host: %HOSTIP:%HTTPPORT
 Accept: */*
 Connection: TE
-TE: deflate, gzip
+TE: gzip
 
 </protocol>
 </verify>

tests/data/test1124

@@ -47,7 +47,7 @@ http
 HTTP GET gzip+chunked transfer-encoded content
  </name>
  <command>
-http://%HOSTIP:%HTTPPORT/1124 -H "TE: gzip"
+http://%HOSTIP:%HTTPPORT/1124 --tr-encoding
 </command>
 </client>
 

tests/data/test1125

@@ -48,7 +48,7 @@ http
 HTTP GET transfer-encoding with custom Connection:
  </name>
  <command>
-http://%HOSTIP:%HTTPPORT/1125 -H "TE: gzip" -H "Connection: close"
+http://%HOSTIP:%HTTPPORT/1125 --tr-encoding -H "Connection: close"
 </command>
 </client>