From 8e4fb01e64bee1893452f25873758cb856898d84 Mon Sep 17 00:00:00 2001 From: Daniel Stenberg Date: Thu, 14 Apr 2011 22:45:42 +0200 Subject: [PATCH] transfer-encoding: added new option and cmdline Added CURLOPT_TRANSFER_ENCODING as the option to set to request Transfer Encoding in HTTP requests (if built zlib enabled). I also renamed CURLOPT_ENCODING to CURLOPT_ACCEPT_ENCODING (while keeping the old name around) to reduce the confusion when we have to encoding options for HTTP. --tr-encoding is now the new command line option for curl to request this, and thus I updated the test cases accordingly. --- docs/libcurl/symbols-in-versions | 2 ++ include/curl/curl.h | 20 +++++++++++++++++--- include/curl/typecheck-gcc.h | 4 ++-- lib/http.c | 19 +++++++++++++------ lib/url.c | 6 +++++- lib/urldata.h | 1 + src/main.c | 23 ++++++++++++++++------- tests/data/test1122 | 2 +- tests/data/test1123 | 4 ++-- tests/data/test1124 | 2 +- tests/data/test1125 | 2 +- 11 files changed, 61 insertions(+), 24 deletions(-) diff --git a/docs/libcurl/symbols-in-versions b/docs/libcurl/symbols-in-versions index aa86b913a..7c2c866be 100644 --- a/docs/libcurl/symbols-in-versions +++ b/docs/libcurl/symbols-in-versions @@ -282,6 +282,7 @@ CURLOPTTYPE_FUNCTIONPOINT 7.1 CURLOPTTYPE_LONG 7.1 CURLOPTTYPE_OBJECTPOINT 7.1 CURLOPTTYPE_OFF_T 7.11.0 +CURLOPT_ACCEPT_ENCODING 7.21.6 CURLOPT_ADDRESS_SCOPE 7.19.0 CURLOPT_APPEND 7.17.0 CURLOPT_AUTOREFERER 7.1 @@ -485,6 +486,7 @@ CURLOPT_TLSAUTH_PASSWORD 7.21.4 CURLOPT_TLSAUTH_TYPE 7.21.4 CURLOPT_TLSAUTH_USERNAME 7.21.4 CURLOPT_TRANSFERTEXT 7.1.1 +CURLOPT_TRANSFER_ENCODING 7.21.6 CURLOPT_UNRESTRICTED_AUTH 7.10.4 CURLOPT_UPLOAD 7.1 CURLOPT_URL 7.1 diff --git a/include/curl/curl.h b/include/curl/curl.h index b44649588..6996257c2 100644 --- a/include/curl/curl.h +++ b/include/curl/curl.h @@ -517,7 +517,8 @@ typedef enum { #ifndef CURL_NO_OLDIES /* define this to test if your app builds with all the obsolete stuff removed! */ -/* Backwards compatibility with older names */ +/* compatibility with older names */ +#define CURLOPT_ENCODING CURLOPT_ACCEPT_ENCODING /* The following were added in 7.21.5, April 2011 */ #define CURLE_UNKNOWN_TELNET_OPTION CURLE_UNKNOWN_OPTION @@ -1103,8 +1104,9 @@ typedef enum { CINIT(PROXYTYPE, LONG, 101), /* Set the Accept-Encoding string. Use this to tell a server you would like - the response to be compressed. */ - CINIT(ENCODING, OBJECTPOINT, 102), + the response to be compressed. Before 7.21.6, this was known as + CURLOPT_ENCODING */ + CINIT(ACCEPT_ENCODING, OBJECTPOINT, 102), /* Set pointer to private data */ CINIT(PRIVATE, OBJECTPOINT, 103), @@ -1462,6 +1464,18 @@ typedef enum { /* Set authentication type for authenticated TLS */ CINIT(TLSAUTH_TYPE, OBJECTPOINT, 206), + /* Set to 1 to enable the "TE:" header in HTTP requests to ask for + compressed transfer-encoded responses. Set to 0 to disable the use of TE: + in outgoing requests. The current default is 0, but it might change in a + future libcurl release. + + libcurl will ask for the compressed methods it knows of, and if that + isn't any, it will not ask for transfer-encoding at all even if this + option is set to 1. + + */ + CINIT(TRANSFER_ENCODING, LONG, 207), + CURLOPT_LASTENTRY /* the last unused */ } CURLoption; diff --git a/include/curl/typecheck-gcc.h b/include/curl/typecheck-gcc.h index e6f74a958..46f92d728 100644 --- a/include/curl/typecheck-gcc.h +++ b/include/curl/typecheck-gcc.h @@ -7,7 +7,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2010, Daniel Stenberg, , et al. + * Copyright (C) 1998 - 2011, Daniel Stenberg, , et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -224,7 +224,7 @@ _CURL_WARNING(_curl_easy_getinfo_err_curl_slist, (option) == CURLOPT_PROXYUSERNAME || \ (option) == CURLOPT_PROXYPASSWORD || \ (option) == CURLOPT_NOPROXY || \ - (option) == CURLOPT_ENCODING || \ + (option) == CURLOPT_ACCEPT_ENCODING || \ (option) == CURLOPT_REFERER || \ (option) == CURLOPT_USERAGENT || \ (option) == CURLOPT_COOKIE || \ diff --git a/lib/http.c b/lib/http.c index 8d3a085d0..4f2b46a59 100644 --- a/lib/http.c +++ b/lib/http.c @@ -1733,21 +1733,28 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) return CURLE_OUT_OF_MEMORY; } - if(Curl_checkheaders(data, "TE:")) { - /* When we insert a TE: header in the request, we must also insert TE in a - Connection: header, so we need to merge the custom provided Connection: - header and prevent the original to get sent */ +#ifdef HAVE_LIBZ + /* we only consider transfer-encoding magic if libz support is built-in */ + + if(!Curl_checkheaders(data, "TE:") && data->set.http_transfer_encoding) { + /* When we are to insert a TE: header in the request, we must also insert + TE in a Connection: header, so we need to merge the custom provided + Connection: header and prevent the original to get sent. Note that if + the user has inserted his/hers own TE: header we don't do this magic + but then assume that the user will handle it all! */ char *cptr = Curl_checkheaders(data, "Connection:"); +#define TE_HEADER "TE: gzip\r\n" Curl_safefree(conn->allocptr.te); /* Create the (updated) Connection: header */ - conn->allocptr.te = cptr? aprintf("%s, TE\r\n", cptr): - strdup("Connection: TE\r\n"); + conn->allocptr.te = cptr? aprintf("%s, TE\r\n" TE_HEADER, cptr): + strdup("Connection: TE\r\n" TE_HEADER); if(!conn->allocptr.te) return CURLE_OUT_OF_MEMORY; } +#endif ptr = Curl_checkheaders(data, "Transfer-Encoding:"); if(ptr) { diff --git a/lib/url.c b/lib/url.c index a31c28033..019f9a24b 100644 --- a/lib/url.c +++ b/lib/url.c @@ -1076,7 +1076,7 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option, data->set.http_auto_referer = (bool)(0 != va_arg(param, long)); break; - case CURLOPT_ENCODING: + case CURLOPT_ACCEPT_ENCODING: /* * String to use at the value of Accept-Encoding header. * @@ -1092,6 +1092,10 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option, (char *) ALL_CONTENT_ENCODINGS: argptr); break; + case CURLOPT_TRANSFER_ENCODING: + data->set.http_transfer_encoding = (bool)(0 != va_arg(param, long)); + break; + case CURLOPT_FOLLOWLOCATION: /* * Follow Location: header hints on a HTTP-server. diff --git a/lib/urldata.h b/lib/urldata.h index 33933b3e1..96814cfb5 100644 --- a/lib/urldata.h +++ b/lib/urldata.h @@ -1456,6 +1456,7 @@ struct UserDefined { bool hide_progress; /* don't use the progress meter */ bool http_fail_on_error; /* fail on HTTP error codes >= 300 */ bool http_follow_location; /* follow HTTP redirects */ + bool http_transfer_encoding; /* request compressed HTTP transfer-encoding */ bool http_disable_hostname_check_before_authentication; bool include_header; /* include received protocol headers in data output */ bool http_set_referer; /* is a custom referer used */ diff --git a/src/main.c b/src/main.c index 307070f8b..a9a1e848b 100644 --- a/src/main.c +++ b/src/main.c @@ -474,6 +474,7 @@ struct Configurable { char *cookiefile; /* read from this file */ bool cookiesession; /* new session? */ bool encoding; /* Accept-Encoding please */ + bool tr_encoding; /* Transfer-Encoding please */ long authtype; /* auth bitmask */ bool use_resume; bool resume_from_current; @@ -904,6 +905,7 @@ static void help(void) " --trace Write a debug trace to the given file", " --trace-ascii Like --trace but without the hex output", " --trace-time Add time stamps to trace/verbose output", + " --tr-encoding Request compressed transfer encoding (H)", " -T/--upload-file Transfer to remote site", " --url Set URL to work with", " -B/--use-ascii Use ASCII/text transfer", @@ -1823,7 +1825,8 @@ static ParameterError getparameter(char *flag, /* f or -long-flag */ {"*g", "trace", TRUE}, {"*h", "trace-ascii", TRUE}, {"*i", "limit-rate", TRUE}, - {"*j", "compressed", FALSE}, /* might take an arg someday */ + {"*j", "compressed", FALSE}, + {"*J", "tr-encoding", FALSE}, {"*k", "digest", FALSE}, {"*l", "negotiate", FALSE}, {"*m", "ntlm", FALSE}, @@ -2148,6 +2151,10 @@ static ParameterError getparameter(char *flag, /* f or -long-flag */ config->encoding = toggle; break; + case 'J': /* --tr-encoding */ + config->tr_encoding = toggle; + break; + case 'k': /* --digest */ if(toggle) config->authtype |= CURLAUTH_DIGEST; @@ -2765,13 +2772,13 @@ static ParameterError getparameter(char *flag, /* f or -long-flag */ GetStr(&config->tls_username, nextarg); else return PARAM_LIBCURL_DOESNT_SUPPORT; - break; + break; case 'l': /* TLS password */ if(curlinfo->features & CURL_VERSION_TLSAUTH_SRP) GetStr(&config->tls_password, nextarg); else return PARAM_LIBCURL_DOESNT_SUPPORT; - break; + break; case 'm': /* TLS authentication type */ if(curlinfo->features & CURL_VERSION_TLSAUTH_SRP) { GetStr(&config->tls_authtype, nextarg); @@ -2780,7 +2787,7 @@ static ParameterError getparameter(char *flag, /* f or -long-flag */ } else return PARAM_LIBCURL_DOESNT_SUPPORT; - break; + break; default: /* certificate file */ { char *ptr = strchr(nextarg, ':'); @@ -5381,9 +5388,11 @@ operate(struct Configurable *config, int argc, argv_item_t argv[]) if(res != CURLE_OK) goto show_error; - /* new in curl 7.10 */ - my_setopt_str(curl, CURLOPT_ENCODING, - (config->encoding) ? "" : NULL); + if(config->encoding) + my_setopt_str(curl, CURLOPT_ACCEPT_ENCODING, ""); + + if(config->tr_encoding) + my_setopt(curl, CURLOPT_TRANSFER_ENCODING, 1); /* new in curl 7.10.7, extended in 7.19.4 but this only sets 0 or 1 */ my_setopt(curl, CURLOPT_FTP_CREATE_MISSING_DIRS, diff --git a/tests/data/test1122 b/tests/data/test1122 index a246c62e3..78d50d77b 100644 --- a/tests/data/test1122 +++ b/tests/data/test1122 @@ -48,7 +48,7 @@ http HTTP GET gzip transfer-encoded content -http://%HOSTIP:%HTTPPORT/1122 -H "TE: gzip" +http://%HOSTIP:%HTTPPORT/1122 --tr-encoding diff --git a/tests/data/test1123 b/tests/data/test1123 index cfd843dae..3172fedb1 100644 --- a/tests/data/test1123 +++ b/tests/data/test1123 @@ -178,7 +178,7 @@ http HTTP GET deflate transfer-encoded content -http://%HOSTIP:%HTTPPORT/1123 -H "TE: deflate, gzip" +http://%HOSTIP:%HTTPPORT/1123 --tr-encoding @@ -193,7 +193,7 @@ GET /1123 HTTP/1.1 Host: %HOSTIP:%HTTPPORT Accept: */* Connection: TE -TE: deflate, gzip +TE: gzip diff --git a/tests/data/test1124 b/tests/data/test1124 index d8b4f9ae7..d860bee13 100644 --- a/tests/data/test1124 +++ b/tests/data/test1124 @@ -47,7 +47,7 @@ http HTTP GET gzip+chunked transfer-encoded content -http://%HOSTIP:%HTTPPORT/1124 -H "TE: gzip" +http://%HOSTIP:%HTTPPORT/1124 --tr-encoding diff --git a/tests/data/test1125 b/tests/data/test1125 index e4d116cac..1883734d9 100644 --- a/tests/data/test1125 +++ b/tests/data/test1125 @@ -48,7 +48,7 @@ http HTTP GET transfer-encoding with custom Connection: -http://%HOSTIP:%HTTPPORT/1125 -H "TE: gzip" -H "Connection: close" +http://%HOSTIP:%HTTPPORT/1125 --tr-encoding -H "Connection: close"