- Shmulik Regev fixed so that the final CRLF of HTTP response headers are sent

to the debug callback.

- Shmulik Regev added CURLOPT_HTTP_CONTENT_DECODING and
  CURLOPT_HTTP_TRANSFER_DECODING that if set to zero will disable libcurl's
  internal decoding of content or transfer encoded content. This may be
  preferable in cases where you use libcurl for proxy purposes or similar. The
  command line tool got a --raw option to disable both at once.

CHANGES

@@ -7,6 +7,15 @@
                                   Changelog
 
 Daniel (12 February 2007)
+- Shmulik Regev fixed so that the final CRLF of HTTP response headers are sent
+  to the debug callback.
+
+- Shmulik Regev added CURLOPT_HTTP_CONTENT_DECODING and
+  CURLOPT_HTTP_TRANSFER_DECODING that if set to zero will disable libcurl's
+  internal decoding of content or transfer encoded content. This may be
+  preferable in cases where you use libcurl for proxy purposes or similar. The
+  command line tool got a --raw option to disable both at once.
+  
 - Jeff Pohlmeyer fixed a flaw in curl_multi_add_handle() when adding a handle
   that has an easy handle present in the "closure" list pending closure.
 

RELEASE-NOTES

@@ -2,8 +2,8 @@ Curl and libcurl 7.16.2
 
  Public curl release number:               98
  Releases counted from the very beginning: 125
- Available command line options:           115
- Available curl_easy_setopt() options:     137
+ Available command line options:           116
+ Available curl_easy_setopt() options:     141
  Number of public functions in libcurl:    54
  Amount of public web site mirrors:        39
  Number of known libcurl bindings:         35
@@ -12,6 +12,8 @@ Curl and libcurl 7.16.2
 This release includes the following changes:
  
  o Added CURLOPT_TIMEOUT_MS and CURLOPT_CONNECTTIMEOUT_MS
+ o Added CURLOPT_HTTP_CONTENT_DECODING, CURLOPT_HTTP_TRANSFER_DECODING and
+   --raw
 
 This release includes the following bugfixes:
 
@@ -34,6 +36,6 @@ New curl mirrors:
 This release would not have looked like this without help, code, reports and
 advice from friends like these:
 
- Yang Tse, Manfred Schwarb, Michael Wallner, Jeff Pohlmeyer
+ Yang Tse, Manfred Schwarb, Michael Wallner, Jeff Pohlmeyer, Shmulik Regev
 
         Thanks! (and sorry if I forgot to mention someone)

docs/curl.1

@@ -21,7 +21,7 @@
 .\" * $Id$
 .\" **************************************************************************
 .\"
-.TH curl 1 "3 Nov 2006" "Curl 7.16.1" "Curl Manual"
+.TH curl 1 "12 Feb 2007" "Curl 7.16.2" "Curl Manual"
 .SH NAME
 curl \- transfer a URL
 .SH SYNOPSIS
@@ -901,6 +901,11 @@ FTP range downloads only support the simple syntax 'start-stop' (optionally
 with one of the numbers omitted). It depends on the non-RFC command SIZE.
 
 If this option is used several times, the last one will be used.
+.IP "--raw"
+When used, it disables all internal HTTP decoding of content or transfer
+encodings and instead makes them passed on unaltered, raw. (Added in 7.16.2)
+
+If this option is used several times, each occurrence toggles this on/off.
 .IP "-R/--remote-time"
 When used, this will make libcurl attempt to figure out the timestamp of the
 remote file, and if that is available make the local file get that same

docs/libcurl/curl_easy_setopt.3

@@ -21,7 +21,7 @@
 .\" * $Id$
 .\" **************************************************************************
 .\"
-.TH curl_easy_setopt 3 "5 Feb 2007" "libcurl 7.16.2" "libcurl Manual"
+.TH curl_easy_setopt 3 "12 Feb 2007" "libcurl 7.16.2" "libcurl Manual"
 .SH NAME
 curl_easy_setopt \- set options for a curl easy handle
 .SH SYNOPSIS
@@ -815,6 +815,16 @@ servers) which will report incorrect content length for files over 2
 gigabytes. If this option is used, curl will not be able to accurately report
 progress, and will simply stop the download when the server ends the
 connection. (added in 7.14.1)
+.IP CURLOPT_HTTP_CONTENT_DECODING
+Pass a long to tell libcurl how to act on content decoding. If set to zero,
+content decoding will be disabled. If set to 1 it is enabled. Note however
+that libcurl has no default content decoding but requires you to use
+\fICURLOPT_ENCODING\fP for that. (added in 7.16.2)
+.IP CURLOPT_HTTP_TRANSFER_DECODING
+Pass a long to tell libcurl how to act on transfer decoding. If set to zero,
+transfer decoding will be disabled, if set to 1 it is enabled
+(default). libcurl does chunked transfer decoding by default unless this
+option is set to zero. (added in 7.16.2)
 .RE
 .SH FTP OPTIONS
 .IP CURLOPT_FTPPORT

include/curl/curl.h

@@ -1058,6 +1058,11 @@ typedef enum {
   CINIT(TIMEOUT_MS, LONG, 155),
   CINIT(CONNECTTIMEOUT_MS, LONG, 156),
 
+  /* set to zero to disable the libcurl's decoding and thus pass the raw body
+     data to the appliction even when it is encoded/compressed */
+  CINIT(HTTP_TRANSFER_DECODING, LONG, 157),
+  CINIT(HTTP_CONTENT_DECODING, LONG, 158),
+
   CURLOPT_LASTENTRY /* the last unused */
 } CURLoption;
 

lib/http_chunks.c

@@ -116,6 +116,12 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn,
 
   *wrote = 0; /* nothing's written yet */
 
+  /* the original data is written to the client, but we go on with the
+     chunk read process, to properly calculate the content length*/
+  if ( data->set.http_te_skip )
+    Curl_client_write(conn, CLIENTWRITE_BODY, datap,datalen);
+
+
   while(length) {
     switch(ch->state) {
     case CHUNK_HEX:
@@ -206,12 +212,17 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn,
 
       /* Write the data portion available */
 #ifdef HAVE_LIBZ
-      switch (data->reqdata.keep.content_encoding) {
+      switch (conn->data->set.http_ce_skip?
+              IDENTITY : data->reqdata.keep.content_encoding) {
         case IDENTITY:
 #endif
-          if(!k->ignorebody)
-            result = Curl_client_write(conn, CLIENTWRITE_BODY, datap,
-                                       piece);
+          if(!k->ignorebody) {
+            if ( !data->set.http_te_skip )
+              result = Curl_client_write(conn, CLIENTWRITE_BODY, datap,
+                                         piece);
+            else
+              result = CURLE_OK;
+          }
 #ifdef HAVE_LIBZ
           break;
 
@@ -334,6 +345,7 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn,
             return(CHUNKE_BAD_CHUNK);
           }
 #endif /* CURL_DOES_CONVERSIONS */
+          if ( !data->set.http_te_skip )
           Curl_client_write(conn, CLIENTWRITE_HEADER,
                             conn->trailer, conn->trlPos);
         }

lib/transfer.c

@@ -689,6 +689,9 @@ CURLcode Curl_readwrite(struct connectdata *conn,
                   k->keepon &= ~KEEP_READ;
                 }
 
+                if(data->set.verbose)
+                  Curl_debug(data, CURLINFO_HEADER_IN,
+                             k->str_start, headerlen, conn);
                 break;          /* exit header line loop */
               }
 
@@ -1286,7 +1289,8 @@ CURLcode Curl_readwrite(struct connectdata *conn,
                  Make sure that ALL_CONTENT_ENCODINGS contains all the
                  encodings handled here. */
 #ifdef HAVE_LIBZ
-              switch (k->content_encoding) {
+              switch (conn->data->set.http_ce_skip ?
+                      IDENTITY : k->content_encoding) {
               case IDENTITY:
 #endif
                 /* This is the default when the server sends no

lib/url.c

@@ -1725,6 +1725,19 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option,
     data->set.ssh_private_key = va_arg(param, char *);
     break;
 
+  case CURLOPT_HTTP_TRANSFER_DECODING:
+    /*
+     * disable libcurl transfer encoding is used
+     */
+    data->set.http_te_skip = (bool)(0 == va_arg(param, long));
+    break;
+
+  case CURLOPT_HTTP_CONTENT_DECODING:
+    /*
+     * raw data passed to the application when content encoding is used
+     */
+    data->set.http_ce_skip = (bool)(0 == va_arg(param, long));
+    break;
   default:
     /* unknown tag and its companion, just ignore: */
     result = CURLE_FAILED_INIT; /* correct this */

lib/urldata.h

@@ -1292,6 +1292,10 @@ struct UserDefined {
                              authentication */
   char *ssh_private_key;  /* the path to the private key file for
                              authentication */
+  bool http_te_skip;     /* pass the raw body data to the user, even when
+                            transfer-encoded (chunked, compressed) */
+  bool http_ce_skip;     /* pass the raw body data to the user, even when
+                            content-encoded (chunked, compressed) */
 };
 
 struct Names {

src/main.c

@@ -20,7 +20,6 @@
  *
  * $Id$
  ***************************************************************************/
-
 #include "setup.h"
 
 #include <stdio.h>
@@ -470,7 +469,7 @@ struct Configurable {
   bool disable_sessionid;
 
   char *libcurl; /* output libcurl code to this file name */
-
+  bool raw;
   struct OutStruct *outs;
 };
 
@@ -683,6 +682,7 @@ static void help(void)
     " -Q/--quote <cmd>   Send command(s) to server before file transfer (F)",
     " -r/--range <range> Retrieve a byte range from a HTTP/1.1 or FTP server",
     "    --random-file <file> File for reading random data from (SSL)",
+    "    --raw           Pass HTTP \"raw\", without any transfer decoding (H)",
     " -R/--remote-time   Set the remote file's time on the local output",
     "    --retry <num>   Retry request <num> times if transient problems occur",
     "    --retry-delay <seconds> When retrying, wait this many seconds between each",
@@ -1473,6 +1473,7 @@ static ParameterError getparameter(char *flag, /* f or -long-flag */
     {"$x", "ftp-ssl-control", FALSE},
     {"$y", "ftp-ssl-ccc", FALSE},
     {"$z", "libcurl",    TRUE},
+    {"$#", "raw",        FALSE},
 
     {"0", "http1.0",     FALSE},
     {"1", "tlsv1",       FALSE},
@@ -1903,6 +1904,9 @@ static ParameterError getparameter(char *flag, /* f or -long-flag */
       case 'z': /* --libcurl */
         GetStr(&config->libcurl, nextarg);
         break;
+      case '#': /* --raw */
+        config->raw ^= TRUE;
+        break;
       }
       break;
     case '#': /* --progress-bar */
@@ -4253,6 +4257,12 @@ operate(struct Configurable *config, int argc, char *argv[])
         my_setopt(curl, CURLOPT_SSL_SESSIONID_CACHE,
                   !config->disable_sessionid);
 
+        /* curl 7.16.2 */
+        if(config->raw) {
+          my_setopt(curl, CURLOPT_HTTP_CONTENT_DECODING, FALSE);
+          my_setopt(curl, CURLOPT_HTTP_TRANSFER_DECODING, FALSE);
+        }
+
         retry_numretries = config->req_retry;
 
         retrystart = curlx_tvnow();