Toby Peterson added CURLOPT_IGNORE_CONTENT_LENGTH to the library, accessible

from the command line tool with --ignore-content-length. This will make it
easier to download files from Apache 1.x (and similar) servers that are
still having problems serving files larger than 2 or 4 GB. When this option
is enabled, curl will simply have to wait for the server to close the
connection to signal end of transfer. I wrote test case 269 that runs a
simple test that this works.

CHANGES

@@ -8,6 +8,14 @@
 
 
 Daniel (24 August 2005)
+- Toby Peterson added CURLOPT_IGNORE_CONTENT_LENGTH to the library, accessible
+  from the command line tool with --ignore-content-length. This will make it
+  easier to download files from Apache 1.x (and similar) servers that are
+  still having problems serving files larger than 2 or 4 GB. When this option
+  is enabled, curl will simply have to wait for the server to close the
+  connection to signal end of transfer. I wrote test case 269 that runs a
+  simple test that this works.
+
 - (Trying hard to exclude emotions now.) valgrind version 3 suddenly renamed
   the --logfile command line option to --log-file, and thus the test script
   valgrind autodetection now has yet another version check to do and then it

RELEASE-NOTES

@@ -2,8 +2,8 @@ Curl and libcurl 7.14.1
 
  Public curl release number:               89
  Releases counted from the very beginning: 116
- Available command line options:           107
- Available curl_easy_setopt() options:     122
+ Available command line options:           108
+ Available curl_easy_setopt() options:     123
  Number of public functions in libcurl:    46
  Amount of public web site mirrors:        25
  Number of known libcurl bindings:         31
@@ -11,6 +11,7 @@ Curl and libcurl 7.14.1
 
 This release includes the following changes:
 
+ o --ignore-content-length and CURLOPT_IGNORE_CONTENT_LENGTH added
  o negotiates data connection SSL earlier when doing FTPS with PASV
  o CURLOPT_COOKIELIST and CURLINFO_COOKIELIST
  o trailer support for chunked encoded data streams 
@@ -64,6 +65,6 @@ advice from friends like these:
  Tupone Alfredo, Gisle Vanem, David Shaw, Andrew Bushnell, Dan Fandrich,
  Adrian Schuur, Diego Casorran, Peteris Krumins, Jon Grubbs, Christopher
  R. Palmer, Mario Schroeder, Richard Clayton, James Bursa, Jeff Pohlmeyer,
- Norbert Novotny
+ Norbert Novotny, Toby Peterson
 
         Thanks! (and sorry if I forgot to mention someone)

docs/curl.1

@@ -429,6 +429,11 @@ for you.
 See also the \fI-A/--user-agent\fP and \fI-e/--referer\fP options.
 
 This option can be used multiple times to add/replace/remove multiple headers.
+.IP "--ignore-content-length"
+(HTTP)
+Ignore the Content-Length header. This is particularly useful for servers
+running Apache 1.x, which will report incorrect Content-Length for files
+larger than 2 gigabytes.
 .IP "-i/--include"
 (HTTP)
 Include the HTTP-header in the output. The HTTP-header includes things

docs/libcurl/curl_easy_setopt.3

@@ -687,6 +687,11 @@ it thinks fit.
 Enforce HTTP 1.0 requests.
 .IP CURL_HTTP_VERSION_1_1
 Enforce HTTP 1.1 requests.
+.IP CURLOPT_IGNORE_CONTENT_LENGTH
+Ignore the Content-Length header. This is useful for Apache 1.x which will
+report incorrect content length for files over 2 gigabytes. If this option
+is used, curl will not be able to accurately report progress, and will
+simply stop the download when the server ends the connection.
 .RE
 .SH FTP OPTIONS
 .IP CURLOPT_FTPPORT

include/curl/curl.h

@@ -893,6 +893,9 @@ typedef enum {
   /* feed cookies into cookie engine */
   CINIT(COOKIELIST, OBJECTPOINT, 135),
 
+  /* ignore Content-Length */
+  CINIT(IGNORE_CONTENT_LENGTH, LONG, 136),
+
   CURLOPT_LASTENTRY /* the last unused */
 } CURLoption;
 

lib/transfer.c

@@ -718,7 +718,7 @@ CURLcode Curl_readwrite(struct connectdata *conn,
                the header completely if we get a 416 response as then we're
                resuming a document that we don't get, and this header contains
                info about the true size of the document we didn't get now. */
-            if (!k->ignorecl &&
+            if (!k->ignorecl && !data->set.ignorecl &&
                 checkprefix("Content-Length:", k->p)) {
               contentlength = curlx_strtoofft(k->p+15, NULL, 10);
               if (data->set.max_filesize &&

lib/url.c

@@ -1443,6 +1443,10 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option,
     data->set.ftp_account = va_arg(param, char *);
     break;
 
+  case CURLOPT_IGNORE_CONTENT_LENGTH:
+    data->set.ignorecl = va_arg(param, long)?TRUE:FALSE;
+    break;
+
   default:
     /* unknown tag and its companion, just ignore: */
     result = CURLE_FAILED_INIT; /* correct this */

lib/urldata.h

@@ -1068,6 +1068,7 @@ struct UserDefined {
   bool global_dns_cache; /* subject for future removal */
   bool tcp_nodelay;      /* whether to enable TCP_NODELAY or not */
 
+  bool ignorecl;         /* ignore content length */
 };
 
 /*

src/main.c

@@ -356,6 +356,8 @@ struct Configurable {
   struct curl_slist *tp_postquote;
   struct curl_slist *tp_prequote;
   char *ftp_account; /* for ACCT */
+
+  bool ignorecl; /* --ignore-content-length */
 };
 
 #define WARN_PREFIX "Warning: "
@@ -514,6 +516,7 @@ static void help(void)
     " -G/--get           Send the -d data with a HTTP GET (H)",
     " -h/--help          This help text",
     " -H/--header <line> Custom header to pass to server (H)",
+    "    --ignore-content-length  Ignore the HTTP Content-Length header",
     " -i/--include       Include protocol headers in the output (H/F)",
     " -I/--head          Show document info only",
     " -j/--junk-session-cookies Ignore session cookies read from file (H)",
@@ -1309,6 +1312,7 @@ static ParameterError getparameter(char *flag, /* f or -long-flag */
     {"$m", "ftp-account", TRUE},
     {"$n", "proxy-anyauth", FALSE},
     {"$o", "trace-time", FALSE},
+    {"$p", "ignore-content-length", FALSE},
 
     {"0", "http1.0",     FALSE},
     {"1", "tlsv1",       FALSE},
@@ -1702,6 +1706,9 @@ static ParameterError getparameter(char *flag, /* f or -long-flag */
       case 'o': /* --trace-time */
         config->tracetime ^= TRUE;
         break;
+      case 'p': /* --ignore-content-length */
+        config->ignorecl ^= TRUE;
+        break;
       }
       break;
     case '#': /* --progress-bar */
@@ -3896,6 +3903,8 @@ operate(struct Configurable *config, int argc, char *argv[])
         curl_easy_setopt(curl, CURLOPT_SOURCE_QUOTE, config->tp_quote);
         curl_easy_setopt(curl, CURLOPT_FTP_ACCOUNT, config->ftp_account);
 
+        curl_easy_setopt(curl, CURLOPT_IGNORE_CONTENT_LENGTH, config->ignorecl);
+
         retry_numretries = config->req_retry;
 
         retrystart = curlx_tvnow();

tests/data/Makefile.am

@@ -33,4 +33,4 @@ EXTRA_DIST = test1 test108 test117 test127 test20 test27 test34 test46	   \
  test237 test238 test239 test243 test245 test246 test247 test248 test249   \
  test250 test251 test252 test253 test254 test255 test521 test522 test523   \
  test256 test257 test258 test259 test260 test261 test262 test263 test264   \
- test265 test266 test267 test268
+ test265 test266 test267 test268 test269

tests/data/test269

@@ -0,0 +1,51 @@
+<info>
+<keywords>
+HTTP
+HTTP GET
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data>
+HTTP/1.1 200 OK swsclose
+Date: Thu, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Accept-Ranges: bytes
+Content-Length: 677654
+Connection: close
+Content-Type: text/html
+Funny-head: yesyes
+
+muahahaha
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+ <name>
+HTTP --ignore-content-length
+ </name>
+ <command>
+http://%HOSTIP:%HTTPPORT/269 --ignore-content-length
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<strip>
+^User-Agent:.*
+</strip>
+<protocol>
+GET /269 HTTP/1.1
+Host: 127.0.0.1:%HTTPPORT
+Accept: */*
+
+</protocol>
+</verify>