Support HTTP/1.1.

src/ChangeLog

@@ -1,3 +1,17 @@
+2010-05-01  Giuseppe Scrivano <gscrivano@gnu.org>
+
+	* http.c (request_send): Specify 1.1 as HTTP version for requests.
+	(skip_short_body): Accept new parameter `chunked'.  New variable
+	`remaining_chunk_size'.  Handle the chunked transfer encoding.
+	(gethttp): New variable `chunked_transfer_encoding`.  Set `keepalive'
+	by default to true.  Check if the server is using the chunked transfer
+	encoding.
+
+	* retr.h: Define `rb_chunked_transfer_encoding'.
+
+	* retr.c (fd_read_body): New variable `chunked'.  New variable
+	`remaining_chunk_size'.  Handle the chunked transfer encoding.
+
 2010-03-04  Steven Schubiger  <stsc@member.fsf.org>
 
 	* ftp.c (ftp_loop_internal): Omit input file from being

src/http.c

@@ -352,7 +352,7 @@ request_send (const struct request *req, int fd)
 
   APPEND (p, req->method); *p++ = ' ';
   APPEND (p, req->arg);    *p++ = ' ';
-  memcpy (p, "HTTP/1.0\r\n", 10); p += 10;
+  memcpy (p, "HTTP/1.1\r\n", 10); p += 10;
 
   for (i = 0; i < req->hcount; i++)
     {
@@ -901,29 +901,54 @@ parse_content_range (const char *hdr, wgint *first_byte_ptr,
    mode, the body is displayed for debugging purposes.  */
 
 static bool
-skip_short_body (int fd, wgint contlen)
+skip_short_body (int fd, wgint contlen, bool chunked)
 {
   enum {
     SKIP_SIZE = 512,                /* size of the download buffer */
     SKIP_THRESHOLD = 4096        /* the largest size we read */
   };
+  wgint remaining_chunk_size = 0;
   char dlbuf[SKIP_SIZE + 1];
   dlbuf[SKIP_SIZE] = '\0';        /* so DEBUGP can safely print it */
 
-  /* We shouldn't get here with unknown contlen.  (This will change
+  assert (contlen != -1 || contlen);
-     with HTTP/1.1, which supports "chunked" transfer.)  */
-  assert (contlen != -1);
 
   /* If the body is too large, it makes more sense to simply close the
      connection than to try to read the body.  */
   if (contlen > SKIP_THRESHOLD)
     return false;
 
+  while (contlen > 0 || chunked)
+    {
+      int ret;
+      if (chunked)
+        {
+          if (remaining_chunk_size == 0)
+            {
+              char *line = fd_read_line (fd);
+              char *endl;
+              if (line == NULL)
+                {
+                  ret = -1;
+                  break;
+                }
+
+              remaining_chunk_size = strtol (line, &endl, 16);
+              if (remaining_chunk_size == 0)
+                {
+                  ret = 0;
+                  if (fd_read_line (fd) == NULL)
+                    ret = -1;
+                  break;
+                }
+            }
+
+          contlen = MIN (remaining_chunk_size, SKIP_SIZE);
+        }
+
       DEBUGP (("Skipping %s bytes of body: [", number_to_static_string (contlen)));
 
-  while (contlen > 0)
+      ret = fd_read (fd, dlbuf, MIN (contlen, SKIP_SIZE), -1);
-    {
-      int ret = fd_read (fd, dlbuf, MIN (contlen, SKIP_SIZE), -1);
       if (ret <= 0)
         {
           /* Don't normally report the error since this is an
@@ -933,6 +958,15 @@ skip_short_body (int fd, wgint contlen)
           return false;
         }
       contlen -= ret;
+
+      if (chunked)
+        {
+          remaining_chunk_size -= ret;
+          if (remaining_chunk_size == 0)
+            if (fd_read_line (fd) == NULL)
+              return false;
+        }
+
       /* Safe even if %.*s bogusly expects terminating \0 because
          we've zero-terminated dlbuf above.  */
       DEBUGP (("%.*s", ret, dlbuf));
@@ -1537,6 +1571,9 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
      is done. */
   bool keep_alive;
 
+  /* Is the server using the chunked transfer encoding?  */
+  bool chunked_transfer_encoding = false;
+
   /* Whether keep-alive should be inhibited.
 
      RFC 2068 requests that 1.0 clients not send keep-alive requests
@@ -1739,11 +1776,13 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
         request_set_header (req, "Proxy-Authorization", proxyauth, rel_value);
     }
 
-  keep_alive = false;
+  keep_alive = true;
 
   /* Establish the connection.  */
 
-  if (!inhibit_keep_alive)
+  if (inhibit_keep_alive)
+    keep_alive = false;
+  else
     {
       /* Look for a persistent connection to target host, unless a
          proxy is used.  The exception is when SSL is in use, in which
@@ -1975,15 +2014,17 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
   /* Check for keep-alive related responses. */
   if (!inhibit_keep_alive && contlen != -1)
     {
-      if (resp_header_copy (resp, "Keep-Alive", NULL, 0))
+      if (resp_header_copy (resp, "Connection", hdrval, sizeof (hdrval)))
-        keep_alive = true;
-      else if (resp_header_copy (resp, "Connection", hdrval, sizeof (hdrval)))
         {
-          if (0 == strcasecmp (hdrval, "Keep-Alive"))
+          if (0 == strcasecmp (hdrval, "Close"))
-            keep_alive = true;
+            keep_alive = false;
         }
     }
 
+  resp_header_copy (resp, "Transfer-Encoding", hdrval, sizeof (hdrval));
+  if (0 == strcasecmp (hdrval, "chunked"))
+    chunked_transfer_encoding = true;
+
   /* Handle (possibly multiple instances of) the Set-Cookie header. */
   if (opt.cookies)
     {
@@ -2010,7 +2051,8 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
   if (statcode == HTTP_STATUS_UNAUTHORIZED)
     {
       /* Authorization is required.  */
-      if (keep_alive && !head_only && skip_short_body (sock, contlen))
+      if (keep_alive && !head_only
+          && skip_short_body (sock, contlen, chunked_transfer_encoding))
         CLOSE_FINISH (sock);
       else
         CLOSE_INVALIDATE (sock);
@@ -2262,7 +2304,8 @@ File %s already there; not retrieving.\n\n"), quote (hs->local_file));
                      _("Location: %s%s\n"),
                      hs->newloc ? escnonprint_uri (hs->newloc) : _("unspecified"),
                      hs->newloc ? _(" [following]") : "");
-          if (keep_alive && !head_only && skip_short_body (sock, contlen))
+          if (keep_alive && !head_only
+              && skip_short_body (sock, contlen, chunked_transfer_encoding))
             CLOSE_FINISH (sock);
           else
             CLOSE_INVALIDATE (sock);
@@ -2392,7 +2435,8 @@ File %s already there; not retrieving.\n\n"), quote (hs->local_file));
            If not, they can be worked around using
            `--no-http-keep-alive'.  */
         CLOSE_FINISH (sock);
-      else if (keep_alive && skip_short_body (sock, contlen))
+      else if (keep_alive
+               && skip_short_body (sock, contlen, chunked_transfer_encoding))
         /* Successfully skipped the body; also keep using the socket. */
         CLOSE_FINISH (sock);
       else
@@ -2493,6 +2537,10 @@ File %s already there; not retrieving.\n\n"), quote (hs->local_file));
     /* If the server ignored our range request, instruct fd_read_body
        to skip the first RESTVAL bytes of body.  */
     flags |= rb_skip_startpos;
+
+  if (chunked_transfer_encoding)
+    flags |= rb_chunked_transfer_encoding;
+
   hs->len = hs->restval;
   hs->rd_size = 0;
   hs->res = fd_read_body (sock, fp, contlen != -1 ? contlen : 0,

src/retr.c

@@ -225,11 +225,15 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
   bool progress_interactive = false;
 
   bool exact = !!(flags & rb_read_exactly);
+
+  /* Used only by HTTP/HTTPS chunked transfer encoding.  */
+  bool chunked = flags & rb_chunked_transfer_encoding;
   wgint skip = 0;
 
   /* How much data we've read/written.  */
   wgint sum_read = 0;
   wgint sum_written = 0;
+  wgint remaining_chunk_size = 0;
 
   if (flags & rb_skip_startpos)
     skip = startpos;
@@ -269,8 +273,36 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
      should be read.  */
   while (!exact || (sum_read < toread))
     {
-      int rdsize = exact ? MIN (toread - sum_read, dlbufsize) : dlbufsize;
+      int rdsize;
       double tmout = opt.read_timeout;
+
+      if (chunked)
+        {
+          if (remaining_chunk_size == 0)
+            {
+              char *line = fd_read_line (fd);
+              char *endl;
+              if (line == NULL)
+                {
+                  ret = -1;
+                  break;
+                }
+
+              remaining_chunk_size = strtol (line, &endl, 16);
+              if (remaining_chunk_size == 0)
+                {
+                  ret = 0;
+                  if (fd_read_line (fd) == NULL)
+                    ret = -1;
+                  break;
+                }
+            }
+
+          rdsize = MIN (remaining_chunk_size, dlbufsize);
+        }
+      else
+        rdsize = exact ? MIN (toread - sum_read, dlbufsize) : dlbufsize;
+
       if (progress_interactive)
         {
           /* For interactive progress gauges, always specify a ~1s
@@ -316,6 +348,16 @@ fd_read_body (int fd, FILE *out, wgint toread, wgint startpos,
               ret = -2;
               goto out;
             }
+          if (chunked)
+            {
+              remaining_chunk_size -= ret;
+              if (remaining_chunk_size == 0)
+                if (fd_read_line (fd) == NULL)
+                  {
+                    ret = -1;
+                    break;
+                  }
+            }
         }
 
       if (opt.limit_rate)

src/retr.h

@@ -43,7 +43,10 @@ extern bool output_stream_regular;
 /* Flags for fd_read_body. */
 enum {
   rb_read_exactly  = 1,
-  rb_skip_startpos = 2
+  rb_skip_startpos = 2,
+
+  /* Used by HTTP/HTTPS*/
+  rb_chunked_transfer_encoding = 4
 };
 
 int fd_read_body (int, FILE *, wgint, wgint, wgint *, wgint *, double *, int);