"Transfer-Encoding: chunked" support added

lib/http.c

@@ -104,6 +104,7 @@
 #include "memdebug.h"
 #endif
 
+/* ------------------------------------------------------------------------- */
 /*
  * The add_buffer series of functions are used to build one large memory chunk
  * from repeated function invokes. Used so that the entire HTTP request can
@@ -205,7 +206,7 @@ CURLcode add_buffer(send_buffer *in, void *inptr, size_t size)
 }
 
 /* end of the add_buffer functions */
-/*****************************************************************************/
+/* ------------------------------------------------------------------------- */
 
 /*
  * Read everything until a newline.
@@ -309,6 +310,9 @@ CURLcode Curl_ConnectHTTPProxyTunnel(struct connectdata *conn,
   return CURLE_OK;
 }
 
+/*
+ * HTTP stuff to do at connect-time.
+ */
 CURLcode Curl_http_connect(struct connectdata *conn)
 {
   struct UrlData *data;

lib/http.h

@@ -35,4 +35,9 @@ CURLcode Curl_http_done(struct connectdata *conn);
 CURLcode Curl_http_connect(struct connectdata *conn);
 CURLcode Curl_http_close(struct connectdata *conn);
 
+/* The following functions are defined in http_chunks.c */
+void Curl_httpchunk_init(struct connectdata *conn);
+CHUNKcode Curl_httpchunk_read(struct connectdata *conn, char *datap,
+                              ssize_t length, ssize_t *wrote);
+
 #endif

lib/http_chunks.c

@@ -82,15 +82,15 @@ void Curl_httpchunk_init(struct connectdata *conn)
 {
   struct Curl_chunker *chunk = &conn->proto.http->chunk;
   chunk->hexindex=0; /* start at 0 */
+  chunk->dataleft=0; /* no data left yet! */
   chunk->state = CHUNK_HEX; /* we get hex first! */
 }
 
 /*
- * chunk_read() returns a 0 for normal operations, or a positive return code
- * for errors. A negative number means this sequence of chunks is complete,
- * and that many ~bytes were NOT used at the end of the buffer passed in.
- * The 'wrote' argument is set to tell the caller how many bytes we actually
- * passed to the client (for byte-counting and whatever).
+ * chunk_read() returns a OK for normal operations, or a positive return code
+ * for errors. STOP means this sequence of chunks is complete.  The 'wrote'
+ * argument is set to tell the caller how many bytes we actually passed to the
+ * client (for byte-counting and whatever).
  *
  * The states and the state-machine is further explained in the header file.
  */
@@ -142,7 +142,7 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn,
           ch->state = CHUNK_STOP; /* stop reading! */
           if(1 == length) {
             /* This was the final byte, return right now */
-            return ~0;
+            return CHUNKE_STOP;
           }
         }
         else
@@ -179,7 +179,10 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn,
 
       break;
     case CHUNK_STOP:
-      return ~length; /* return the data size left */
+      /* If we arrive here, there is data left in the end of the buffer
+         even if there's no more chunks to read */
+      ch->dataleft = length;
+      return CHUNKE_STOP; /* return stop */
     default:
       return CHUNKE_STATE_ERROR;
     }

lib/http_chunks.h

@@ -49,15 +49,19 @@ typedef enum {
      HEX state. */
   CHUNK_DATA,
 
-  /* This is only used to really mark that we're out of the game */
+  /* This is mainly used to really mark that we're out of the game.
+     NOTE: that there's a 'dataleft' field in the struct that will tell how
+     many bytes that were not passed to the client in the end of the last
+     buffer! */
   CHUNK_STOP,
 
   CHUNK_LAST /* never use */
 } ChunkyState;
 
 typedef enum {
-  CHUNKE_OK,
-  CHUNKE_TOO_LONG_HEX,
+  CHUNKE_STOP = -1,
+  CHUNKE_OK = 0,
+  CHUNKE_TOO_LONG_HEX = 1,
   CHUNKE_WRITE_ERROR,
   CHUNKE_STATE_ERROR,
   CHUNKE_LAST
@@ -67,7 +71,8 @@ struct Curl_chunker {
   char hexbuffer[ MAXNUM_SIZE + 1];
   int hexindex;
   ChunkyState state;
-  unsigned long datasize;
+  size_t datasize;
+  size_t dataleft; /* untouched data amount at the end of the last buffer */
 };
 
 #endif

lib/transfer.c

@@ -89,6 +89,7 @@
 #include "getpass.h"
 #include "progress.h"
 #include "getdate.h"
+#include "http.h"
 
 #define _MPRINTF_REPLACE /* use our functions only */
 #include <curl/mprintf.h>
@@ -390,6 +391,9 @@ _Transfer(struct connectdata *c_conn)
                  * of chunks, and a chunk-data set to zero signals the
                  * end-of-chunks. */
                 conn->bits.chunk = TRUE; /* chunks coming our way */
+
+                /* init our chunky engine */
+                Curl_httpchunk_init(conn);
               }
               else if (strnequal("Content-Range", p, 13)) {
                 if (sscanf (p+13, ": bytes %d-", &offset) ||
@@ -536,8 +540,24 @@ _Transfer(struct connectdata *c_conn)
             if(conn->bits.chunk) {
               /*
                * Bless me father for I have sinned. Here come a chunked
-               * transfer flighing and we need to decode this properly.
-               */
+               * transfer flying and we need to decode this properly.  While
+               * the name says read, this function both reads and writes away
+               * the data. The returned 'nread' holds the number of actual
+               * data it wrote to the client.  */
+              CHUNKcode res =
+                Curl_httpchunk_read(conn, str, nread, &nread);
+
+              if(CHUNKE_OK < res)
+                return CURLE_READ_ERROR;
+              else if(CHUNKE_STOP == res) {
+                /* we're done reading chunks! */
+                keepon &= ~KEEP_READ; /* read no more */
+
+                /* There are now (~res) bytes at the end of the str buffer
+                   that weren't written to the client, but we don't care
+                   about them right now. */
+              }
+              /* If it returned OK, we just keep going */
             }
 
             if(conn->maxdownload &&
@@ -552,9 +572,12 @@ _Transfer(struct connectdata *c_conn)
 
             Curl_pgrsSetDownloadCounter(data, (double)bytecount);
             
-            urg = Curl_client_write(data, CLIENTWRITE_BODY, str, nread);
-            if(urg)
-              return urg;
+            if(! conn->bits.chunk) {
+              /* If this is chunky transfer, it was already written */
+              urg = Curl_client_write(data, CLIENTWRITE_BODY, str, nread);
+              if(urg)
+                return urg;
+            }
 
           } /* if (! header and data to read ) */
         } /* if( read from socket ) */

lib/urldata.h

@@ -79,6 +79,8 @@
 
 #include <curl/curl.h>
 
+#include "http_chunks.h" /* for the structs and enum stuff */
+
 /* Download buffer size, keep it fairly big for speed reasons */
 #define BUFSIZE (1024*50)
 
@@ -167,6 +169,8 @@ struct HTTP {
   struct Form form;
   size_t (*storefread)(char *, size_t , size_t , FILE *);
   FILE *in;
+
+  struct Curl_chunker chunk;
 };
 
 /****************************************************************************