Adrian Schuur added trailer support in the chunked encoding stream. The

trailer is then sent to the normal header callback/stream.

CHANGES

@@ -7,6 +7,12 @@
                                   Changelog
 
 
+Daniel (12 July 2005)
+- Adrian Schuur added trailer support in the chunked encoding stream. The
+  trailer is then sent to the normal header callback/stream. I wrote up test
+  case 266 to verify the basic functionality. Do note that test case 34
+  contains a flawed chunked encoding stream that still works the same.
+
 Daniel (5 July 2005)
 - Gisle Vanem came up with a nice little work-around for bug #1230118. It
   seems the Windows (MSVC) libc time functions may return data one hour off if

RELEASE-NOTES

@@ -10,7 +10,8 @@ Curl and libcurl 7.14.1
  Number of contributors:                   437
 
 This release includes the following changes:
- 
+
+ o trailer support for chunked encoded data streams 
  o -x/CURL_PROXY strings may now contain user+password
  o --trace-time now outputs the full microsecond, all 6 digits
 
@@ -41,6 +42,7 @@ This release would not have looked like this without help, code, reports and
 advice from friends like these:
 
  John McGowan, Georg Wicherski, Andres Garcia, Eric Cooper, Todd Kulesza,
- Tupone Alfredo, Gisle Vanem, David Shaw, Andrew Bushnell, Dan Fandrich
+ Tupone Alfredo, Gisle Vanem, David Shaw, Andrew Bushnell, Dan Fandrich,
+ Adrian Schuur
 
         Thanks! (and sorry if I forgot to mention someone)

lib/http_chunks.c

@@ -153,10 +153,17 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn,
       if(*datap == '\n') {
         /* we're now expecting data to come, unless size was zero! */
         if(0 == ch->datasize) {
-          ch->state = CHUNK_STOP; /* stop reading! */
+          if (conn->bits.trailerHdrPresent!=TRUE) {
-          if(1 == length) {
+            /* No Trailer: header found - revert to original Curl processing */
-            /* This was the final byte, return right now */
+            ch->state = CHUNK_STOP;
-            return CHUNKE_STOP;
+            if (1 == length) {
+               /* This is the final byte, return right now */
+               return CHUNKE_STOP;
+            }
+          }
+          else {
+            ch->state = CHUNK_TRAILER; /* attempt to read trailers */
+            conn->trlPos=0;
           }
         }
         else
@@ -250,6 +257,64 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn,
         return CHUNKE_BAD_CHUNK;
       break;
 
+    case CHUNK_TRAILER:
+      /* conn->trailer is assumed to be freed in url.c on a
+         connection basis */
+      if (conn->trlPos >= conn->trlMax) {
+        char *ptr;
+        if(conn->trlMax) {
+          conn->trlMax *= 2;
+          ptr = (char*)realloc(conn->trailer,conn->trlMax);
+        }
+        else {
+          conn->trlMax=128;
+          ptr = (char*)malloc(conn->trlMax);
+        }
+        if(!ptr)
+          return CHUNKE_OUT_OF_MEMORY;
+        conn->trailer = ptr;
+      }
+      conn->trailer[conn->trlPos++]=*datap;
+
+      if(*datap == '\r')
+        ch->state = CHUNK_TRAILER_CR;
+      else {
+        datap++;
+        length--;
+     }
+      break;
+
+    case CHUNK_TRAILER_CR:
+      if(*datap == '\r') {
+        ch->state = CHUNK_TRAILER_POSTCR;
+        datap++;
+        length--;
+      }
+      else
+        return CHUNKE_BAD_CHUNK;
+      break;
+
+    case CHUNK_TRAILER_POSTCR:
+      if (*datap == '\n') {
+        conn->trailer[conn->trlPos++]='\n';
+        conn->trailer[conn->trlPos]=0;
+        if (conn->trlPos==2) {
+          ch->state = CHUNK_STOP;
+          return CHUNKE_STOP;
+        }
+        else {
+          Curl_client_write(conn->data, CLIENTWRITE_HEADER,
+                            conn->trailer, conn->trlPos);
+        }
+        ch->state = CHUNK_TRAILER;
+        conn->trlPos=0;
+        datap++;
+        length--;
+      }
+      else
+        return CHUNKE_BAD_CHUNK;
+      break;
+
     case CHUNK_STOP:
       /* If we arrive here, there is data left in the end of the buffer
          even if there's no more chunks to read */

lib/http_chunks.h

@@ -52,8 +52,8 @@ typedef enum {
   /* POSTCR should get a CR and nothing else, then move to POSTLF */
   CHUNK_POSTCR,
 
-  /* POSTLF should get a LF and nothing else, then move back to HEX as
+  /* POSTLF should get a LF and nothing else, then move back to HEX as the
-     the CRLF combination marks the end of a chunk */
+     CRLF combination marks the end of a chunk */
   CHUNK_POSTLF,
 
   /* This is mainly used to really mark that we're out of the game.
@@ -62,7 +62,22 @@ typedef enum {
      buffer! */
   CHUNK_STOP,
 
+  /* At this point optional trailer headers can be found, unless the next line
+     is CRLF */
+  CHUNK_TRAILER,
+
+  /* A trailer CR has been found - next state is CHUNK_TRAILER_POSTCR.
+     Next char must be a LF */
+  CHUNK_TRAILER_CR,
+
+  /* A trailer LF must be found now, otherwise CHUNKE_BAD_CHUNK will be
+     signalled If this is an empty trailer CHUNKE_STOP will be signalled.
+     Otherwise the trailer will be broadcasted via Curl_client_write() and the
+     next state will be CHUNK_TRAILER */
+  CHUNK_TRAILER_POSTCR,
+
   CHUNK_LAST /* never use */
+
 } ChunkyState;
 
 typedef enum {
@@ -74,6 +89,7 @@ typedef enum {
   CHUNKE_WRITE_ERROR,
   CHUNKE_STATE_ERROR,
   CHUNKE_BAD_ENCODING,
+  CHUNKE_OUT_OF_MEMORY,
   CHUNKE_LAST
 } CHUNKcode;
 

lib/transfer.c

@@ -833,6 +833,20 @@ CURLcode Curl_readwrite(struct connectdata *conn,
               /* init our chunky engine */
               Curl_httpchunk_init(conn);
             }
+
+            else if (checkprefix("Trailer:", k->p) ||
+                     checkprefix("Trailers:", k->p)) {
+              /*
+               * This test helps Curl_httpchunk_read() to determine to look
+               * for well formed trailers after the zero chunksize record. In
+               * this case a CRLF is required after the zero chunksize record
+               * when no trailers are sent, or after the last trailer record.
+               *
+               * It seems both Trailer: and Trailers: occur in the wild.
+               */
+              conn->bits.trailerHdrPresent = TRUE;
+            }
+
             else if (checkprefix("Content-Encoding:", k->p) &&
                      data->set.encoding) {
               /*
@@ -1074,6 +1088,7 @@ CURLcode Curl_readwrite(struct connectdata *conn,
              * the name says read, this function both reads and writes away
              * the data. The returned 'nread' holds the number of actual
              * data it wrote to the client.  */
+
             CHUNKcode res =
               Curl_httpchunk_read(conn, k->str, nread, &nread);
 

lib/url.c

@@ -1496,6 +1496,7 @@ CURLcode Curl_disconnect(struct connectdata *conn)
   Curl_safefree(conn->allocptr.host);
   Curl_safefree(conn->allocptr.cookiehost);
   Curl_safefree(conn->ip_addr_str);
+  Curl_safefree(conn->trailer);
 
   /* possible left-overs from the async name resolvers */
 #if defined(USE_ARES)

lib/urldata.h

@@ -421,6 +421,10 @@ struct ConnectBits {
                          LPRT doesn't work we disable it for the forthcoming
                          requests */
   bool netrc;         /* name+password provided by netrc */
+  
+  bool trailerHdrPresent; /* Set when Trailer: header found in HTTP response.
+                             Required to determine whether to look for trailers 
+                             in case of Transfer-Encoding: chunking */ 
 };
 
 struct hostname {
@@ -726,6 +730,12 @@ struct connectdata {
                                      transfer */
 
   enum { NORMAL, SOURCE3RD, TARGET3RD } xfertype;
+
+  /* These three are used for chunked-encoding trailer support */
+  char *trailer; /* allocated buffer to store trailer in */
+  int trlMax;    /* allocated buffer size */
+  int trlPos;    /* index of where to store data */
+
 };
 
 /* The end of connectdata. */

tests/data/Makefile.am

@@ -33,4 +33,4 @@ EXTRA_DIST = test1 test108 test117 test127 test20 test27 test34 test46	   \
  test237 test238 test239 test243 test245 test246 test247 test248 test249   \
  test250 test251 test252 test253 test254 test255 test521 test522 test523   \
  test256 test257 test258 test259 test260 test261 test262 test263 test264   \
- test265
+ test265 test266

tests/data/test266

@@ -0,0 +1,76 @@
+<info>
+<keywords>
+HTTP
+HTTP GET
+chunked Transfer-Encoding
+</keywords>
+</info>
+#
+# Server-side
+<reply>
+<data>
+HTTP/1.1 200 funky chunky!
+Server: fakeit/0.9 fakeitbad/1.0
+Transfer-Encoding: chunked
+Trailer: chunky-trailer
+Connection: mooo
+
+40
+aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+30
+bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
+21;heresatest=moooo
+cccccccccccccccccccccccccccccccc
+
+0
+chunky-trailer: header data
+
+</data>
+<datacheck>
+HTTP/1.1 200 funky chunky!
+Server: fakeit/0.9 fakeitbad/1.0
+Transfer-Encoding: chunked
+Trailer: chunky-trailer
+Connection: mooo
+
+aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbcccccccccccccccccccccccccccccccc
+</datacheck>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+ <name>
+HTTP GET with chunked Transfer-Encoding and chunked trailer
+ </name>
+ <command>
+http://%HOSTIP:%HTTPPORT/266 -D log/heads266
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<strip>
+^User-Agent:.*
+</strip>
+<protocol>
+GET /266 HTTP/1.1
+Host: 127.0.0.1:%HTTPPORT
+Accept: */*
+
+</protocol>
+<file name="log/heads266">
+HTTP/1.1 200 funky chunky!
+Server: fakeit/0.9 fakeitbad/1.0
+Transfer-Encoding: chunked
+Trailer: chunky-trailer
+Connection: mooo
+
+chunky-trailer: header data
+</file>
+</verify>
+