From 465e19dbe9603bacba53bd73a607d5fbb37c08a4 Mon Sep 17 00:00:00 2001 From: Daniel Stenberg Date: Tue, 12 Jul 2005 18:15:34 +0000 Subject: [PATCH] Adrian Schuur added trailer support in the chunked encoding stream. The trailer is then sent to the normal header callback/stream. --- CHANGES | 6 ++++ RELEASE-NOTES | 6 ++-- lib/http_chunks.c | 73 +++++++++++++++++++++++++++++++++++++--- lib/http_chunks.h | 20 +++++++++-- lib/transfer.c | 15 +++++++++ lib/url.c | 1 + lib/urldata.h | 10 ++++++ tests/data/Makefile.am | 2 +- tests/data/test266 | 76 ++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 200 insertions(+), 9 deletions(-) create mode 100644 tests/data/test266 diff --git a/CHANGES b/CHANGES index 434266ca6..d4d788f2c 100644 --- a/CHANGES +++ b/CHANGES @@ -7,6 +7,12 @@ Changelog +Daniel (12 July 2005) +- Adrian Schuur added trailer support in the chunked encoding stream. The + trailer is then sent to the normal header callback/stream. I wrote up test + case 266 to verify the basic functionality. Do note that test case 34 + contains a flawed chunked encoding stream that still works the same. + Daniel (5 July 2005) - Gisle Vanem came up with a nice little work-around for bug #1230118. It seems the Windows (MSVC) libc time functions may return data one hour off if diff --git a/RELEASE-NOTES b/RELEASE-NOTES index 45406badd..da40688bf 100644 --- a/RELEASE-NOTES +++ b/RELEASE-NOTES @@ -10,7 +10,8 @@ Curl and libcurl 7.14.1 Number of contributors: 437 This release includes the following changes: - + + o trailer support for chunked encoded data streams o -x/CURL_PROXY strings may now contain user+password o --trace-time now outputs the full microsecond, all 6 digits @@ -41,6 +42,7 @@ This release would not have looked like this without help, code, reports and advice from friends like these: John McGowan, Georg Wicherski, Andres Garcia, Eric Cooper, Todd Kulesza, - Tupone Alfredo, Gisle Vanem, David Shaw, Andrew Bushnell, Dan Fandrich + Tupone Alfredo, Gisle Vanem, David Shaw, Andrew Bushnell, Dan Fandrich, + Adrian Schuur Thanks! (and sorry if I forgot to mention someone) diff --git a/lib/http_chunks.c b/lib/http_chunks.c index f333f9559..63e136cb7 100644 --- a/lib/http_chunks.c +++ b/lib/http_chunks.c @@ -153,10 +153,17 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn, if(*datap == '\n') { /* we're now expecting data to come, unless size was zero! */ if(0 == ch->datasize) { - ch->state = CHUNK_STOP; /* stop reading! */ - if(1 == length) { - /* This was the final byte, return right now */ - return CHUNKE_STOP; + if (conn->bits.trailerHdrPresent!=TRUE) { + /* No Trailer: header found - revert to original Curl processing */ + ch->state = CHUNK_STOP; + if (1 == length) { + /* This is the final byte, return right now */ + return CHUNKE_STOP; + } + } + else { + ch->state = CHUNK_TRAILER; /* attempt to read trailers */ + conn->trlPos=0; } } else @@ -250,6 +257,64 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn, return CHUNKE_BAD_CHUNK; break; + case CHUNK_TRAILER: + /* conn->trailer is assumed to be freed in url.c on a + connection basis */ + if (conn->trlPos >= conn->trlMax) { + char *ptr; + if(conn->trlMax) { + conn->trlMax *= 2; + ptr = (char*)realloc(conn->trailer,conn->trlMax); + } + else { + conn->trlMax=128; + ptr = (char*)malloc(conn->trlMax); + } + if(!ptr) + return CHUNKE_OUT_OF_MEMORY; + conn->trailer = ptr; + } + conn->trailer[conn->trlPos++]=*datap; + + if(*datap == '\r') + ch->state = CHUNK_TRAILER_CR; + else { + datap++; + length--; + } + break; + + case CHUNK_TRAILER_CR: + if(*datap == '\r') { + ch->state = CHUNK_TRAILER_POSTCR; + datap++; + length--; + } + else + return CHUNKE_BAD_CHUNK; + break; + + case CHUNK_TRAILER_POSTCR: + if (*datap == '\n') { + conn->trailer[conn->trlPos++]='\n'; + conn->trailer[conn->trlPos]=0; + if (conn->trlPos==2) { + ch->state = CHUNK_STOP; + return CHUNKE_STOP; + } + else { + Curl_client_write(conn->data, CLIENTWRITE_HEADER, + conn->trailer, conn->trlPos); + } + ch->state = CHUNK_TRAILER; + conn->trlPos=0; + datap++; + length--; + } + else + return CHUNKE_BAD_CHUNK; + break; + case CHUNK_STOP: /* If we arrive here, there is data left in the end of the buffer even if there's no more chunks to read */ diff --git a/lib/http_chunks.h b/lib/http_chunks.h index c5b7fdf43..211818ab7 100644 --- a/lib/http_chunks.h +++ b/lib/http_chunks.h @@ -52,8 +52,8 @@ typedef enum { /* POSTCR should get a CR and nothing else, then move to POSTLF */ CHUNK_POSTCR, - /* POSTLF should get a LF and nothing else, then move back to HEX as - the CRLF combination marks the end of a chunk */ + /* POSTLF should get a LF and nothing else, then move back to HEX as the + CRLF combination marks the end of a chunk */ CHUNK_POSTLF, /* This is mainly used to really mark that we're out of the game. @@ -62,7 +62,22 @@ typedef enum { buffer! */ CHUNK_STOP, + /* At this point optional trailer headers can be found, unless the next line + is CRLF */ + CHUNK_TRAILER, + + /* A trailer CR has been found - next state is CHUNK_TRAILER_POSTCR. + Next char must be a LF */ + CHUNK_TRAILER_CR, + + /* A trailer LF must be found now, otherwise CHUNKE_BAD_CHUNK will be + signalled If this is an empty trailer CHUNKE_STOP will be signalled. + Otherwise the trailer will be broadcasted via Curl_client_write() and the + next state will be CHUNK_TRAILER */ + CHUNK_TRAILER_POSTCR, + CHUNK_LAST /* never use */ + } ChunkyState; typedef enum { @@ -74,6 +89,7 @@ typedef enum { CHUNKE_WRITE_ERROR, CHUNKE_STATE_ERROR, CHUNKE_BAD_ENCODING, + CHUNKE_OUT_OF_MEMORY, CHUNKE_LAST } CHUNKcode; diff --git a/lib/transfer.c b/lib/transfer.c index fdc825bd2..a2cd39179 100644 --- a/lib/transfer.c +++ b/lib/transfer.c @@ -833,6 +833,20 @@ CURLcode Curl_readwrite(struct connectdata *conn, /* init our chunky engine */ Curl_httpchunk_init(conn); } + + else if (checkprefix("Trailer:", k->p) || + checkprefix("Trailers:", k->p)) { + /* + * This test helps Curl_httpchunk_read() to determine to look + * for well formed trailers after the zero chunksize record. In + * this case a CRLF is required after the zero chunksize record + * when no trailers are sent, or after the last trailer record. + * + * It seems both Trailer: and Trailers: occur in the wild. + */ + conn->bits.trailerHdrPresent = TRUE; + } + else if (checkprefix("Content-Encoding:", k->p) && data->set.encoding) { /* @@ -1074,6 +1088,7 @@ CURLcode Curl_readwrite(struct connectdata *conn, * the name says read, this function both reads and writes away * the data. The returned 'nread' holds the number of actual * data it wrote to the client. */ + CHUNKcode res = Curl_httpchunk_read(conn, k->str, nread, &nread); diff --git a/lib/url.c b/lib/url.c index 469e6b33c..9c05c803a 100644 --- a/lib/url.c +++ b/lib/url.c @@ -1496,6 +1496,7 @@ CURLcode Curl_disconnect(struct connectdata *conn) Curl_safefree(conn->allocptr.host); Curl_safefree(conn->allocptr.cookiehost); Curl_safefree(conn->ip_addr_str); + Curl_safefree(conn->trailer); /* possible left-overs from the async name resolvers */ #if defined(USE_ARES) diff --git a/lib/urldata.h b/lib/urldata.h index 9bd245980..4200c3818 100644 --- a/lib/urldata.h +++ b/lib/urldata.h @@ -421,6 +421,10 @@ struct ConnectBits { LPRT doesn't work we disable it for the forthcoming requests */ bool netrc; /* name+password provided by netrc */ + + bool trailerHdrPresent; /* Set when Trailer: header found in HTTP response. + Required to determine whether to look for trailers + in case of Transfer-Encoding: chunking */ }; struct hostname { @@ -726,6 +730,12 @@ struct connectdata { transfer */ enum { NORMAL, SOURCE3RD, TARGET3RD } xfertype; + + /* These three are used for chunked-encoding trailer support */ + char *trailer; /* allocated buffer to store trailer in */ + int trlMax; /* allocated buffer size */ + int trlPos; /* index of where to store data */ + }; /* The end of connectdata. */ diff --git a/tests/data/Makefile.am b/tests/data/Makefile.am index b32e9ee62..a153ae9dd 100644 --- a/tests/data/Makefile.am +++ b/tests/data/Makefile.am @@ -33,4 +33,4 @@ EXTRA_DIST = test1 test108 test117 test127 test20 test27 test34 test46 \ test237 test238 test239 test243 test245 test246 test247 test248 test249 \ test250 test251 test252 test253 test254 test255 test521 test522 test523 \ test256 test257 test258 test259 test260 test261 test262 test263 test264 \ - test265 + test265 test266 diff --git a/tests/data/test266 b/tests/data/test266 new file mode 100644 index 000000000..f3c613293 --- /dev/null +++ b/tests/data/test266 @@ -0,0 +1,76 @@ + + +HTTP +HTTP GET +chunked Transfer-Encoding + + +# +# Server-side + + +HTTP/1.1 200 funky chunky! +Server: fakeit/0.9 fakeitbad/1.0 +Transfer-Encoding: chunked +Trailer: chunky-trailer +Connection: mooo + +40 +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +30 +bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +21;heresatest=moooo +cccccccccccccccccccccccccccccccc + +0 +chunky-trailer: header data + + + +HTTP/1.1 200 funky chunky! +Server: fakeit/0.9 fakeitbad/1.0 +Transfer-Encoding: chunked +Trailer: chunky-trailer +Connection: mooo + +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbcccccccccccccccccccccccccccccccc + + + +# +# Client-side + + +http + + +HTTP GET with chunked Transfer-Encoding and chunked trailer + + +http://%HOSTIP:%HTTPPORT/266 -D log/heads266 + + + +# +# Verify data after the test has been "shot" + + +^User-Agent:.* + + +GET /266 HTTP/1.1 +Host: 127.0.0.1:%HTTPPORT +Accept: */* + + + +HTTP/1.1 200 funky chunky! +Server: fakeit/0.9 fakeitbad/1.0 +Transfer-Encoding: chunked +Trailer: chunky-trailer +Connection: mooo + +chunky-trailer: header data + + +