diff --git a/lib/content_encoding.c b/lib/content_encoding.c index c2ba26972..01a155246 100644 --- a/lib/content_encoding.c +++ b/lib/content_encoding.c @@ -73,14 +73,17 @@ typedef enum { ZLIB_UNINIT, /* uninitialized */ ZLIB_INIT, /* initialized */ + ZLIB_INFLATING, /* Inflating started. */ ZLIB_GZIP_HEADER, /* reading gzip header */ + ZLIB_GZIP_TRAILER, /* reading gzip trailer */ ZLIB_GZIP_INFLATING, /* inflating gzip stream */ ZLIB_INIT_GZIP /* initialized in transparent gzip mode */ } zlibInitState; /* Writer parameters. */ typedef struct { - zlibInitState zlib_init; /* zlib init state */ + zlibInitState zlib_init; /* zlib init state */ + uInt trailerlen; /* Remaining trailer byte count. */ z_stream z; /* State structure for zlib. */ } zlib_params; @@ -130,80 +133,120 @@ exit_zlib(struct connectdata *conn, return result; } -static CURLcode -inflate_stream(struct connectdata *conn, contenc_writer *writer) +static CURLcode process_trailer(struct connectdata *conn, zlib_params *zp) +{ + z_stream *z = &zp->z; + CURLcode result = CURLE_OK; + uInt len = z->avail_in < zp->trailerlen? z->avail_in: zp->trailerlen; + + /* Consume expected trailer bytes. Terminate stream if exhausted. + Issue an error if unexpected bytes follow. */ + + zp->trailerlen -= len; + z->avail_in -= len; + z->next_in += len; + if(z->avail_in) + result = CURLE_WRITE_ERROR; + if(result || !zp->trailerlen) + result = exit_zlib(conn, z, &zp->zlib_init, result); + else { + /* Only occurs for gzip with zlib < 1.2.0.4. */ + zp->zlib_init = ZLIB_GZIP_TRAILER; + } + return result; +} + +static CURLcode inflate_stream(struct connectdata *conn, + contenc_writer *writer, zlibInitState started) { zlib_params *zp = (zlib_params *) &writer->params; - int allow_restart = 1; z_stream *z = &zp->z; /* zlib state structure */ uInt nread = z->avail_in; Bytef *orig_in = z->next_in; int status; /* zlib status */ + bool done = FALSE; CURLcode result = CURLE_OK; /* Curl_client_write status */ char *decomp; /* Put the decompressed data here. */ + /* Check state. */ + if(zp->zlib_init != ZLIB_INIT && + zp->zlib_init != ZLIB_INFLATING && + zp->zlib_init != ZLIB_INIT_GZIP && + zp->zlib_init != ZLIB_GZIP_INFLATING) + return exit_zlib(conn, z, &zp->zlib_init, CURLE_WRITE_ERROR); + /* Dynamically allocate a buffer for decompression because it's uncommonly large to hold on the stack */ decomp = malloc(DSIZ); - if(decomp == NULL) { + if(decomp == NULL) return exit_zlib(conn, z, &zp->zlib_init, CURLE_OUT_OF_MEMORY); - } /* because the buffer size is fixed, iteratively decompress and transfer to - the client via client_write. */ - for(;;) { - if(z->avail_in == 0) { - free(decomp); - return result; - } + the client via downstream_write function. */ + while(!done) { + done = TRUE; /* (re)set buffer for decompressed output for every iteration */ z->next_out = (Bytef *) decomp; z->avail_out = DSIZ; - status = inflate(z, Z_SYNC_FLUSH); - if(status == Z_OK || status == Z_STREAM_END) { - allow_restart = 0; - result = Curl_unencode_write(conn, writer->downstream, decomp, - DSIZ - z->avail_out); - /* if !CURLE_OK, clean up, return */ - if(result) { - free(decomp); - return exit_zlib(conn, z, &zp->zlib_init, result); + status = inflate(z, Z_BLOCK); + + /* Flush output data if some. */ + if(z->avail_out != DSIZ) { + if(status == Z_OK || status == Z_STREAM_END) { + zp->zlib_init = started; /* Data started. */ + result = Curl_unencode_write(conn, writer->downstream, decomp, + DSIZ - z->avail_out); + if(result) { + exit_zlib(conn, z, &zp->zlib_init, result); + break; + } } - - /* Done? clean up, return */ - if(status == Z_STREAM_END) { - free(decomp); - return exit_zlib(conn, z, &zp->zlib_init, result); - } - - /* Done with these bytes, exit */ - - /* status is always Z_OK at this point! */ - continue; } - else if(allow_restart && status == Z_DATA_ERROR) { + + /* Dispatch by inflate() status. */ + switch(status) { + case Z_OK: + /* Always loop: there may be unflushed latched data in zlib state. */ + done = FALSE; + break; + case Z_BUF_ERROR: + /* No more data to flush: just exit loop. */ + break; + case Z_STREAM_END: + result = process_trailer(conn, zp); + break; + case Z_DATA_ERROR: /* some servers seem to not generate zlib headers, so this is an attempt to fix and continue anyway */ - - (void) inflateEnd(z); /* don't care about the return code */ - if(inflateInit2(z, -MAX_WBITS) != Z_OK) { - free(decomp); - zp->zlib_init = ZLIB_UNINIT; /* inflateEnd() already called. */ - return exit_zlib(conn, z, &zp->zlib_init, process_zlib_error(conn, z)); + if(zp->zlib_init == ZLIB_INIT) { + /* Do not use inflateReset2(): only available since zlib 1.2.3.4. */ + (void) inflateEnd(z); /* don't care about the return code */ + if(inflateInit2(z, -MAX_WBITS) == Z_OK) { + z->next_in = orig_in; + z->avail_in = nread; + zp->zlib_init = ZLIB_INFLATING; + done = FALSE; + break; + } + zp->zlib_init = ZLIB_UNINIT; /* inflateEnd() already called. */ } - z->next_in = orig_in; - z->avail_in = nread; - allow_restart = 0; - continue; - } - else { /* Error; exit loop, handle below */ - free(decomp); - return exit_zlib(conn, z, &zp->zlib_init, process_zlib_error(conn, z)); + /* FALLTHROUGH */ + default: + result = exit_zlib(conn, z, &zp->zlib_init, process_zlib_error(conn, z)); + break; } } - /* UNREACHED */ + free(decomp); + + /* We're about to leave this call so the `nread' data bytes won't be seen + again. If we are in a state that would wrongly allow restart in raw mode + at the next call, assume output has already started. */ + if(nread && zp->zlib_init == ZLIB_INIT) + zp->zlib_init = started; /* Cannot restart anymore. */ + + return result; } @@ -239,7 +282,7 @@ static CURLcode deflate_unencode_write(struct connectdata *conn, z->avail_in = (uInt) nbytes; /* Now uncompress the data */ - return inflate_stream(conn, writer); + return inflate_stream(conn, writer, ZLIB_INFLATING); } static void deflate_close_writer(struct connectdata *conn, @@ -283,11 +326,12 @@ static CURLcode gzip_init_writer(struct connectdata *conn, zp->zlib_init = ZLIB_INIT_GZIP; /* Transparent gzip decompress state */ } else { - /* we must parse the gzip header ourselves */ + /* we must parse the gzip header and trailer ourselves */ if(inflateInit2(z, -MAX_WBITS) != Z_OK) { return process_zlib_error(conn, z); } - zp->zlib_init = ZLIB_INIT; /* Initial call state */ + zp->trailerlen = 8; /* A CRC-32 and a 32-bit input size (RFC 1952, 2.2) */ + zp->zlib_init = ZLIB_INIT; /* Initial call state */ } return CURLE_OK; @@ -389,7 +433,7 @@ static CURLcode gzip_unencode_write(struct connectdata *conn, z->next_in = (Bytef *) buf; z->avail_in = (uInt) nbytes; /* Now uncompress the data */ - return inflate_stream(conn, writer); + return inflate_stream(conn, writer, ZLIB_INIT_GZIP); } #ifndef OLD_ZLIB_SUPPORT @@ -482,6 +526,11 @@ static CURLcode gzip_unencode_write(struct connectdata *conn, } break; + case ZLIB_GZIP_TRAILER: + z->next_in = (Bytef *) buf; + z->avail_in = (uInt) nbytes; + return process_trailer(conn, zp); + case ZLIB_GZIP_INFLATING: default: /* Inflating stream state */ @@ -496,7 +545,7 @@ static CURLcode gzip_unencode_write(struct connectdata *conn, } /* We've parsed the header, now uncompress the data */ - return inflate_stream(conn, writer); + return inflate_stream(conn, writer, ZLIB_GZIP_INFLATING); #endif } diff --git a/tests/data/Makefile.inc b/tests/data/Makefile.inc index c9e2dc2f6..d8721512d 100644 --- a/tests/data/Makefile.inc +++ b/tests/data/Makefile.inc @@ -45,7 +45,7 @@ test190 test191 test192 test193 test194 test195 test196 test197 test198 \ test199 test200 test201 test202 test203 test204 test205 test206 test207 \ test208 test209 test210 test211 test212 test213 test214 test215 test216 \ test217 test218 test219 test220 test221 test222 test223 test224 test225 \ -test226 test227 test228 test229 test230 test231 test233 test234 \ +test226 test227 test228 test229 test230 test231 test232 test233 test234 \ test235 test236 test237 test238 test239 test240 test241 test242 test243 \ test244 test245 test246 test247 test248 test249 test250 test251 test252 \ test253 test254 test255 test256 test257 test258 test259 test260 test261 \ diff --git a/tests/data/test232 b/tests/data/test232 new file mode 100644 index 000000000..d0860c616 --- /dev/null +++ b/tests/data/test232 @@ -0,0 +1,202 @@ + + + +HTTP +HTTP GET +compressed + + +# +# Server-side + + +SFRUUC8xLjEgMjAwIE9LDQpEYXRlOiBNb24sIDI5IE5vdiAyMDA0IDIxOjU2OjUzIEdNVA0KU2Vy +dmVyOiBBcGFjaGUvMS4zLjMxIChEZWJpYW4gR05VL0xpbnV4KSBtb2RfZ3ppcC8xLjMuMjYuMWEg +UEhQLzQuMy45LTEgbW9kX3NzbC8yLjguMjAgT3BlblNTTC8wLjkuN2QgbW9kX3BlcmwvMS4yOQ0K +VmFyeTogQWNjZXB0LUVuY29kaW5nDQpDb250ZW50LVR5cGU6IHRleHQvaHRtbDsgY2hhcnNldD1J +U08tODg1OS0xDQpDb250ZW50LUVuY29kaW5nOiBkZWZsYXRlDQpDb250ZW50LUxlbmd0aDogMTI4 +Nw0KDQrdWNtu4zYQfTeQf2D91AK2brYTOXC0KHLZBM02wdoFuk8GLdE2G0kUSMq5PPTbO5QoibEc +r3eDvgRIYnLmnMPhDEmTmXx6SmK0IVxQlp51XcvpIpKGLKLp6qx7M73r+/5o3He7n4LO5JeLu/PZ +t/tLlHH2DwllP6ZCAhBNv01nl19Qdy1ldmrbS07EOiFYWimR9gKHDySN7GXS1zzRd6yhFcmoC6Jb +WkEHocqm2k1vTqPAPXa9iW0YSkSEJZnjKCKAGI/9vuP3PRc5w1PHPfXGE9vwG4Q8Ux9R4DnOsO86 +fddHjnfqwc9AUyrEqzhSnJC5WDMugzDncR2OYW/jl3kcF3CE0wjFdNGmFhAdHhGhljpnSVJSUoIk +Y7UAx/wZLRlHYUxJKvuCRgT99fUWSY5TsYSKWjCLRshQ3hkMogLhcgSlWqlwVd8ljYk46uRCddQY +4jmV+MlCNxKJPMtAX6Dr2ey+V/yd9tAVtI86V0X74uZ81kOzy9s/L+Hz9uJ3gF3d3F72iuE/391f +X36FNgzwSGKIqZTqZ0zInm7m0AoZe6BEFNooz2KGIxgCllqekKiZdQ9lWIhHxiPVhMjSPFkU9un0 +9qgTEi7pkoZQVzD9QTj4mChDgWo8wQjFtCAbGXsknERHncVzlaQekmvyZsarslhHndkaqAjD74Km +ajMJSG2dapVgBpsOec5RJ8bpKscrIooYSLqhnKUJDCBAR5fQWBsbKnFM5fNchIyTYHTiD63RycTe +sm+BM8JDkAwGlntsYCvzFhrm8wB7bWwgC5Ne1yzLY8ybsY5HY4hhCMt529MiVAO6A8t3XxFeh2I4 +ymCc0Su0EQ7HxbnhWyNnYuuO6ZmHLAddz6282vAKUw7iD2qMMYDIFyLkNJNwRIpgoE6H16YSBqVP +w/Vc7eXggixxHsuJbRpLGNR/Xh1gGZQ92HloVielrdaLPbFbrEZszRLythAsYMpLFXV42iZD69YC +jaZcvRwuB2CtpGiNyOLFO1wEwFpE0RqRF5odLgJgLaJojUi4hj1GYrY6XKqmaMFGopHlWXK4IIC1 +lKI1IhFZHC4CYC2iaI0IE0+HiwBYiyiaUS8RqfPyB2pWEqq6abqxzHMOaRMk0Ou36hqF2YgfKMlG +VMXYCENE3RwOV1FoLVMQG52Ecs744UolXmtpslnXhAVVraBZemIKhxyk4MvNzP4bncPpASmjeYJu +S8fErhAar76n5JyTmNSZa5nn+v4WnFiuZ8EF6Q33G2x1rzo5dvxRi1hdsNocdS/afXHaBSznYu+a +zATOUQITXjM5l2v4qoactUwlEucSbjKiDqnsV93aoE9gnFISo6kkKXzDrya26WxRoEq76/7vAq8i +oopsIFt0zmIS3D2mhNe4wlRFapuhVr1qCasveE4TmmJpzk5yuCEUtYGC1p2W1/OO97kHe7n7nK7v +7+W6e8eFpbE/6r1u93i4zz3eS/bHe73OXrc7+k7c3wlsf2SD1tjl/W67/LAmMngywUMMrqO1Tm18 +RvI5I2ddTkJ4HSibeknVi7LBmRvZUUPtcuwk6nsLuE+Gqhg7XTuZxuOsRd1+uL3FlVSqDQV2uLOj +X/Vt6redWiW23mkN4u28seLehuP/L2nOT2dsOHhnxtT76uMnyvUGI/cdmXqBp9jHz9LAc4Yn78jS +NaFJhOOPn6jhcDTw3pGosA9PffEzeTIs+qyv/ysUdOpm89+x/wA= + + + +HTTP/1.1 200 OK +Date: Mon, 29 Nov 2004 21:56:53 GMT +Server: Apache/1.3.31 (Debian GNU/Linux) mod_gzip/1.3.26.1a PHP/4.3.9-1 mod_ssl/2.8.20 OpenSSL/0.9.7d mod_perl/1.29 +Vary: Accept-Encoding +Content-Type: text/html; charset=ISO-8859-1 +Content-Encoding: deflate +Content-Length: 1287 + + + + + + 1612 + 1998-08-21 04:01:29 + 2004-10-18 02:22:23 + curl + curl and libcurl + Command line tool and library for client-side URL transfers. + curl and libcurl is a tool for transferring files +using URL syntax. It supports HTTP, HTTPS, FTP, +FTPS, DICT, TELNET, LDAP, FILE, and GOPHER, as +well as HTTP-post, HTTP-put, cookies, FTP upload, +resumed transfers, passwords, portnumbers, SSL +certificates, Kerberos, and proxies. It is powered +by libcurl, the client-side URL transfer library. +There are bindings to libcurl for over 20 +languages and environments. + + 5784.57 + 3.16 + 169 + 6594.54 + 13.81 + 105 + 8.50 + 21 + 183 + 323 + Default + http://freshmeat.net/projects/curl/ + http://freshmeat.net/redir/curl/1612/url_homepage/ + http://freshmeat.net/redir/curl/1612/url_tgz/ + http://freshmeat.net/redir/curl/1612/url_bz2/ + http://freshmeat.net/redir/curl/1612/url_zip/ + http://freshmeat.net/redir/curl/1612/url_changelog/ + http://freshmeat.net/redir/curl/1612/url_rpm/ + http://freshmeat.net/redir/curl/1612/url_deb/ + http://freshmeat.net/redir/curl/1612/url_osx/ + http://freshmeat.net/redir/curl/1612/url_bsdport/ + + http://freshmeat.net/redir/curl/1612/url_cvs/ + http://freshmeat.net/redir/curl/1612/url_list/ + http://freshmeat.net/redir/curl/1612/url_mirror/ + + MIT/X Consortium License + + 7.12.2 + 176085 + 2004-10-18 02:22:23 + + + + + Daniel Stenberg + http://freshmeat.net/~bagder/ + Owner + + + + 12 + 226 + 3 + 2 + 188 + 216 + 200 + 220 + 164 + 90 + 89 + 809 + 150 + 224 + 900 + 839 + + + + 0 + 7464 + 7464 + OpenSSL (Default) + + + 0 + 0 + 7443 + OpenLDAP + + + 0 + 0 + 12351 + zlib + + + 0 + 0 + 32047 + Heimdal + + + 0 + 0 + 44532 + c-ares + + + + + + + + +# +# Client-side + + +libz + + +http + + +HTTP GET deflate raw-compressed content + + +http://%HOSTIP:%HTTPPORT/232 --compressed + + + +# +# Verify data after the test has been "shot" + + +^User-Agent:.* + + +s/^Accept-Encoding: .*/Accept-Encoding: xxx/ + + +GET /232 HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +Accept: */* +Accept-Encoding: xxx + + + +