From 11bf1796cd015373a996e6eb26212e2e1aadb066 Mon Sep 17 00:00:00 2001 From: Patrick Monnerat Date: Sun, 5 Nov 2017 15:28:16 +0100 Subject: [PATCH] HTTP: implement Brotli content encoding This uses the brotli external library (https://github.com/google/brotli). Brotli becomes a feature: additional curl_version_info() bit and structure fields are provided for it and CURLVERSION_NOW bumped. Tests 314 and 315 check Brotli content unencoding with correct and erroneous data. Some tests are updated to accomodate with the now configuration dependent parameters of the Accept-Encoding header. --- configure.ac | 93 +++++++++++++++ docs/INTERNALS.md | 21 ++-- docs/RESOURCES | 2 + docs/TODO | 15 +-- docs/libcurl/curl_version_info.3 | 8 ++ docs/libcurl/symbols-in-versions | 2 + include/curl/curl.h | 10 +- lib/content_encoding.c | 138 ++++++++++++++++++++- lib/urldata.h | 4 + lib/version.c | 36 +++++- src/tool_help.c | 1 + tests/data/Makefile.inc | 2 +- tests/data/test220 | 5 +- tests/data/test221 | 5 +- tests/data/test222 | 5 +- tests/data/test223 | 5 +- tests/data/test224 | 5 +- tests/data/test230 | 5 +- tests/data/test314 | 198 +++++++++++++++++++++++++++++++ tests/data/test315 | 91 ++++++++++++++ tests/runtests.pl | 14 +++ 21 files changed, 633 insertions(+), 32 deletions(-) create mode 100644 tests/data/test314 create mode 100644 tests/data/test315 diff --git a/configure.ac b/configure.ac index 5272feaa0..c15ff4e4c 100755 --- a/configure.ac +++ b/configure.ac @@ -148,6 +148,7 @@ dnl initialize all the info variables curl_ssl_msg="no (--with-{ssl,gnutls,nss,polarssl,mbedtls,cyassl,axtls,winssl,darwinssl} )" curl_ssh_msg="no (--with-libssh2)" curl_zlib_msg="no (--with-zlib)" + curl_brotli_msg="no (--with-brotli)" curl_gss_msg="no (--with-gssapi)" curl_tls_srp_msg="no (--enable-tls-srp)" curl_res_msg="default (--enable-ares / --enable-threaded-resolver)" @@ -975,6 +976,94 @@ dnl set variable for use in automakefile(s) AM_CONDITIONAL(HAVE_LIBZ, test x"$AMFIXLIB" = x1) AC_SUBST(ZLIB_LIBS) +dnl ********************************************************************** +dnl Check for the presence of BROTLI decoder libraries and headers +dnl ********************************************************************** + +dnl Brotli project home page: https://github.com/google/brotli + +dnl Default to compiler & linker defaults for BROTLI files & libraries. +OPT_BROTLI=off +AC_ARG_WITH(brotli,dnl +AC_HELP_STRING([--with-brotli=PATH],[Where to look for brotli, PATH points to the BROTLI installation; when possible, set the PKG_CONFIG_PATH environment variable instead of using this option]) +AC_HELP_STRING([--without-brotli], [disable BROTLI]), + OPT_BROTLI=$withval) + +if test X"$OPT_BROTLI" != Xno; then + dnl backup the pre-brotli variables + CLEANLDFLAGS="$LDFLAGS" + CLEANCPPFLAGS="$CPPFLAGS" + CLEANLIBS="$LIBS" + + case "$OPT_BROTLI" in + yes) + dnl --with-brotli (without path) used + CURL_CHECK_PKGCONFIG(libbrotlidec) + + if test "$PKGCONFIG" != "no" ; then + LIB_BROTLI=`$PKGCONFIG --libs-only-l libbrotlidec` + LD_BROTLI=`$PKGCONFIG --libs-only-L libbrotlidec` + CPP_BROTLI=`$PKGCONFIG --cflags-only-I libbrotlidec` + version=`$PKGCONFIG --modversion libbrotlidec` + DIR_BROTLI=`echo $LD_BROTLI | $SED -e 's/-L//'` + fi + + ;; + off) + dnl no --with-brotli option given, just check default places + ;; + *) + dnl use the given --with-brotli spot + PREFIX_BROTLI=$OPT_BROTLI + ;; + esac + + dnl if given with a prefix, we set -L and -I based on that + if test -n "$PREFIX_BROTLI"; then + LIB_BROTLI="-lbrotlidec" + LD_BROTLI=-L${PREFIX_BROTLI}/lib$libsuff + CPP_BROTLI=-I${PREFIX_BROTLI}/include + DIR_BROTLI=${PREFIX_BROTLI}/lib$libsuff + fi + + LDFLAGS="$LDFLAGS $LD_BROTLI" + CPPFLAGS="$CPPFLAGS $CPP_BROTLI" + LIBS="$LIB_BROTLI $LIBS" + + AC_CHECK_LIB(brotlidec, BrotliDecoderDecompress) + + AC_CHECK_HEADERS(brotli/decode.h, + curl_brotli_msg="enabled (libbrotlidec)" + HAVE_BROTLI=1 + AC_DEFINE(HAVE_BROTLI, 1, [if BROTLI is in use]) + AC_SUBST(HAVE_BROTLI, [1]) + ) + + if test X"$OPT_BROTLI" != Xoff && + test "$HAVE_BROTLI" != "1"; then + AC_MSG_ERROR([BROTLI libs and/or directories were not found where specified!]) + fi + + if test "$HAVE_BROTLI" = "1"; then + if test -n "$DIR_BROTLI"; then + dnl when the brotli shared libs were found in a path that the run-time + dnl linker doesn't search through, we need to add it to LD_LIBRARY_PATH + dnl to prevent further configure tests to fail due to this + + if test "x$cross_compiling" != "xyes"; then + LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$DIR_BROTLI" + export LD_LIBRARY_PATH + AC_MSG_NOTICE([Added $DIR_BROTLI to LD_LIBRARY_PATH]) + fi + fi + else + dnl no brotli, revert back to clean variables + LDFLAGS=$CLEANLDFLAGS + CPPFLAGS=$CLEANCPPFLAGS + LIBS=$CLEANLIBS + fi +fi + dnl ********************************************************************** dnl Check for LDAP dnl ********************************************************************** @@ -3786,6 +3875,9 @@ fi if test "x$HAVE_LIBZ" = "x1"; then SUPPORT_FEATURES="$SUPPORT_FEATURES libz" fi +if test "x$HAVE_BROTLI" = "x1"; then + SUPPORT_FEATURES="$SUPPORT_FEATURES brotli" +fi if test "x$USE_ARES" = "x1" -o "x$USE_THREADS_POSIX" = "x1" \ -o "x$USE_THREADS_WIN32" = "x1"; then SUPPORT_FEATURES="$SUPPORT_FEATURES AsynchDNS" @@ -4003,6 +4095,7 @@ AC_MSG_NOTICE([Configured to build curl/libcurl: SSL support: ${curl_ssl_msg} SSH support: ${curl_ssh_msg} zlib support: ${curl_zlib_msg} + brotli support: ${curl_brotli_msg} GSS-API support: ${curl_gss_msg} TLS-SRP support: ${curl_tls_srp_msg} resolver: ${curl_res_msg} diff --git a/docs/INTERNALS.md b/docs/INTERNALS.md index fb9d50378..1a4350e27 100644 --- a/docs/INTERNALS.md +++ b/docs/INTERNALS.md @@ -644,9 +644,9 @@ Content Encoding [HTTP/1.1][4] specifies that a client may request that a server encode its response. This is usually used to compress a response using one (or more) encodings from a set of commonly available compression techniques. These - schemes include 'deflate' (the zlib algorithm), 'gzip' and 'compress'. A - client requests that the server perform an encoding by including an - Accept-Encoding header in the request document. The value of the header + schemes include 'deflate' (the zlib algorithm), 'gzip' 'br' (brotli) and + 'compress'. A client requests that the server perform an encoding by including + an Accept-Encoding header in the request document. The value of the header should be one of the recognized tokens 'deflate', ... (there's a way to register new schemes/tokens, see sec 3.5 of the spec). A server MAY honor the client's encoding request. When a response is encoded, the server @@ -661,9 +661,10 @@ Content Encoding ## Supported content encodings - The 'deflate' and 'gzip' content encodings are supported by libcurl. Both - regular and chunked transfers work fine. The zlib library is required for - this feature. + The 'deflate', 'gzip' and 'br' content encodings are supported by libcurl. + Both regular and chunked transfers work fine. The zlib library is required + for the 'deflate' and 'gzip' encodings, while the brotli decoding library is + for the 'br' encoding. ## The libcurl interface @@ -674,10 +675,10 @@ Content Encoding where string is the intended value of the Accept-Encoding header. Currently, libcurl does support multiple encodings but only - understands how to process responses that use the "deflate" or "gzip" - Content-Encoding, so the only values for [`CURLOPT_ACCEPT_ENCODING`][5] - that will work (besides "identity," which does nothing) are "deflate" - and "gzip". If a response is encoded using the "compress" or methods, + understands how to process responses that use the "deflate", "gzip" and/or + "br" content encodings, so the only values for [`CURLOPT_ACCEPT_ENCODING`][5] + that will work (besides "identity," which does nothing) are "deflate", + "gzip" and "br". If a response is encoded using the "compress" or methods, libcurl will return an error indicating that the response could not be decoded. If is NULL no Accept-Encoding header is generated. If is a zero-length string, then an Accept-Encoding header diff --git a/docs/RESOURCES b/docs/RESOURCES index 1ad8aac31..2880b8fb0 100644 --- a/docs/RESOURCES +++ b/docs/RESOURCES @@ -81,3 +81,5 @@ This document lists documents and standards used by curl. RFC 4616 - PLAIN authentication RFC 4954 - SMTP Authentication + + RFC 7932 - Brotli Compressed Data Format diff --git a/docs/TODO b/docs/TODO index 3e5f8bd75..d2f93b270 100644 --- a/docs/TODO +++ b/docs/TODO @@ -64,9 +64,8 @@ 5.4 HTTP Digest using SHA-256 5.5 auth= in URLs 5.6 Refuse "downgrade" redirects - 5.7 Brotli compression - 5.8 QUIC - 5.9 Leave secure cookies alone + 5.7 QUIC + 5.8 Leave secure cookies alone 6. TELNET 6.1 ditch stdin @@ -514,13 +513,7 @@ This is not detailed in any FTP specification. Consider a way to tell curl to refuse to "downgrade" protocol with a redirect and/or possibly a bit that refuses redirect to change protocol completely. -5.7 Brotli compression - - Brotli compression performs better than gzip and is being implemented by - browsers and servers widely. The algorithm: https://github.com/google/brotli - The Firefox bug: https://bugzilla.mozilla.org/show_bug.cgi?id=366559 - -5.8 QUIC +5.7 QUIC The standardization process of QUIC has been taken to the IETF and can be followed on the [IETF QUIC Mailing @@ -530,7 +523,7 @@ This is not detailed in any FTP specification. implemented. This, to allow other projects to benefit from the work and to thus broaden the interest and chance of others to participate. -5.9 Leave secure cookies alone +5.8 Leave secure cookies alone Non-secure origins (HTTP sites) should not be allowed to set or modify cookies with the 'secure' property: diff --git a/docs/libcurl/curl_version_info.3 b/docs/libcurl/curl_version_info.3 index 26af1c277..1154f4ce7 100644 --- a/docs/libcurl/curl_version_info.3 +++ b/docs/libcurl/curl_version_info.3 @@ -72,6 +72,12 @@ typedef struct { const char *libssh_version; /* human readable string */ + /* when 'age' is 4 or higher (7.57.0 or later), the members below also + exist */ + unsigned int brotli_ver_num; /* Numeric Brotli version + (MAJOR << 24) | (MINOR << 12) | PATCH */ + const char *brotli_version; /* human readable string. */ + } curl_version_info_data; .fi @@ -160,6 +166,8 @@ libcurl was built with support for HTTPS-proxy. libcurl was built with multiple SSL backends. For details, see \fIcurl_global_sslset(3)\fP. (Added in 7.56.0) +.IP CURL_VERSION_BROTLI +supports HTTP Brotli content encoding using libbrotlidec (Added in 7.57.0) .RE \fIssl_version\fP is an ASCII string for the OpenSSL version used. If libcurl has no SSL support, this is NULL. diff --git a/docs/libcurl/symbols-in-versions b/docs/libcurl/symbols-in-versions index f912fb719..7878b227f 100644 --- a/docs/libcurl/symbols-in-versions +++ b/docs/libcurl/symbols-in-versions @@ -699,6 +699,7 @@ CURLUSESSL_ALL 7.17.0 CURLUSESSL_CONTROL 7.17.0 CURLUSESSL_NONE 7.17.0 CURLUSESSL_TRY 7.17.0 +CURLVERSION_FIFTH 7.57.0 CURLVERSION_FIRST 7.10 CURLVERSION_FOURTH 7.16.1 CURLVERSION_NOW 7.10 @@ -829,6 +830,7 @@ CURL_TIMECOND_NONE 7.9.7 CURL_TLSAUTH_NONE 7.21.4 CURL_TLSAUTH_SRP 7.21.4 CURL_VERSION_ASYNCHDNS 7.10.7 +CURL_VERSION_BROTLI 7.57.0 CURL_VERSION_CONV 7.15.4 CURL_VERSION_CURLDEBUG 7.19.6 CURL_VERSION_DEBUG 7.10.6 diff --git a/include/curl/curl.h b/include/curl/curl.h index 7139a3311..764cbc703 100644 --- a/include/curl/curl.h +++ b/include/curl/curl.h @@ -2592,6 +2592,7 @@ typedef enum { CURLVERSION_SECOND, CURLVERSION_THIRD, CURLVERSION_FOURTH, + CURLVERSION_FIFTH, CURLVERSION_LAST /* never actually use this */ } CURLversion; @@ -2600,7 +2601,7 @@ typedef enum { meant to be a built-in version number for what kind of struct the caller expects. If the struct ever changes, we redefine the NOW to another enum from above. */ -#define CURLVERSION_NOW CURLVERSION_FOURTH +#define CURLVERSION_NOW CURLVERSION_FIFTH typedef struct { CURLversion age; /* age of the returned struct */ @@ -2628,6 +2629,12 @@ typedef struct { const char *libssh_version; /* human readable string */ + /* These fields were added in CURLVERSION_FIFTH */ + + unsigned int brotli_ver_num; /* Numeric Brotli version + (MAJOR << 24) | (MINOR << 12) | PATCH */ + const char *brotli_version; /* human readable string. */ + } curl_version_info_data; #define CURL_VERSION_IPV6 (1<<0) /* IPv6-enabled */ @@ -2658,6 +2665,7 @@ typedef struct { for cookie domain verification */ #define CURL_VERSION_HTTPS_PROXY (1<<21) /* HTTPS-proxy support built-in */ #define CURL_VERSION_MULTI_SSL (1<<22) /* Multiple SSL backends available */ +#define CURL_VERSION_BROTLI (1<<23) /* Brotli features are present. */ /* * NAME curl_version_info() diff --git a/lib/content_encoding.c b/lib/content_encoding.c index 76a9e6866..6b3168573 100644 --- a/lib/content_encoding.c +++ b/lib/content_encoding.c @@ -37,14 +37,15 @@ #ifndef CURL_DISABLE_HTTP +#define DSIZ CURL_MAX_WRITE_SIZE /* buffer size for decompressed data */ + + #ifdef HAVE_LIBZ /* Comment this out if zlib is always going to be at least ver. 1.2.0.4 (doing so will reduce code size slightly). */ #define OLD_ZLIB_SUPPORT 1 -#define DSIZ CURL_MAX_WRITE_SIZE /* buffer size for decompressed data */ - #define GZIP_MAGIC_0 0x1f #define GZIP_MAGIC_1 0x8b @@ -505,6 +506,136 @@ static const content_encoding gzip_encoding = { #endif /* HAVE_LIBZ */ +#ifdef HAVE_BROTLI + +/* Writer parameters. */ +typedef struct { + BrotliDecoderState *br; /* State structure for brotli. */ +} brotli_params; + + +static CURLcode brotli_map_error(BrotliDecoderErrorCode be) +{ + switch(be) { + case BROTLI_DECODER_ERROR_FORMAT_EXUBERANT_NIBBLE: + case BROTLI_DECODER_ERROR_FORMAT_EXUBERANT_META_NIBBLE: + case BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_ALPHABET: + case BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_SAME: + case BROTLI_DECODER_ERROR_FORMAT_CL_SPACE: + case BROTLI_DECODER_ERROR_FORMAT_HUFFMAN_SPACE: + case BROTLI_DECODER_ERROR_FORMAT_CONTEXT_MAP_REPEAT: + case BROTLI_DECODER_ERROR_FORMAT_BLOCK_LENGTH_1: + case BROTLI_DECODER_ERROR_FORMAT_BLOCK_LENGTH_2: + case BROTLI_DECODER_ERROR_FORMAT_TRANSFORM: + case BROTLI_DECODER_ERROR_FORMAT_DICTIONARY: + case BROTLI_DECODER_ERROR_FORMAT_WINDOW_BITS: + case BROTLI_DECODER_ERROR_FORMAT_PADDING_1: + case BROTLI_DECODER_ERROR_FORMAT_PADDING_2: + case BROTLI_DECODER_ERROR_COMPOUND_DICTIONARY: + case BROTLI_DECODER_ERROR_DICTIONARY_NOT_SET: + case BROTLI_DECODER_ERROR_INVALID_ARGUMENTS: + return CURLE_BAD_CONTENT_ENCODING; + case BROTLI_DECODER_ERROR_ALLOC_CONTEXT_MODES: + case BROTLI_DECODER_ERROR_ALLOC_TREE_GROUPS: + case BROTLI_DECODER_ERROR_ALLOC_CONTEXT_MAP: + case BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_1: + case BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_2: + case BROTLI_DECODER_ERROR_ALLOC_BLOCK_TYPE_TREES: + return CURLE_OUT_OF_MEMORY; + default: + break; + } + return CURLE_WRITE_ERROR; +} + +static CURLcode brotli_init_writer(struct connectdata *conn, + contenc_writer *writer) +{ + brotli_params *bp = (brotli_params *) &writer->params; + + (void) conn; + + if(!writer->downstream) + return CURLE_WRITE_ERROR; + + bp->br = BrotliDecoderCreateInstance(NULL, NULL, NULL); + return bp->br? CURLE_OK: CURLE_OUT_OF_MEMORY; +} + +static CURLcode brotli_unencode_write(struct connectdata *conn, + contenc_writer *writer, + const char *buf, size_t nbytes) +{ + brotli_params *bp = (brotli_params *) &writer->params; + const uint8_t *src = (const uint8_t *) buf; + char *decomp; + uint8_t *dst; + size_t dstleft; + CURLcode result = CURLE_OK; + + if(!nbytes) + return CURLE_OK; + if(!bp->br) + return CURLE_WRITE_ERROR; /* Stream already ended. */ + + decomp = malloc(DSIZ); + if(!decomp) + return CURLE_OUT_OF_MEMORY; + + while(nbytes && result == CURLE_OK) { + BrotliDecoderResult r; + + dst = (uint8_t *) decomp; + dstleft = DSIZ; + r = BrotliDecoderDecompressStream(bp->br, + &nbytes, &src, &dstleft, &dst, NULL); + result = Curl_unencode_write(conn, writer->downstream, + decomp, DSIZ - dstleft); + if(result) + break; + switch(r) { + case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT: + case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT: + break; + case BROTLI_DECODER_RESULT_SUCCESS: + BrotliDecoderDestroyInstance(bp->br); + bp->br = NULL; + if(nbytes) + result = CURLE_WRITE_ERROR; + break; + default: + result = brotli_map_error(BrotliDecoderGetErrorCode(bp->br)); + break; + } + } + free(decomp); + return result; +} + +static void brotli_close_writer(struct connectdata *conn, + contenc_writer *writer) +{ + brotli_params *bp = (brotli_params *) &writer->params; + + (void) conn; + + if(bp->br) { + BrotliDecoderDestroyInstance(bp->br); + bp->br = NULL; + } +} + +static const content_encoding brotli_encoding = { + "br", + NULL, + brotli_init_writer, + brotli_unencode_write, + brotli_close_writer, + sizeof(brotli_params) +}; +#endif + + /* Identity handler. */ static CURLcode identity_init_writer(struct connectdata *conn, contenc_writer *writer) @@ -543,6 +674,9 @@ static const content_encoding * const encodings[] = { #ifdef HAVE_LIBZ &deflate_encoding, &gzip_encoding, +#endif +#ifdef HAVE_BROTLI + &brotli_encoding, #endif NULL }; diff --git a/lib/urldata.h b/lib/urldata.h index cfdd8d028..7dfb26b6d 100644 --- a/lib/urldata.h +++ b/lib/urldata.h @@ -96,6 +96,10 @@ #endif #endif +#ifdef HAVE_BROTLI +#include +#endif + #include #include "http_chunks.h" /* for the structs and enum stuff */ diff --git a/lib/version.c b/lib/version.c index ebd600635..66c761e34 100644 --- a/lib/version.c +++ b/lib/version.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2016, Daniel Stenberg, , et al. + * Copyright (C) 1998 - 2017, Daniel Stenberg, , et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -74,6 +74,18 @@ void Curl_version_init(void) curl_version_info(CURLVERSION_NOW); } +#ifdef HAVE_BROTLI +static size_t brotli_version(char *buf, size_t bufsz) +{ + uint32_t brotli_version = BrotliDecoderVersion(); + unsigned int major = brotli_version >> 24; + unsigned int minor = (brotli_version & 0x00FFFFFF) >> 12; + unsigned int patch = brotli_version & 0x00000FFF; + + return snprintf(buf, bufsz, "%u.%u.%u", major, minor, patch); +} +#endif + char *curl_version(void) { static bool initialized; @@ -105,6 +117,14 @@ char *curl_version(void) left -= len; ptr += len; #endif +#ifdef HAVE_BROTLI + len = snprintf(ptr, left, "%s", " brotli/"); + left -= len; + ptr += len; + len = brotli_version(ptr, left); + left -= len; + ptr += len; +#endif #ifdef USE_ARES /* this function is only present in c-ares, not in the original ares */ len = snprintf(ptr, left, " c-ares/%s", ares_version(NULL)); @@ -326,6 +346,9 @@ static curl_version_info_data version_info = { #endif #if defined(CURL_WITH_MULTI_SSL) | CURL_VERSION_MULTI_SSL +#endif +#if defined(HAVE_BROTLI) + | CURL_VERSION_BROTLI #endif , NULL, /* ssl_version */ @@ -337,6 +360,8 @@ static curl_version_info_data version_info = { NULL, /* libidn version */ 0, /* iconv version */ NULL, /* ssh lib version */ + 0, /* brotli_ver_num */ + NULL, /* brotli version */ }; curl_version_info_data *curl_version_info(CURLversion stamp) @@ -348,6 +373,9 @@ curl_version_info_data *curl_version_info(CURLversion stamp) #ifdef USE_SSL static char ssl_buffer[80]; #endif +#ifdef HAVE_BROTLI + static char brotli_buffer[80]; +#endif if(initialized) return &version_info; @@ -396,6 +424,12 @@ curl_version_info_data *curl_version_info(CURLversion stamp) version_info.libssh_version = ssh_buffer; #endif +#ifdef HAVE_BROTLI + version_info.brotli_ver_num = BrotliDecoderVersion(); + brotli_version(brotli_buffer, sizeof brotli_buffer); + version_info.brotli_version = brotli_buffer; +#endif + (void)stamp; /* avoid compiler warnings, we don't use this */ initialized = true; diff --git a/src/tool_help.c b/src/tool_help.c index 486f65dc8..c6d329cdf 100644 --- a/src/tool_help.c +++ b/src/tool_help.c @@ -499,6 +499,7 @@ static const struct feat feats[] = { {"NTLM_WB", CURL_VERSION_NTLM_WB}, {"SSL", CURL_VERSION_SSL}, {"libz", CURL_VERSION_LIBZ}, + {"brotli", CURL_VERSION_BROTLI}, {"CharConv", CURL_VERSION_CONV}, {"TLS-SRP", CURL_VERSION_TLSAUTH_SRP}, {"HTTP2", CURL_VERSION_HTTP2}, diff --git a/tests/data/Makefile.inc b/tests/data/Makefile.inc index 305f6f318..6d253afa0 100644 --- a/tests/data/Makefile.inc +++ b/tests/data/Makefile.inc @@ -54,7 +54,7 @@ test271 test272 test273 test274 test275 test276 test277 test278 test279 \ test280 test281 test282 test283 test284 test285 test286 test287 test288 \ test289 test290 test291 test292 test293 test294 test295 test296 test297 \ test298 test299 test300 test301 test302 test303 test304 test305 test306 \ -test307 test308 test309 test310 test311 test312 test313 \ +test307 test308 test309 test310 test311 test312 test313 test314 test315 \ test320 test321 test322 test323 test324 \ test325 \ test350 test351 test352 test353 test354 \ diff --git a/tests/data/test220 b/tests/data/test220 index 2fb0b8a6a..7fd264345 100644 --- a/tests/data/test220 +++ b/tests/data/test220 @@ -57,11 +57,14 @@ http://%HOSTIP:%HTTPPORT/220 --compressed ^User-Agent:.* + +s/^Accept-Encoding: .*/Accept-Encoding: xxx/ + GET /220 HTTP/1.1 Host: %HOSTIP:%HTTPPORT Accept: */* -Accept-Encoding: deflate, gzip +Accept-Encoding: xxx diff --git a/tests/data/test221 b/tests/data/test221 index 95edb4990..3a85439d0 100644 --- a/tests/data/test221 +++ b/tests/data/test221 @@ -57,11 +57,14 @@ http://%HOSTIP:%HTTPPORT/221 --compressed ^User-Agent:.* + +s/^Accept-Encoding: .*/Accept-Encoding: xxx/ + GET /221 HTTP/1.1 Host: %HOSTIP:%HTTPPORT Accept: */* -Accept-Encoding: deflate, gzip +Accept-Encoding: xxx diff --git a/tests/data/test222 b/tests/data/test222 index a4594869e..865266e07 100644 --- a/tests/data/test222 +++ b/tests/data/test222 @@ -188,11 +188,14 @@ http://%HOSTIP:%HTTPPORT/222 --compressed ^User-Agent:.* + +s/^Accept-Encoding: .*/Accept-Encoding: xxx/ + GET /222 HTTP/1.1 Host: %HOSTIP:%HTTPPORT Accept: */* -Accept-Encoding: deflate, gzip +Accept-Encoding: xxx diff --git a/tests/data/test223 b/tests/data/test223 index 196e78c80..884967e3f 100644 --- a/tests/data/test223 +++ b/tests/data/test223 @@ -78,11 +78,14 @@ http://%HOSTIP:%HTTPPORT/223 --compressed ^User-Agent:.* + +s/^Accept-Encoding: .*/Accept-Encoding: xxx/ + GET /223 HTTP/1.1 Host: %HOSTIP:%HTTPPORT Accept: */* -Accept-Encoding: deflate, gzip +Accept-Encoding: xxx diff --git a/tests/data/test224 b/tests/data/test224 index 1c8ad2380..a56046873 100644 --- a/tests/data/test224 +++ b/tests/data/test224 @@ -93,11 +93,14 @@ http://%HOSTIP:%HTTPPORT/224 --compressed ^User-Agent:.* + +s/^Accept-Encoding: .*/Accept-Encoding: xxx/ + GET /224 HTTP/1.1 Host: %HOSTIP:%HTTPPORT Accept: */* -Accept-Encoding: deflate, gzip +Accept-Encoding: xxx diff --git a/tests/data/test230 b/tests/data/test230 index cc166a3f1..2174434b3 100644 --- a/tests/data/test230 +++ b/tests/data/test230 @@ -189,11 +189,14 @@ http://%HOSTIP:%HTTPPORT/230 --compressed ^User-Agent:.* + +s/^Accept-Encoding: .*/Accept-Encoding: xxx/ + GET /230 HTTP/1.1 Host: %HOSTIP:%HTTPPORT Accept: */* -Accept-Encoding: deflate, gzip +Accept-Encoding: xxx diff --git a/tests/data/test314 b/tests/data/test314 new file mode 100644 index 000000000..f4703cdeb --- /dev/null +++ b/tests/data/test314 @@ -0,0 +1,198 @@ + + + +HTTP +HTTP GET +compressed + + +# +# Server-side + + +SFRUUC8xLjEgMjAwIE9LDQpEYXRlOiBNb24sIDI5IE5vdiAyMDA0IDIxOjU2OjUzIEdNVA0KU2Vy +dmVyOiBBcGFjaGUvMS4zLjMxIChEZWJpYW4gR05VL0xpbnV4KSBtb2RfZ3ppcC8xLjMuMjYuMWEg +UEhQLzQuMy45LTEgbW9kX3NzbC8yLjguMjAgT3BlblNTTC8wLjkuN2QgbW9kX3BlcmwvMS4yOQ0K +VmFyeTogQWNjZXB0LUVuY29kaW5nDQpDb250ZW50LVR5cGU6IHRleHQvaHRtbDsgY2hhcnNldD1J +U08tODg1OS0xDQpDb250ZW50LUVuY29kaW5nOiBicg0KQ29udGVudC1MZW5ndGg6IDEwNTYNCg0K +G7ATAJwFdhtdgaQ8i+mZBoO/lwogPKuqHpeP38jV5TDITTB7/oJVCS69FFDKWDVtMk8y4SfMSu/a +9vvLxWPweDCKePH/2y9VIkbF+EgCYSNs9v53J8QTIHT4ZucHCCRQiXRdT6XdE60KSlbIvobr5rJQ +sRn7ipIjMVMq3Go+/UXtY2d0yP1qaaGSxCn8nZuUNGh74KOI7EEkgFl1tjYytkpc9mJJy9J+wTTI ++HroUQP2VR2DYkNoUECqgtOLlGcVEln4+eVzEWcrb8fNrcrVxLArJBpSd8FX8eZs8ebJUO7aBZ5e +pHz6zel7lhLlfHoQIkGh34riaSVr7VTGDmmO6HjSCzKO27LybZ9I3CtMSD2Il4mB131Tlcbut1Bd +zL4XU4DZYMLBN4jwVZEoHpjzHX+vQ3prnrNw4oB7OWOr/fBzjvfjDuO24WxwzPqPo+V6VNcthz1p +fF1+sMK4yWY7He33m32EuQgQFSZ3a5Wu4FyQcAb45Z+wUxM5XCmX52YmdUR2YTs+W+bNw2EZSfMR +cP3CinyJI/cTT+JubL3T4COkhz0Rffeoh/3E4c/6ugma1ubhokYecXp8HBwmeDL48d62H26u69DO +yMhg1PFj+oVDWnK4K+L5AlRr0mpJLqoGHrzflMLQ6qL2oIo9hN6qCeZEEqXM+/KunVYpWVeTY+ht +hA0y5p5RLLTTS4cehaJOpbFyAVxZOardIkJAVx0NshOZY4hDbts9BXsXzFEOgsFhrIQYgh04StZz +llIRMVDptYlwGmpZCHHmVECdGiFIfEhkQ2INSwMCuuKpaycgSOO9hJA9UFKDBdzTiLJBP9oUVkKL +bHjwicICCi3k0HcppcvQaW27AMI06kuQU4WUGizgnkaUDcZqCgsotMgG528UFlBo8SFpb05OAjJq +2gEI0UgN93KS1OvAOYSLN5IaLOCeRnQpJXuLUwcm7urpg6lYxAk26uEoADdsRytHGkSWjOKP6T07 +wiceuNo7CXyu7ohtUZXoEWawRHGVkPDVJYqH+xa0DDRKSSgM4K3efLVPSTaUPvBGIZgnn2JBFFWa +MsKZguUuUnz6qaSGqnmGAYiupdC1EFye58V4CLbWVjJU4NF2jrOUYR/Dv04zYwVQtQcFzgmK6H4N +HAhmb0a6pQRKxZaZ+x2vCC7sCuIu4dNCATwqzk12ue6oEsxzYybLPNGJd084M43O9W8E+5/drd/F +QVB2X4jlFlCuHuWeQxQo+w73Tb9swW692v3BlfQTP1ClWzuJ+RwuSb9m4V3QVa4MEL+0Xzc5FX9P ++YX1cgaL+6oMHw7L+IOjOt+n1BOloyqk35lLHX7RZmu8SckMnGP95XjWc4FRKP9x/iXrKaeCnut/ +zstyZdJS5FRmBT/wb5KK9YWBGnqPLO8isN2HS8gA + + + +HTTP/1.1 200 OK +Date: Mon, 29 Nov 2004 21:56:53 GMT +Server: Apache/1.3.31 (Debian GNU/Linux) mod_gzip/1.3.26.1a PHP/4.3.9-1 mod_ssl/2.8.20 OpenSSL/0.9.7d mod_perl/1.29 +Vary: Accept-Encoding +Content-Type: text/html; charset=ISO-8859-1 +Content-Encoding: br +Content-Length: 1056 + + + + + + 1612 + 1998-08-21 04:01:29 + 2004-10-18 02:22:23 + curl + curl and libcurl + Command line tool and library for client-side URL transfers. + curl and libcurl is a tool for transferring files +using URL syntax. It supports HTTP, HTTPS, FTP, +FTPS, DICT, TELNET, LDAP, FILE, and GOPHER, as +well as HTTP-post, HTTP-put, cookies, FTP upload, +resumed transfers, passwords, portnumbers, SSL +certificates, Kerberos, and proxies. It is powered +by libcurl, the client-side URL transfer library. +There are bindings to libcurl for over 20 +languages and environments. + + 5784.57 + 3.16 + 169 + 6594.54 + 13.81 + 105 + 8.50 + 21 + 183 + 323 + Default + http://freshmeat.net/projects/curl/ + http://freshmeat.net/redir/curl/1612/url_homepage/ + http://freshmeat.net/redir/curl/1612/url_tgz/ + http://freshmeat.net/redir/curl/1612/url_bz2/ + http://freshmeat.net/redir/curl/1612/url_zip/ + http://freshmeat.net/redir/curl/1612/url_changelog/ + http://freshmeat.net/redir/curl/1612/url_rpm/ + http://freshmeat.net/redir/curl/1612/url_deb/ + http://freshmeat.net/redir/curl/1612/url_osx/ + http://freshmeat.net/redir/curl/1612/url_bsdport/ + + http://freshmeat.net/redir/curl/1612/url_cvs/ + http://freshmeat.net/redir/curl/1612/url_list/ + http://freshmeat.net/redir/curl/1612/url_mirror/ + + MIT/X Consortium License + + 7.12.2 + 176085 + 2004-10-18 02:22:23 + + + + + Daniel Stenberg + http://freshmeat.net/~bagder/ + Owner + + + + 12 + 226 + 3 + 2 + 188 + 216 + 200 + 220 + 164 + 90 + 89 + 809 + 150 + 224 + 900 + 839 + + + + 0 + 7464 + 7464 + OpenSSL (Default) + + + 0 + 0 + 7443 + OpenLDAP + + + 0 + 0 + 12351 + zlib + + + 0 + 0 + 32047 + Heimdal + + + 0 + 0 + 44532 + c-ares + + + + + + + + +# +# Client-side + + +brotli + + +http + + +HTTP GET brotli compressed content + + +http://%HOSTIP:%HTTPPORT/314 --compressed + + + +# +# Verify data after the test has been "shot" + + +^User-Agent:.* + + +s/^Accept-Encoding: .*/Accept-Encoding: xxx/ + + +GET /314 HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +Accept: */* +Accept-Encoding: xxx + + + + diff --git a/tests/data/test315 b/tests/data/test315 new file mode 100644 index 000000000..c75d9ae7b --- /dev/null +++ b/tests/data/test315 @@ -0,0 +1,91 @@ + + + +HTTP +HTTP GET +compressed +FAILURE + + +# +# Server-side + +# this brotli chunk has three bytes removed from the beginning + +SFRUUC8xLjEgMjAwIE9LDQpEYXRlOiBNb24sIDI5IE5vdiAyMDA0IDIxOjU2OjUzIEdNVA0KU2Vy +dmVyOiBBcGFjaGUvMS4zLjMxIChEZWJpYW4gR05VL0xpbnV4KSBtb2RfZ3ppcC8xLjMuMjYuMWEg +UEhQLzQuMy45LTEgbW9kX3NzbC8yLjguMjAgT3BlblNTTC8wLjkuN2QgbW9kX3BlcmwvMS4yOQ0K +VmFyeTogQWNjZXB0LUVuY29kaW5nDQpDb250ZW50LVR5cGU6IHRleHQvaHRtbDsgY2hhcnNldD1J +U08tODg1OS0xDQpDb250ZW50LUVuY29kaW5nOiBicg0KQ29udGVudC1MZW5ndGg6IDEwNTYNCg0K +AJwFdhtdgaQ8i+mZBoO/lwogPKuqHpeP38jV5TDITTB7/oJVCS69FFDKWDVtMk8y4SfMSu/a9vvL +xWPweDCKePH/2y9VIkbF+EgCYSNs9v53J8QTIHT4ZucHCCRQiXRdT6XdE60KSlbIvobr5rJQsRn7 +ipIjMVMq3Go+/UXtY2d0yP1qaaGSxCn8nZuUNGh74KOI7EEkgFl1tjYytkpc9mJJy9J+wTTI+Hro +UQP2VR2DYkNoUECqgtOLlGcVEln4+eVzEWcrb8fNrcrVxLArJBpSd8FX8eZs8ebJUO7aBZ5epHz6 +zel7lhLlfHoQIkGh34riaSVr7VTGDmmO6HjSCzKO27LybZ9I3CtMSD2Il4mB131Tlcbut1BdzL4X +U4DZYMLBN4jwVZEoHpjzHX+vQ3prnrNw4oB7OWOr/fBzjvfjDuO24WxwzPqPo+V6VNcthz1pfF1+ +sMK4yWY7He33m32EuQgQFSZ3a5Wu4FyQcAb45Z+wUxM5XCmX52YmdUR2YTs+W+bNw2EZSfMRcP3C +inyJI/cTT+JubL3T4COkhz0Rffeoh/3E4c/6ugma1ubhokYecXp8HBwmeDL48d62H26u69DOyMhg +1PFj+oVDWnK4K+L5AlRr0mpJLqoGHrzflMLQ6qL2oIo9hN6qCeZEEqXM+/KunVYpWVeTY+hthA0y +5p5RLLTTS4cehaJOpbFyAVxZOardIkJAVx0NshOZY4hDbts9BXsXzFEOgsFhrIQYgh04StZzllIR +MVDptYlwGmpZCHHmVECdGiFIfEhkQ2INSwMCuuKpaycgSOO9hJA9UFKDBdzTiLJBP9oUVkKLbHjw +icICCi3k0HcppcvQaW27AMI06kuQU4WUGizgnkaUDcZqCgsotMgG528UFlBo8SFpb05OAjJq2gEI +0UgN93KS1OvAOYSLN5IaLOCeRnQpJXuLUwcm7urpg6lYxAk26uEoADdsRytHGkSWjOKP6T07wice +uNo7CXyu7ohtUZXoEWawRHGVkPDVJYqH+xa0DDRKSSgM4K3efLVPSTaUPvBGIZgnn2JBFFWaMsKZ +guUuUnz6qaSGqnmGAYiupdC1EFye58V4CLbWVjJU4NF2jrOUYR/Dv04zYwVQtQcFzgmK6H4NHAhm +b0a6pQRKxZaZ+x2vCC7sCuIu4dNCATwqzk12ue6oEsxzYybLPNGJd084M43O9W8E+5/drd/FQVB2 +X4jlFlCuHuWeQxQo+w73Tb9swW692v3BlfQTP1ClWzuJ+RwuSb9m4V3QVa4MEL+0Xzc5FX9P+YX1 +cgaL+6oMHw7L+IOjOt+n1BOloyqk35lLHX7RZmu8SckMnGP95XjWc4FRKP9x/iXrKaeCnut/zsty +ZdJS5FRmBT/wb5KK9YWBGnqPLO8isN2HS8gA + + + +HTTP/1.1 200 OK +Date: Mon, 29 Nov 2004 21:56:53 GMT +Server: Apache/1.3.31 (Debian GNU/Linux) mod_gzip/1.3.26.1a PHP/4.3.9-1 mod_ssl/2.8.20 OpenSSL/0.9.7d mod_perl/1.29 +Vary: Accept-Encoding +Content-Type: text/html; charset=ISO-8859-1 +Content-Encoding: br +Content-Length: 1056 + + + + + +# +# Client-side + + +brotli + + +http + + +HTTP GET brotli compressed content with broken header + + +http://%HOSTIP:%HTTPPORT/315 --compressed + + + +# +# Verify data after the test has been "shot" + + +^User-Agent:.* + + +s/^Accept-Encoding: .*/Accept-Encoding: xxx/ + + +GET /315 HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +Accept: */* +Accept-Encoding: xxx + + + +61 + + + diff --git a/tests/runtests.pl b/tests/runtests.pl index 7f6f81291..dcd9f8419 100755 --- a/tests/runtests.pl +++ b/tests/runtests.pl @@ -217,6 +217,7 @@ my $gopher_ipv6; # set if Gopher server has IPv6 support my $has_ipv6; # set if libcurl is built with IPv6 support my $has_unix; # set if libcurl is built with Unix sockets support my $has_libz; # set if libcurl is built with libz support +my $has_brotli; # set if libcurl is built with brotli support my $has_getrlimit; # set if system has getrlimit() my $has_ntlm; # set if libcurl is built with NTLM support my $has_ntlm_wb; # set if libcurl is built with NTLM delegation to winbind @@ -2880,6 +2881,9 @@ sub checksystem { if($feat =~ /libz/i) { $has_libz = 1; } + if($feat =~ /brotli/i) { + $has_brotli = 1; + } if($feat =~ /NTLM/i) { # NTLM enabled $has_ntlm=1; @@ -3396,6 +3400,11 @@ sub singletest { next; } } + elsif($1 eq "brotli") { + if($has_brotli) { + next; + } + } elsif($1 eq "NTLM") { if($has_ntlm) { next; @@ -3552,6 +3561,11 @@ sub singletest { next; } } + elsif($1 eq "brotli") { + if(!$has_brotli) { + next; + } + } elsif($1 eq "NTLM") { if(!$has_ntlm) { next;