HTTP: implement Brotli content encoding

This uses the brotli external library (https://github.com/google/brotli).
Brotli becomes a feature: additional curl_version_info() bit and
structure fields are provided for it and CURLVERSION_NOW bumped.

Tests 314 and 315 check Brotli content unencoding with correct and
erroneous data.

Some tests are updated to accomodate with the now configuration dependent
parameters of the Accept-Encoding header.

configure.ac

@@ -148,6 +148,7 @@ dnl initialize all the info variables
     curl_ssl_msg="no      (--with-{ssl,gnutls,nss,polarssl,mbedtls,cyassl,axtls,winssl,darwinssl} )"
     curl_ssh_msg="no      (--with-libssh2)"
    curl_zlib_msg="no      (--with-zlib)"
+ curl_brotli_msg="no      (--with-brotli)"
     curl_gss_msg="no      (--with-gssapi)"
 curl_tls_srp_msg="no      (--enable-tls-srp)"
     curl_res_msg="default (--enable-ares / --enable-threaded-resolver)"
@@ -975,6 +976,94 @@ dnl set variable for use in automakefile(s)
 AM_CONDITIONAL(HAVE_LIBZ, test x"$AMFIXLIB" = x1)
 AC_SUBST(ZLIB_LIBS)
 
+dnl **********************************************************************
+dnl Check for the presence of BROTLI decoder libraries and headers
+dnl **********************************************************************
+
+dnl Brotli project home page: https://github.com/google/brotli
+
+dnl Default to compiler & linker defaults for BROTLI files & libraries.
+OPT_BROTLI=off
+AC_ARG_WITH(brotli,dnl
+AC_HELP_STRING([--with-brotli=PATH],[Where to look for brotli, PATH points to the BROTLI installation; when possible, set the PKG_CONFIG_PATH environment variable instead of using this option])
+AC_HELP_STRING([--without-brotli], [disable BROTLI]),
+  OPT_BROTLI=$withval)
+
+if test X"$OPT_BROTLI" != Xno; then
+  dnl backup the pre-brotli variables
+  CLEANLDFLAGS="$LDFLAGS"
+  CLEANCPPFLAGS="$CPPFLAGS"
+  CLEANLIBS="$LIBS"
+
+  case "$OPT_BROTLI" in
+  yes)
+    dnl --with-brotli (without path) used
+    CURL_CHECK_PKGCONFIG(libbrotlidec)
+
+    if test "$PKGCONFIG" != "no" ; then
+      LIB_BROTLI=`$PKGCONFIG --libs-only-l libbrotlidec`
+      LD_BROTLI=`$PKGCONFIG --libs-only-L libbrotlidec`
+      CPP_BROTLI=`$PKGCONFIG --cflags-only-I libbrotlidec`
+      version=`$PKGCONFIG --modversion libbrotlidec`
+      DIR_BROTLI=`echo $LD_BROTLI | $SED -e 's/-L//'`
+    fi
+
+    ;;
+  off)
+    dnl no --with-brotli option given, just check default places
+    ;;
+  *)
+    dnl use the given --with-brotli spot
+    PREFIX_BROTLI=$OPT_BROTLI
+    ;;
+  esac
+
+  dnl if given with a prefix, we set -L and -I based on that
+  if test -n "$PREFIX_BROTLI"; then
+    LIB_BROTLI="-lbrotlidec"
+    LD_BROTLI=-L${PREFIX_BROTLI}/lib$libsuff
+    CPP_BROTLI=-I${PREFIX_BROTLI}/include
+    DIR_BROTLI=${PREFIX_BROTLI}/lib$libsuff
+  fi
+
+  LDFLAGS="$LDFLAGS $LD_BROTLI"
+  CPPFLAGS="$CPPFLAGS $CPP_BROTLI"
+  LIBS="$LIB_BROTLI $LIBS"
+
+  AC_CHECK_LIB(brotlidec, BrotliDecoderDecompress)
+
+  AC_CHECK_HEADERS(brotli/decode.h,
+    curl_brotli_msg="enabled (libbrotlidec)"
+    HAVE_BROTLI=1
+    AC_DEFINE(HAVE_BROTLI, 1, [if BROTLI is in use])
+    AC_SUBST(HAVE_BROTLI, [1])
+  )
+
+  if test X"$OPT_BROTLI" != Xoff &&
+     test "$HAVE_BROTLI" != "1"; then
+    AC_MSG_ERROR([BROTLI libs and/or directories were not found where specified!])
+  fi
+
+  if test "$HAVE_BROTLI" = "1"; then
+    if test -n "$DIR_BROTLI"; then
+       dnl when the brotli shared libs were found in a path that the run-time
+       dnl linker doesn't search through, we need to add it to LD_LIBRARY_PATH
+       dnl to prevent further configure tests to fail due to this
+
+       if test "x$cross_compiling" != "xyes"; then
+         LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$DIR_BROTLI"
+         export LD_LIBRARY_PATH
+         AC_MSG_NOTICE([Added $DIR_BROTLI to LD_LIBRARY_PATH])
+       fi
+    fi
+  else
+    dnl no brotli, revert back to clean variables
+    LDFLAGS=$CLEANLDFLAGS
+    CPPFLAGS=$CLEANCPPFLAGS
+    LIBS=$CLEANLIBS
+  fi
+fi
+
 dnl **********************************************************************
 dnl Check for LDAP
 dnl **********************************************************************
@@ -3786,6 +3875,9 @@ fi
 if test "x$HAVE_LIBZ" = "x1"; then
   SUPPORT_FEATURES="$SUPPORT_FEATURES libz"
 fi
+if test "x$HAVE_BROTLI" = "x1"; then
+  SUPPORT_FEATURES="$SUPPORT_FEATURES brotli"
+fi
 if test "x$USE_ARES" = "x1" -o "x$USE_THREADS_POSIX" = "x1" \
                             -o "x$USE_THREADS_WIN32" = "x1"; then
   SUPPORT_FEATURES="$SUPPORT_FEATURES AsynchDNS"
@@ -4003,6 +4095,7 @@ AC_MSG_NOTICE([Configured to build curl/libcurl:
   SSL support:      ${curl_ssl_msg}
   SSH support:      ${curl_ssh_msg}
   zlib support:     ${curl_zlib_msg}
+  brotli support:   ${curl_brotli_msg}
   GSS-API support:  ${curl_gss_msg}
   TLS-SRP support:  ${curl_tls_srp_msg}
   resolver:         ${curl_res_msg}

docs/INTERNALS.md

@@ -644,9 +644,9 @@ Content Encoding
  [HTTP/1.1][4] specifies that a client may request that a server encode its
  response. This is usually used to compress a response using one (or more)
  encodings from a set of commonly available compression techniques. These
- schemes include 'deflate' (the zlib algorithm), 'gzip' and 'compress'. A
+ schemes include 'deflate' (the zlib algorithm), 'gzip' 'br' (brotli) and
- client requests that the server perform an encoding by including an
+ 'compress'. A client requests that the server perform an encoding by including
- Accept-Encoding header in the request document. The value of the header
+ an Accept-Encoding header in the request document. The value of the header
  should be one of the recognized tokens 'deflate', ... (there's a way to
  register new schemes/tokens, see sec 3.5 of the spec). A server MAY honor
  the client's encoding request. When a response is encoded, the server
@@ -661,9 +661,10 @@ Content Encoding
 
 ## Supported content encodings
 
- The 'deflate' and 'gzip' content encodings are supported by libcurl. Both
+ The 'deflate', 'gzip' and 'br' content encodings are supported by libcurl.
- regular and chunked transfers work fine.  The zlib library is required for
+ Both regular and chunked transfers work fine.  The zlib library is required
- this feature.
+ for the 'deflate' and 'gzip' encodings, while the brotli decoding library is
+ for the 'br' encoding.
 
 ## The libcurl interface
 
@@ -674,10 +675,10 @@ Content Encoding
  where string is the intended value of the Accept-Encoding header.
 
  Currently, libcurl does support multiple encodings but only
- understands how to process responses that use the "deflate" or "gzip"
+ understands how to process responses that use the "deflate", "gzip" and/or
- Content-Encoding, so the only values for [`CURLOPT_ACCEPT_ENCODING`][5]
+ "br" content encodings, so the only values for [`CURLOPT_ACCEPT_ENCODING`][5]
- that will work (besides "identity," which does nothing) are "deflate"
+ that will work (besides "identity," which does nothing) are "deflate",
- and "gzip". If a response is encoded using the "compress" or methods,
+ "gzip" and "br". If a response is encoded using the "compress" or methods,
  libcurl will return an error indicating that the response could
  not be decoded.  If <string> is NULL no Accept-Encoding header is generated.
  If <string> is a zero-length string, then an Accept-Encoding header

docs/RESOURCES

@@ -81,3 +81,5 @@ This document lists documents and standards used by curl.
   RFC 4616 - PLAIN authentication
 
   RFC 4954 - SMTP Authentication
+
+  RFC 7932 - Brotli Compressed Data Format

docs/TODO

@@ -64,9 +64,8 @@
  5.4 HTTP Digest using SHA-256
  5.5 auth= in URLs
  5.6 Refuse "downgrade" redirects
- 5.7 Brotli compression
+ 5.7 QUIC
- 5.8 QUIC
+ 5.8 Leave secure cookies alone
- 5.9 Leave secure cookies alone
 
  6. TELNET
  6.1 ditch stdin
@@ -514,13 +513,7 @@ This is not detailed in any FTP specification.
  Consider a way to tell curl to refuse to "downgrade" protocol with a redirect
  and/or possibly a bit that refuses redirect to change protocol completely.
 
-5.7 Brotli compression
+5.7 QUIC
-
- Brotli compression performs better than gzip and is being implemented by
- browsers and servers widely. The algorithm: https://github.com/google/brotli
- The Firefox bug: https://bugzilla.mozilla.org/show_bug.cgi?id=366559
-
-5.8 QUIC
 
  The standardization process of QUIC has been taken to the IETF and can be
  followed on the [IETF QUIC Mailing
@@ -530,7 +523,7 @@ This is not detailed in any FTP specification.
  implemented. This, to allow other projects to benefit from the work and to
  thus broaden the interest and chance of others to participate.
 
-5.9 Leave secure cookies alone
+5.8 Leave secure cookies alone
 
  Non-secure origins (HTTP sites) should not be allowed to set or modify
  cookies with the 'secure' property:

docs/libcurl/curl_version_info.3

@@ -72,6 +72,12 @@ typedef struct {
 
   const char *libssh_version; /* human readable string */
 
+  /* when 'age' is 4 or higher (7.57.0 or later), the members below also
+     exist  */
+  unsigned int brotli_ver_num; /* Numeric Brotli version
+                                  (MAJOR << 24) | (MINOR << 12) | PATCH */
+  const char *brotli_version; /* human readable string. */
+
 } curl_version_info_data;
 .fi
 
@@ -160,6 +166,8 @@ libcurl was built with support for HTTPS-proxy.
 libcurl was built with multiple SSL backends. For details, see
 \fIcurl_global_sslset(3)\fP.
 (Added in 7.56.0)
+.IP CURL_VERSION_BROTLI
+supports HTTP Brotli content encoding using libbrotlidec (Added in 7.57.0)
 .RE
 \fIssl_version\fP is an ASCII string for the OpenSSL version used. If libcurl
 has no SSL support, this is NULL.

docs/libcurl/symbols-in-versions

@@ -699,6 +699,7 @@ CURLUSESSL_ALL                  7.17.0
 CURLUSESSL_CONTROL              7.17.0
 CURLUSESSL_NONE                 7.17.0
 CURLUSESSL_TRY                  7.17.0
+CURLVERSION_FIFTH               7.57.0
 CURLVERSION_FIRST               7.10
 CURLVERSION_FOURTH              7.16.1
 CURLVERSION_NOW                 7.10
@@ -829,6 +830,7 @@ CURL_TIMECOND_NONE              7.9.7
 CURL_TLSAUTH_NONE               7.21.4
 CURL_TLSAUTH_SRP                7.21.4
 CURL_VERSION_ASYNCHDNS          7.10.7
+CURL_VERSION_BROTLI             7.57.0
 CURL_VERSION_CONV               7.15.4
 CURL_VERSION_CURLDEBUG          7.19.6
 CURL_VERSION_DEBUG              7.10.6

include/curl/curl.h

@@ -2592,6 +2592,7 @@ typedef enum {
   CURLVERSION_SECOND,
   CURLVERSION_THIRD,
   CURLVERSION_FOURTH,
+  CURLVERSION_FIFTH,
   CURLVERSION_LAST /* never actually use this */
 } CURLversion;
 
@@ -2600,7 +2601,7 @@ typedef enum {
    meant to be a built-in version number for what kind of struct the caller
    expects. If the struct ever changes, we redefine the NOW to another enum
    from above. */
-#define CURLVERSION_NOW CURLVERSION_FOURTH
+#define CURLVERSION_NOW CURLVERSION_FIFTH
 
 typedef struct {
   CURLversion age;          /* age of the returned struct */
@@ -2628,6 +2629,12 @@ typedef struct {
 
   const char *libssh_version; /* human readable string */
 
+  /* These fields were added in CURLVERSION_FIFTH */
+
+  unsigned int brotli_ver_num; /* Numeric Brotli version
+                                  (MAJOR << 24) | (MINOR << 12) | PATCH */
+  const char *brotli_version; /* human readable string. */
+
 } curl_version_info_data;
 
 #define CURL_VERSION_IPV6         (1<<0)  /* IPv6-enabled */
@@ -2658,6 +2665,7 @@ typedef struct {
                                              for cookie domain verification */
 #define CURL_VERSION_HTTPS_PROXY  (1<<21) /* HTTPS-proxy support built-in */
 #define CURL_VERSION_MULTI_SSL    (1<<22) /* Multiple SSL backends available */
+#define CURL_VERSION_BROTLI       (1<<23) /* Brotli features are present. */
 
  /*
  * NAME curl_version_info()

lib/content_encoding.c

@@ -37,14 +37,15 @@
 
 #ifndef CURL_DISABLE_HTTP
 
+#define DSIZ CURL_MAX_WRITE_SIZE /* buffer size for decompressed data */
+
+
 #ifdef HAVE_LIBZ
 
 /* Comment this out if zlib is always going to be at least ver. 1.2.0.4
    (doing so will reduce code size slightly). */
 #define OLD_ZLIB_SUPPORT 1
 
-#define DSIZ CURL_MAX_WRITE_SIZE /* buffer size for decompressed data */
-
 #define GZIP_MAGIC_0 0x1f
 #define GZIP_MAGIC_1 0x8b
 
@@ -505,6 +506,136 @@ static const content_encoding gzip_encoding = {
 #endif /* HAVE_LIBZ */
 
 
+#ifdef HAVE_BROTLI
+
+/* Writer parameters. */
+typedef struct {
+  BrotliDecoderState *br;    /* State structure for brotli. */
+}  brotli_params;
+
+
+static CURLcode brotli_map_error(BrotliDecoderErrorCode be)
+{
+  switch(be) {
+  case BROTLI_DECODER_ERROR_FORMAT_EXUBERANT_NIBBLE:
+  case BROTLI_DECODER_ERROR_FORMAT_EXUBERANT_META_NIBBLE:
+  case BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_ALPHABET:
+  case BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_SAME:
+  case BROTLI_DECODER_ERROR_FORMAT_CL_SPACE:
+  case BROTLI_DECODER_ERROR_FORMAT_HUFFMAN_SPACE:
+  case BROTLI_DECODER_ERROR_FORMAT_CONTEXT_MAP_REPEAT:
+  case BROTLI_DECODER_ERROR_FORMAT_BLOCK_LENGTH_1:
+  case BROTLI_DECODER_ERROR_FORMAT_BLOCK_LENGTH_2:
+  case BROTLI_DECODER_ERROR_FORMAT_TRANSFORM:
+  case BROTLI_DECODER_ERROR_FORMAT_DICTIONARY:
+  case BROTLI_DECODER_ERROR_FORMAT_WINDOW_BITS:
+  case BROTLI_DECODER_ERROR_FORMAT_PADDING_1:
+  case BROTLI_DECODER_ERROR_FORMAT_PADDING_2:
+  case BROTLI_DECODER_ERROR_COMPOUND_DICTIONARY:
+  case BROTLI_DECODER_ERROR_DICTIONARY_NOT_SET:
+  case BROTLI_DECODER_ERROR_INVALID_ARGUMENTS:
+    return CURLE_BAD_CONTENT_ENCODING;
+  case BROTLI_DECODER_ERROR_ALLOC_CONTEXT_MODES:
+  case BROTLI_DECODER_ERROR_ALLOC_TREE_GROUPS:
+  case BROTLI_DECODER_ERROR_ALLOC_CONTEXT_MAP:
+  case BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_1:
+  case BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_2:
+  case BROTLI_DECODER_ERROR_ALLOC_BLOCK_TYPE_TREES:
+    return CURLE_OUT_OF_MEMORY;
+  default:
+    break;
+  }
+  return CURLE_WRITE_ERROR;
+}
+
+static CURLcode brotli_init_writer(struct connectdata *conn,
+                                   contenc_writer *writer)
+{
+  brotli_params *bp = (brotli_params *) &writer->params;
+
+  (void) conn;
+
+  if(!writer->downstream)
+    return CURLE_WRITE_ERROR;
+
+  bp->br = BrotliDecoderCreateInstance(NULL, NULL, NULL);
+  return bp->br? CURLE_OK: CURLE_OUT_OF_MEMORY;
+}
+
+static CURLcode brotli_unencode_write(struct connectdata *conn,
+                                      contenc_writer *writer,
+                                      const char *buf, size_t nbytes)
+{
+  brotli_params *bp = (brotli_params *) &writer->params;
+  const uint8_t *src = (const uint8_t *) buf;
+  char *decomp;
+  uint8_t *dst;
+  size_t dstleft;
+  CURLcode result = CURLE_OK;
+
+  if(!nbytes)
+    return CURLE_OK;
+  if(!bp->br)
+    return CURLE_WRITE_ERROR;  /* Stream already ended. */
+
+  decomp = malloc(DSIZ);
+  if(!decomp)
+    return CURLE_OUT_OF_MEMORY;
+
+  while(nbytes && result == CURLE_OK) {
+    BrotliDecoderResult r;
+
+    dst = (uint8_t *) decomp;
+    dstleft = DSIZ;
+    r = BrotliDecoderDecompressStream(bp->br,
+                                      &nbytes, &src, &dstleft, &dst, NULL);
+    result = Curl_unencode_write(conn, writer->downstream,
+                                 decomp, DSIZ - dstleft);
+    if(result)
+      break;
+    switch(r) {
+    case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
+    case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
+      break;
+    case BROTLI_DECODER_RESULT_SUCCESS:
+      BrotliDecoderDestroyInstance(bp->br);
+      bp->br = NULL;
+      if(nbytes)
+        result = CURLE_WRITE_ERROR;
+      break;
+    default:
+      result = brotli_map_error(BrotliDecoderGetErrorCode(bp->br));
+      break;
+    }
+  }
+  free(decomp);
+  return result;
+}
+
+static void brotli_close_writer(struct connectdata *conn,
+                                contenc_writer *writer)
+{
+  brotli_params *bp = (brotli_params *) &writer->params;
+
+  (void) conn;
+
+  if(bp->br) {
+    BrotliDecoderDestroyInstance(bp->br);
+    bp->br = NULL;
+  }
+}
+
+static const content_encoding brotli_encoding = {
+  "br",
+  NULL,
+  brotli_init_writer,
+  brotli_unencode_write,
+  brotli_close_writer,
+  sizeof(brotli_params)
+};
+#endif
+
+
 /* Identity handler. */
 static CURLcode identity_init_writer(struct connectdata *conn,
                                      contenc_writer *writer)
@@ -543,6 +674,9 @@ static const content_encoding * const encodings[] = {
 #ifdef HAVE_LIBZ
   &deflate_encoding,
   &gzip_encoding,
+#endif
+#ifdef HAVE_BROTLI
+  &brotli_encoding,
 #endif
   NULL
 };

lib/urldata.h

@@ -96,6 +96,10 @@
 #endif
 #endif
 
+#ifdef HAVE_BROTLI
+#include <brotli/decode.h>
+#endif
+
 #include <curl/curl.h>
 
 #include "http_chunks.h" /* for the structs and enum stuff */

lib/version.c

@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -74,6 +74,18 @@ void Curl_version_init(void)
   curl_version_info(CURLVERSION_NOW);
 }
 
+#ifdef HAVE_BROTLI
+static size_t brotli_version(char *buf, size_t bufsz)
+{
+  uint32_t brotli_version = BrotliDecoderVersion();
+  unsigned int major = brotli_version >> 24;
+  unsigned int minor = (brotli_version & 0x00FFFFFF) >> 12;
+  unsigned int patch = brotli_version & 0x00000FFF;
+
+  return snprintf(buf, bufsz, "%u.%u.%u", major, minor, patch);
+}
+#endif
+
 char *curl_version(void)
 {
   static bool initialized;
@@ -105,6 +117,14 @@ char *curl_version(void)
   left -= len;
   ptr += len;
 #endif
+#ifdef HAVE_BROTLI
+  len = snprintf(ptr, left, "%s", " brotli/");
+  left -= len;
+  ptr += len;
+  len = brotli_version(ptr, left);
+  left -= len;
+  ptr += len;
+#endif
 #ifdef USE_ARES
   /* this function is only present in c-ares, not in the original ares */
   len = snprintf(ptr, left, " c-ares/%s", ares_version(NULL));
@@ -326,6 +346,9 @@ static curl_version_info_data version_info = {
 #endif
 #if defined(CURL_WITH_MULTI_SSL)
   | CURL_VERSION_MULTI_SSL
+#endif
+#if defined(HAVE_BROTLI)
+  | CURL_VERSION_BROTLI
 #endif
   ,
   NULL, /* ssl_version */
@@ -337,6 +360,8 @@ static curl_version_info_data version_info = {
   NULL, /* libidn version */
   0,    /* iconv version */
   NULL, /* ssh lib version */
+  0,    /* brotli_ver_num */
+  NULL, /* brotli version */
 };
 
 curl_version_info_data *curl_version_info(CURLversion stamp)
@@ -348,6 +373,9 @@ curl_version_info_data *curl_version_info(CURLversion stamp)
 #ifdef USE_SSL
   static char ssl_buffer[80];
 #endif
+#ifdef HAVE_BROTLI
+  static char brotli_buffer[80];
+#endif
 
   if(initialized)
     return &version_info;
@@ -396,6 +424,12 @@ curl_version_info_data *curl_version_info(CURLversion stamp)
   version_info.libssh_version = ssh_buffer;
 #endif
 
+#ifdef HAVE_BROTLI
+  version_info.brotli_ver_num = BrotliDecoderVersion();
+  brotli_version(brotli_buffer, sizeof brotli_buffer);
+  version_info.brotli_version = brotli_buffer;
+#endif
+
   (void)stamp; /* avoid compiler warnings, we don't use this */
 
   initialized = true;

src/tool_help.c

@@ -499,6 +499,7 @@ static const struct feat feats[] = {
   {"NTLM_WB",        CURL_VERSION_NTLM_WB},
   {"SSL",            CURL_VERSION_SSL},
   {"libz",           CURL_VERSION_LIBZ},
+  {"brotli",         CURL_VERSION_BROTLI},
   {"CharConv",       CURL_VERSION_CONV},
   {"TLS-SRP",        CURL_VERSION_TLSAUTH_SRP},
   {"HTTP2",          CURL_VERSION_HTTP2},

tests/data/Makefile.inc

@@ -54,7 +54,7 @@ test271 test272 test273 test274 test275 test276 test277 test278 test279 \
 test280 test281 test282 test283 test284 test285 test286 test287 test288 \
 test289 test290 test291 test292 test293 test294 test295 test296 test297 \
 test298 test299 test300 test301 test302 test303 test304 test305 test306 \
-test307 test308 test309 test310 test311 test312 test313                 \
+test307 test308 test309 test310 test311 test312 test313 test314 test315 \
                                 test320 test321 test322 test323 test324 \
 test325 \
 test350 test351 test352 test353 test354 \

tests/data/test220

@@ -57,11 +57,14 @@ http://%HOSTIP:%HTTPPORT/220 --compressed
 <strip>
 ^User-Agent:.*
 </strip>
+<strippart>
+s/^Accept-Encoding: .*/Accept-Encoding: xxx/
+</strippart>
 <protocol>
 GET /220 HTTP/1.1
 Host: %HOSTIP:%HTTPPORT
 Accept: */*
-Accept-Encoding: deflate, gzip
+Accept-Encoding: xxx
 
 </protocol>
 </verify>

tests/data/test221

@@ -57,11 +57,14 @@ http://%HOSTIP:%HTTPPORT/221 --compressed
 <strip>
 ^User-Agent:.*
 </strip>
+<strippart>
+s/^Accept-Encoding: .*/Accept-Encoding: xxx/
+</strippart>
 <protocol>
 GET /221 HTTP/1.1
 Host: %HOSTIP:%HTTPPORT
 Accept: */*
-Accept-Encoding: deflate, gzip
+Accept-Encoding: xxx
 
 </protocol>
 <errorcode>

tests/data/test222

@@ -188,11 +188,14 @@ http://%HOSTIP:%HTTPPORT/222 --compressed
 <strip>
 ^User-Agent:.*
 </strip>
+<strippart>
+s/^Accept-Encoding: .*/Accept-Encoding: xxx/
+</strippart>
 <protocol>
 GET /222 HTTP/1.1
 Host: %HOSTIP:%HTTPPORT
 Accept: */*
-Accept-Encoding: deflate, gzip
+Accept-Encoding: xxx
 
 </protocol>
 </verify>

tests/data/test223

@@ -78,11 +78,14 @@ http://%HOSTIP:%HTTPPORT/223 --compressed
 <strip>
 ^User-Agent:.*
 </strip>
+<strippart>
+s/^Accept-Encoding: .*/Accept-Encoding: xxx/
+</strippart>
 <protocol>
 GET /223 HTTP/1.1
 Host: %HOSTIP:%HTTPPORT
 Accept: */*
-Accept-Encoding: deflate, gzip
+Accept-Encoding: xxx
 
 </protocol>
 <errorcode>

tests/data/test224

@@ -93,11 +93,14 @@ http://%HOSTIP:%HTTPPORT/224 --compressed
 <strip>
 ^User-Agent:.*
 </strip>
+<strippart>
+s/^Accept-Encoding: .*/Accept-Encoding: xxx/
+</strippart>
 <protocol>
 GET /224 HTTP/1.1
 Host: %HOSTIP:%HTTPPORT
 Accept: */*
-Accept-Encoding: deflate, gzip
+Accept-Encoding: xxx
 
 </protocol>
 </verify>

tests/data/test230

@@ -189,11 +189,14 @@ http://%HOSTIP:%HTTPPORT/230 --compressed
 <strip>
 ^User-Agent:.*
 </strip>
+<strippart>
+s/^Accept-Encoding: .*/Accept-Encoding: xxx/
+</strippart>
 <protocol>
 GET /230 HTTP/1.1
 Host: %HOSTIP:%HTTPPORT
 Accept: */*
-Accept-Encoding: deflate, gzip
+Accept-Encoding: xxx
 
 </protocol>
 </verify>

tests/data/test314

@@ -0,0 +1,198 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+compressed
+</keywords>
+</info>
+#
+# Server-side
+<reply>
+<data base64="yes">
+SFRUUC8xLjEgMjAwIE9LDQpEYXRlOiBNb24sIDI5IE5vdiAyMDA0IDIxOjU2OjUzIEdNVA0KU2Vy
+dmVyOiBBcGFjaGUvMS4zLjMxIChEZWJpYW4gR05VL0xpbnV4KSBtb2RfZ3ppcC8xLjMuMjYuMWEg
+UEhQLzQuMy45LTEgbW9kX3NzbC8yLjguMjAgT3BlblNTTC8wLjkuN2QgbW9kX3BlcmwvMS4yOQ0K
+VmFyeTogQWNjZXB0LUVuY29kaW5nDQpDb250ZW50LVR5cGU6IHRleHQvaHRtbDsgY2hhcnNldD1J
+U08tODg1OS0xDQpDb250ZW50LUVuY29kaW5nOiBicg0KQ29udGVudC1MZW5ndGg6IDEwNTYNCg0K
+G7ATAJwFdhtdgaQ8i+mZBoO/lwogPKuqHpeP38jV5TDITTB7/oJVCS69FFDKWDVtMk8y4SfMSu/a
+9vvLxWPweDCKePH/2y9VIkbF+EgCYSNs9v53J8QTIHT4ZucHCCRQiXRdT6XdE60KSlbIvobr5rJQ
+sRn7ipIjMVMq3Go+/UXtY2d0yP1qaaGSxCn8nZuUNGh74KOI7EEkgFl1tjYytkpc9mJJy9J+wTTI
++HroUQP2VR2DYkNoUECqgtOLlGcVEln4+eVzEWcrb8fNrcrVxLArJBpSd8FX8eZs8ebJUO7aBZ5e
+pHz6zel7lhLlfHoQIkGh34riaSVr7VTGDmmO6HjSCzKO27LybZ9I3CtMSD2Il4mB131Tlcbut1Bd
+zL4XU4DZYMLBN4jwVZEoHpjzHX+vQ3prnrNw4oB7OWOr/fBzjvfjDuO24WxwzPqPo+V6VNcthz1p
+fF1+sMK4yWY7He33m32EuQgQFSZ3a5Wu4FyQcAb45Z+wUxM5XCmX52YmdUR2YTs+W+bNw2EZSfMR
+cP3CinyJI/cTT+JubL3T4COkhz0Rffeoh/3E4c/6ugma1ubhokYecXp8HBwmeDL48d62H26u69DO
+yMhg1PFj+oVDWnK4K+L5AlRr0mpJLqoGHrzflMLQ6qL2oIo9hN6qCeZEEqXM+/KunVYpWVeTY+ht
+hA0y5p5RLLTTS4cehaJOpbFyAVxZOardIkJAVx0NshOZY4hDbts9BXsXzFEOgsFhrIQYgh04StZz
+llIRMVDptYlwGmpZCHHmVECdGiFIfEhkQ2INSwMCuuKpaycgSOO9hJA9UFKDBdzTiLJBP9oUVkKL
+bHjwicICCi3k0HcppcvQaW27AMI06kuQU4WUGizgnkaUDcZqCgsotMgG528UFlBo8SFpb05OAjJq
+2gEI0UgN93KS1OvAOYSLN5IaLOCeRnQpJXuLUwcm7urpg6lYxAk26uEoADdsRytHGkSWjOKP6T07
+wiceuNo7CXyu7ohtUZXoEWawRHGVkPDVJYqH+xa0DDRKSSgM4K3efLVPSTaUPvBGIZgnn2JBFFWa
+MsKZguUuUnz6qaSGqnmGAYiupdC1EFye58V4CLbWVjJU4NF2jrOUYR/Dv04zYwVQtQcFzgmK6H4N
+HAhmb0a6pQRKxZaZ+x2vCC7sCuIu4dNCATwqzk12ue6oEsxzYybLPNGJd084M43O9W8E+5/drd/F
+QVB2X4jlFlCuHuWeQxQo+w73Tb9swW692v3BlfQTP1ClWzuJ+RwuSb9m4V3QVa4MEL+0Xzc5FX9P
++YX1cgaL+6oMHw7L+IOjOt+n1BOloyqk35lLHX7RZmu8SckMnGP95XjWc4FRKP9x/iXrKaeCnut/
+zstyZdJS5FRmBT/wb5KK9YWBGnqPLO8isN2HS8gA
+</data>
+
+<datacheck>
+HTTP/1.1 200 OK
+Date: Mon, 29 Nov 2004 21:56:53 GMT
+Server: Apache/1.3.31 (Debian GNU/Linux) mod_gzip/1.3.26.1a PHP/4.3.9-1 mod_ssl/2.8.20 OpenSSL/0.9.7d mod_perl/1.29
+Vary: Accept-Encoding
+Content-Type: text/html; charset=ISO-8859-1
+Content-Encoding: br
+Content-Length: 1056
+
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<!DOCTYPE project-listing SYSTEM "http://freshmeat.net/backend/fm-projects-0.4.dtd">
+<project-listing>
+  <project>
+    <project_id>1612</project_id>
+    <date_added>1998-08-21 04:01:29</date_added>
+    <date_updated>2004-10-18 02:22:23</date_updated>
+    <projectname_short>curl</projectname_short>
+    <projectname_full>curl and libcurl</projectname_full>
+    <desc_short>Command line tool and library for client-side URL transfers.</desc_short>
+    <desc_full>curl and libcurl is a tool for transferring files
+using URL syntax. It supports HTTP, HTTPS, FTP,
+FTPS, DICT, TELNET, LDAP, FILE, and GOPHER, as
+well as HTTP-post, HTTP-put, cookies, FTP upload,
+resumed transfers, passwords, portnumbers, SSL
+certificates, Kerberos, and proxies. It is powered
+by libcurl, the client-side URL transfer library.
+There are bindings to libcurl for over 20
+languages and environments.
+</desc_full>
+    <vitality_score>5784.57</vitality_score>
+    <vitality_percent>3.16</vitality_percent>
+    <vitality_rank>169</vitality_rank>
+    <popularity_score>6594.54</popularity_score>
+    <popularity_percent>13.81</popularity_percent>
+    <popularity_rank>105</popularity_rank>
+    <rating>8.50</rating>
+    <rating_count>21</rating_count>
+    <rating_rank>183</rating_rank>
+    <subscriptions>323</subscriptions>
+    <branch_name>Default</branch_name>
+    <url_project_page>http://freshmeat.net/projects/curl/</url_project_page>
+    <url_homepage>http://freshmeat.net/redir/curl/1612/url_homepage/</url_homepage>
+    <url_tgz>http://freshmeat.net/redir/curl/1612/url_tgz/</url_tgz>
+    <url_bz2>http://freshmeat.net/redir/curl/1612/url_bz2/</url_bz2>
+    <url_zip>http://freshmeat.net/redir/curl/1612/url_zip/</url_zip>
+    <url_changelog>http://freshmeat.net/redir/curl/1612/url_changelog/</url_changelog>
+    <url_rpm>http://freshmeat.net/redir/curl/1612/url_rpm/</url_rpm>
+    <url_deb>http://freshmeat.net/redir/curl/1612/url_deb/</url_deb>
+    <url_osx>http://freshmeat.net/redir/curl/1612/url_osx/</url_osx>
+    <url_bsdport>http://freshmeat.net/redir/curl/1612/url_bsdport/</url_bsdport>
+    <url_purchase></url_purchase>
+    <url_cvs>http://freshmeat.net/redir/curl/1612/url_cvs/</url_cvs>
+    <url_list>http://freshmeat.net/redir/curl/1612/url_list/</url_list>
+    <url_mirror>http://freshmeat.net/redir/curl/1612/url_mirror/</url_mirror>
+    <url_demo></url_demo>
+    <license>MIT/X Consortium License</license>
+    <latest_release>
+      <latest_release_version>7.12.2</latest_release_version>
+      <latest_release_id>176085</latest_release_id>
+      <latest_release_date>2004-10-18 02:22:23</latest_release_date>
+    </latest_release>
+    <screenshot_thumb></screenshot_thumb>
+    <authors>
+      <author>
+        <author_name>Daniel Stenberg</author_name>
+        <author_url>http://freshmeat.net/~bagder/</author_url>
+        <author_role>Owner</author_role>
+      </author>
+    </authors>
+    <descriminators>
+      <trove_id>12</trove_id>
+      <trove_id>226</trove_id>
+      <trove_id>3</trove_id>
+      <trove_id>2</trove_id>
+      <trove_id>188</trove_id>
+      <trove_id>216</trove_id>
+      <trove_id>200</trove_id>
+      <trove_id>220</trove_id>
+      <trove_id>164</trove_id>
+      <trove_id>90</trove_id>
+      <trove_id>89</trove_id>
+      <trove_id>809</trove_id>
+      <trove_id>150</trove_id>
+      <trove_id>224</trove_id>
+      <trove_id>900</trove_id>
+      <trove_id>839</trove_id>
+    </descriminators>
+    <dependencies>
+      <dependency type="recommended">
+        <dependency_release_id>0</dependency_release_id>
+        <dependency_branch_id>7464</dependency_branch_id>
+        <dependency_project_id>7464</dependency_project_id>
+        <dependency_project_title>OpenSSL (Default)</dependency_project_title>
+      </dependency>
+      <dependency type="optional">
+        <dependency_release_id>0</dependency_release_id>
+        <dependency_branch_id>0</dependency_branch_id>
+        <dependency_project_id>7443</dependency_project_id>
+        <dependency_project_title>OpenLDAP</dependency_project_title>
+      </dependency>
+      <dependency type="optional">
+        <dependency_release_id>0</dependency_release_id>
+        <dependency_branch_id>0</dependency_branch_id>
+        <dependency_project_id>12351</dependency_project_id>
+        <dependency_project_title>zlib</dependency_project_title>
+      </dependency>
+      <dependency type="optional">
+        <dependency_release_id>0</dependency_release_id>
+        <dependency_branch_id>0</dependency_branch_id>
+        <dependency_project_id>32047</dependency_project_id>
+        <dependency_project_title>Heimdal</dependency_project_title>
+      </dependency>
+      <dependency type="optional">
+        <dependency_release_id>0</dependency_release_id>
+        <dependency_branch_id>0</dependency_branch_id>
+        <dependency_project_id>44532</dependency_project_id>
+        <dependency_project_title>c-ares</dependency_project_title>
+      </dependency>
+    </dependencies>
+  </project>
+</project-listing>
+</datacheck>
+
+</reply>
+
+#
+# Client-side
+<client>
+<features>
+brotli
+</features>
+<server>
+http
+</server>
+ <name>
+HTTP GET brotli compressed content
+ </name>
+ <command>
+http://%HOSTIP:%HTTPPORT/314 --compressed
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<strip>
+^User-Agent:.*
+</strip>
+<strippart>
+s/^Accept-Encoding: .*/Accept-Encoding: xxx/
+</strippart>
+<protocol>
+GET /314 HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+Accept: */*
+Accept-Encoding: xxx
+
+</protocol>
+</verify>
+</testcase>

tests/data/test315

@@ -0,0 +1,91 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+compressed
+FAILURE
+</keywords>
+</info>
+#
+# Server-side
+<reply>
+# this brotli chunk has three bytes removed from the beginning
+<data base64="yes">
+SFRUUC8xLjEgMjAwIE9LDQpEYXRlOiBNb24sIDI5IE5vdiAyMDA0IDIxOjU2OjUzIEdNVA0KU2Vy
+dmVyOiBBcGFjaGUvMS4zLjMxIChEZWJpYW4gR05VL0xpbnV4KSBtb2RfZ3ppcC8xLjMuMjYuMWEg
+UEhQLzQuMy45LTEgbW9kX3NzbC8yLjguMjAgT3BlblNTTC8wLjkuN2QgbW9kX3BlcmwvMS4yOQ0K
+VmFyeTogQWNjZXB0LUVuY29kaW5nDQpDb250ZW50LVR5cGU6IHRleHQvaHRtbDsgY2hhcnNldD1J
+U08tODg1OS0xDQpDb250ZW50LUVuY29kaW5nOiBicg0KQ29udGVudC1MZW5ndGg6IDEwNTYNCg0K
+AJwFdhtdgaQ8i+mZBoO/lwogPKuqHpeP38jV5TDITTB7/oJVCS69FFDKWDVtMk8y4SfMSu/a9vvL
+xWPweDCKePH/2y9VIkbF+EgCYSNs9v53J8QTIHT4ZucHCCRQiXRdT6XdE60KSlbIvobr5rJQsRn7
+ipIjMVMq3Go+/UXtY2d0yP1qaaGSxCn8nZuUNGh74KOI7EEkgFl1tjYytkpc9mJJy9J+wTTI+Hro
+UQP2VR2DYkNoUECqgtOLlGcVEln4+eVzEWcrb8fNrcrVxLArJBpSd8FX8eZs8ebJUO7aBZ5epHz6
+zel7lhLlfHoQIkGh34riaSVr7VTGDmmO6HjSCzKO27LybZ9I3CtMSD2Il4mB131Tlcbut1BdzL4X
+U4DZYMLBN4jwVZEoHpjzHX+vQ3prnrNw4oB7OWOr/fBzjvfjDuO24WxwzPqPo+V6VNcthz1pfF1+
+sMK4yWY7He33m32EuQgQFSZ3a5Wu4FyQcAb45Z+wUxM5XCmX52YmdUR2YTs+W+bNw2EZSfMRcP3C
+inyJI/cTT+JubL3T4COkhz0Rffeoh/3E4c/6ugma1ubhokYecXp8HBwmeDL48d62H26u69DOyMhg
+1PFj+oVDWnK4K+L5AlRr0mpJLqoGHrzflMLQ6qL2oIo9hN6qCeZEEqXM+/KunVYpWVeTY+hthA0y
+5p5RLLTTS4cehaJOpbFyAVxZOardIkJAVx0NshOZY4hDbts9BXsXzFEOgsFhrIQYgh04StZzllIR
+MVDptYlwGmpZCHHmVECdGiFIfEhkQ2INSwMCuuKpaycgSOO9hJA9UFKDBdzTiLJBP9oUVkKLbHjw
+icICCi3k0HcppcvQaW27AMI06kuQU4WUGizgnkaUDcZqCgsotMgG528UFlBo8SFpb05OAjJq2gEI
+0UgN93KS1OvAOYSLN5IaLOCeRnQpJXuLUwcm7urpg6lYxAk26uEoADdsRytHGkSWjOKP6T07wice
+uNo7CXyu7ohtUZXoEWawRHGVkPDVJYqH+xa0DDRKSSgM4K3efLVPSTaUPvBGIZgnn2JBFFWaMsKZ
+guUuUnz6qaSGqnmGAYiupdC1EFye58V4CLbWVjJU4NF2jrOUYR/Dv04zYwVQtQcFzgmK6H4NHAhm
+b0a6pQRKxZaZ+x2vCC7sCuIu4dNCATwqzk12ue6oEsxzYybLPNGJd084M43O9W8E+5/drd/FQVB2
+X4jlFlCuHuWeQxQo+w73Tb9swW692v3BlfQTP1ClWzuJ+RwuSb9m4V3QVa4MEL+0Xzc5FX9P+YX1
+cgaL+6oMHw7L+IOjOt+n1BOloyqk35lLHX7RZmu8SckMnGP95XjWc4FRKP9x/iXrKaeCnut/zsty
+ZdJS5FRmBT/wb5KK9YWBGnqPLO8isN2HS8gA
+</data>
+
+<datacheck>
+HTTP/1.1 200 OK
+Date: Mon, 29 Nov 2004 21:56:53 GMT
+Server: Apache/1.3.31 (Debian GNU/Linux) mod_gzip/1.3.26.1a PHP/4.3.9-1 mod_ssl/2.8.20 OpenSSL/0.9.7d mod_perl/1.29
+Vary: Accept-Encoding
+Content-Type: text/html; charset=ISO-8859-1
+Content-Encoding: br
+Content-Length: 1056
+
+</datacheck>
+
+</reply>
+
+#
+# Client-side
+<client>
+<features>
+brotli
+</features>
+<server>
+http
+</server>
+ <name>
+HTTP GET brotli compressed content with broken header
+ </name>
+ <command>
+http://%HOSTIP:%HTTPPORT/315 --compressed
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<strip>
+^User-Agent:.*
+</strip>
+<strippart>
+s/^Accept-Encoding: .*/Accept-Encoding: xxx/
+</strippart>
+<protocol>
+GET /315 HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+Accept: */*
+Accept-Encoding: xxx
+
+</protocol>
+<errorcode>
+61
+</errorcode>
+</verify>
+</testcase>

tests/runtests.pl

@@ -217,6 +217,7 @@ my $gopher_ipv6;    # set if Gopher server has IPv6 support
 my $has_ipv6;       # set if libcurl is built with IPv6 support
 my $has_unix;       # set if libcurl is built with Unix sockets support
 my $has_libz;       # set if libcurl is built with libz support
+my $has_brotli;     # set if libcurl is built with brotli support
 my $has_getrlimit;  # set if system has getrlimit()
 my $has_ntlm;       # set if libcurl is built with NTLM support
 my $has_ntlm_wb;    # set if libcurl is built with NTLM delegation to winbind
@@ -2880,6 +2881,9 @@ sub checksystem {
             if($feat =~ /libz/i) {
                 $has_libz = 1;
             }
+            if($feat =~ /brotli/i) {
+                $has_brotli = 1;
+            }
             if($feat =~ /NTLM/i) {
                 # NTLM enabled
                 $has_ntlm=1;
@@ -3396,6 +3400,11 @@ sub singletest {
                     next;
                 }
             }
+            elsif($1 eq "brotli") {
+                if($has_brotli) {
+                    next;
+                }
+            }
             elsif($1 eq "NTLM") {
                 if($has_ntlm) {
                     next;
@@ -3552,6 +3561,11 @@ sub singletest {
                         next;
                     }
                 }
+                elsif($1 eq "brotli") {
+                    if(!$has_brotli) {
+                        next;
+                    }
+                }
                 elsif($1 eq "NTLM") {
                     if(!$has_ntlm) {
                         next;