moparisthebest
/
curl
mirror of https://github.com/moparisthebest/curl
1
0
Fork 0
Code Issues Releases Wiki Activity

HTTP: implement Brotli content encoding 

This uses the brotli external library (https://github.com/google/brotli).
Brotli becomes a feature: additional curl_version_info() bit and
structure fields are provided for it and CURLVERSION_NOW bumped.

Tests 314 and 315 check Brotli content unencoding with correct and
erroneous data.

Some tests are updated to accomodate with the now configuration dependent
parameters of the Accept-Encoding header.
This commit is contained in:
Patrick Monnerat 2017-11-05 15:28:16 +01:00
parent dbcced8e32
commit 11bf1796cd
21 changed files with 633 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

93
configure.ac
View File

@ -148,6 +148,7 @@ dnl initialize all the info variables
curl_ssl_msg="no (--with-{ssl,gnutls,nss,polarssl,mbedtls,cyassl,axtls,winssl,darwinssl} )"
curl_ssh_msg="no (--with-libssh2)"
curl_zlib_msg="no (--with-zlib)"
curl_brotli_msg="no (--with-brotli)"
curl_gss_msg="no (--with-gssapi)"
curl_tls_srp_msg="no (--enable-tls-srp)"
curl_res_msg="default (--enable-ares / --enable-threaded-resolver)"
@ -975,6 +976,94 @@ dnl set variable for use in automakefile(s)
AM_CONDITIONAL(HAVE_LIBZ, test x"$AMFIXLIB" = x1)
AC_SUBST(ZLIB_LIBS)
dnl **********************************************************************
dnl Check for the presence of BROTLI decoder libraries and headers
dnl **********************************************************************
dnl Brotli project home page: https://github.com/google/brotli
dnl Default to compiler & linker defaults for BROTLI files & libraries.
OPT_BROTLI=off
AC_ARG_WITH(brotli,dnl
AC_HELP_STRING([--with-brotli=PATH],[Where to look for brotli, PATH points to the BROTLI installation; when possible, set the PKG_CONFIG_PATH environment variable instead of using this option])
AC_HELP_STRING([--without-brotli], [disable BROTLI]),
OPT_BROTLI=$withval)
if test X"$OPT_BROTLI" != Xno; then
dnl backup the pre-brotli variables
CLEANLDFLAGS="$LDFLAGS"
CLEANCPPFLAGS="$CPPFLAGS"
CLEANLIBS="$LIBS"
case "$OPT_BROTLI" in
yes)
dnl --with-brotli (without path) used
CURL_CHECK_PKGCONFIG(libbrotlidec)
if test "$PKGCONFIG" != "no" ; then
LIB_BROTLI=`$PKGCONFIG --libs-only-l libbrotlidec`
LD_BROTLI=`$PKGCONFIG --libs-only-L libbrotlidec`
CPP_BROTLI=`$PKGCONFIG --cflags-only-I libbrotlidec`
version=`$PKGCONFIG --modversion libbrotlidec`
DIR_BROTLI=`echo $LD_BROTLI | $SED -e 's/-L//'`
fi
;;
off)
dnl no --with-brotli option given, just check default places
;;
*)
dnl use the given --with-brotli spot
PREFIX_BROTLI=$OPT_BROTLI
;;
esac
dnl if given with a prefix, we set -L and -I based on that
if test -n "$PREFIX_BROTLI"; then
LIB_BROTLI="-lbrotlidec"
LD_BROTLI=-L${PREFIX_BROTLI}/lib$libsuff
CPP_BROTLI=-I${PREFIX_BROTLI}/include
DIR_BROTLI=${PREFIX_BROTLI}/lib$libsuff
fi
LDFLAGS="$LDFLAGS $LD_BROTLI"
CPPFLAGS="$CPPFLAGS $CPP_BROTLI"
LIBS="$LIB_BROTLI $LIBS"
AC_CHECK_LIB(brotlidec, BrotliDecoderDecompress)
AC_CHECK_HEADERS(brotli/decode.h,
curl_brotli_msg="enabled (libbrotlidec)"
HAVE_BROTLI=1
AC_DEFINE(HAVE_BROTLI, 1, [if BROTLI is in use])
AC_SUBST(HAVE_BROTLI, [1])
)
if test X"$OPT_BROTLI" != Xoff &&
test "$HAVE_BROTLI" != "1"; then
AC_MSG_ERROR([BROTLI libs and/or directories were not found where specified!])
fi
if test "$HAVE_BROTLI" = "1"; then
if test -n "$DIR_BROTLI"; then
dnl when the brotli shared libs were found in a path that the run-time
dnl linker doesn't search through, we need to add it to LD_LIBRARY_PATH
dnl to prevent further configure tests to fail due to this
if test "x$cross_compiling" != "xyes"; then
LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$DIR_BROTLI"
export LD_LIBRARY_PATH
AC_MSG_NOTICE([Added $DIR_BROTLI to LD_LIBRARY_PATH])
fi
fi
else
dnl no brotli, revert back to clean variables
LDFLAGS=$CLEANLDFLAGS
CPPFLAGS=$CLEANCPPFLAGS
LIBS=$CLEANLIBS
fi
fi
dnl **********************************************************************
dnl Check for LDAP
dnl **********************************************************************
@ -3786,6 +3875,9 @@ fi
if test "x$HAVE_LIBZ" = "x1"; then
SUPPORT_FEATURES="$SUPPORT_FEATURES libz"
fi
if test "x$HAVE_BROTLI" = "x1"; then
SUPPORT_FEATURES="$SUPPORT_FEATURES brotli"
fi
if test "x$USE_ARES" = "x1" -o "x$USE_THREADS_POSIX" = "x1" \
-o "x$USE_THREADS_WIN32" = "x1"; then
SUPPORT_FEATURES="$SUPPORT_FEATURES AsynchDNS"
@ -4003,6 +4095,7 @@ AC_MSG_NOTICE([Configured to build curl/libcurl:
SSL support: ${curl_ssl_msg}
SSH support: ${curl_ssh_msg}
zlib support: ${curl_zlib_msg}
brotli support: ${curl_brotli_msg}
GSS-API support: ${curl_gss_msg}
TLS-SRP support: ${curl_tls_srp_msg}
resolver: ${curl_res_msg}

21
docs/INTERNALS.md
View File

@ -644,9 +644,9 @@ Content Encoding
[HTTP/1.1][4] specifies that a client may request that a server encode its
response. This is usually used to compress a response using one (or more)
encodings from a set of commonly available compression techniques. These
schemes include 'deflate' (the zlib algorithm), 'gzip' and 'compress'. A
client requests that the server perform an encoding by including an
Accept-Encoding header in the request document. The value of the header
schemes include 'deflate' (the zlib algorithm), 'gzip' 'br' (brotli) and
'compress'. A client requests that the server perform an encoding by including
an Accept-Encoding header in the request document. The value of the header
should be one of the recognized tokens 'deflate', ... (there's a way to
register new schemes/tokens, see sec 3.5 of the spec). A server MAY honor
the client's encoding request. When a response is encoded, the server
@ -661,9 +661,10 @@ Content Encoding
## Supported content encodings
The 'deflate' and 'gzip' content encodings are supported by libcurl. Both
regular and chunked transfers work fine. The zlib library is required for
this feature.
The 'deflate', 'gzip' and 'br' content encodings are supported by libcurl.
Both regular and chunked transfers work fine. The zlib library is required
for the 'deflate' and 'gzip' encodings, while the brotli decoding library is
for the 'br' encoding.
## The libcurl interface
@ -674,10 +675,10 @@ Content Encoding
where string is the intended value of the Accept-Encoding header.
Currently, libcurl does support multiple encodings but only
understands how to process responses that use the "deflate" or "gzip"
Content-Encoding, so the only values for [`CURLOPT_ACCEPT_ENCODING`][5]
that will work (besides "identity," which does nothing) are "deflate"
and "gzip". If a response is encoded using the "compress" or methods,
understands how to process responses that use the "deflate", "gzip" and/or
"br" content encodings, so the only values for [`CURLOPT_ACCEPT_ENCODING`][5]
that will work (besides "identity," which does nothing) are "deflate",
"gzip" and "br". If a response is encoded using the "compress" or methods,
libcurl will return an error indicating that the response could
not be decoded. If <string> is NULL no Accept-Encoding header is generated.
If <string> is a zero-length string, then an Accept-Encoding header

2
docs/RESOURCES
View File

@ -81,3 +81,5 @@ This document lists documents and standards used by curl.
RFC 4616 - PLAIN authentication
RFC 4954 - SMTP Authentication
RFC 7932 - Brotli Compressed Data Format

15
docs/TODO
View File

@ -64,9 +64,8 @@
5.4 HTTP Digest using SHA-256
5.5 auth= in URLs
5.6 Refuse "downgrade" redirects
5.7 Brotli compression
5.8 QUIC
5.9 Leave secure cookies alone
5.7 QUIC
5.8 Leave secure cookies alone
6. TELNET
6.1 ditch stdin
@ -514,13 +513,7 @@ This is not detailed in any FTP specification.
Consider a way to tell curl to refuse to "downgrade" protocol with a redirect
and/or possibly a bit that refuses redirect to change protocol completely.
5.7 Brotli compression
Brotli compression performs better than gzip and is being implemented by
browsers and servers widely. The algorithm: https://github.com/google/brotli
The Firefox bug: https://bugzilla.mozilla.org/show_bug.cgi?id=366559
5.8 QUIC
5.7 QUIC
The standardization process of QUIC has been taken to the IETF and can be
followed on the [IETF QUIC Mailing
@ -530,7 +523,7 @@ This is not detailed in any FTP specification.
implemented. This, to allow other projects to benefit from the work and to
thus broaden the interest and chance of others to participate.
5.9 Leave secure cookies alone
5.8 Leave secure cookies alone
Non-secure origins (HTTP sites) should not be allowed to set or modify
cookies with the 'secure' property:

8
docs/libcurl/curl_version_info.3
View File

@ -72,6 +72,12 @@ typedef struct {
const char *libssh_version; /* human readable string */
/* when 'age' is 4 or higher (7.57.0 or later), the members below also
exist */
unsigned int brotli_ver_num; /* Numeric Brotli version
(MAJOR << 24) | (MINOR << 12) | PATCH */
const char *brotli_version; /* human readable string. */
} curl_version_info_data;
.fi
@ -160,6 +166,8 @@ libcurl was built with support for HTTPS-proxy.
libcurl was built with multiple SSL backends. For details, see
\fIcurl_global_sslset(3)\fP.
(Added in 7.56.0)
.IP CURL_VERSION_BROTLI
supports HTTP Brotli content encoding using libbrotlidec (Added in 7.57.0)
.RE
\fIssl_version\fP is an ASCII string for the OpenSSL version used. If libcurl
has no SSL support, this is NULL.

2
docs/libcurl/symbols-in-versions
View File

@ -699,6 +699,7 @@ CURLUSESSL_ALL 7.17.0
CURLUSESSL_CONTROL 7.17.0
CURLUSESSL_NONE 7.17.0
CURLUSESSL_TRY 7.17.0
CURLVERSION_FIFTH 7.57.0
CURLVERSION_FIRST 7.10
CURLVERSION_FOURTH 7.16.1
CURLVERSION_NOW 7.10
@ -829,6 +830,7 @@ CURL_TIMECOND_NONE 7.9.7
CURL_TLSAUTH_NONE 7.21.4
CURL_TLSAUTH_SRP 7.21.4
CURL_VERSION_ASYNCHDNS 7.10.7
CURL_VERSION_BROTLI 7.57.0
CURL_VERSION_CONV 7.15.4
CURL_VERSION_CURLDEBUG 7.19.6
CURL_VERSION_DEBUG 7.10.6

10
include/curl/curl.h
View File

@ -2592,6 +2592,7 @@ typedef enum {
CURLVERSION_SECOND,
CURLVERSION_THIRD,
CURLVERSION_FOURTH,
CURLVERSION_FIFTH,
CURLVERSION_LAST /* never actually use this */
} CURLversion;
@ -2600,7 +2601,7 @@ typedef enum {
meant to be a built-in version number for what kind of struct the caller
expects. If the struct ever changes, we redefine the NOW to another enum
from above. */
#define CURLVERSION_NOW CURLVERSION_FOURTH
#define CURLVERSION_NOW CURLVERSION_FIFTH
typedef struct {
CURLversion age; /* age of the returned struct */
@ -2628,6 +2629,12 @@ typedef struct {
const char *libssh_version; /* human readable string */
/* These fields were added in CURLVERSION_FIFTH */
unsigned int brotli_ver_num; /* Numeric Brotli version
(MAJOR << 24) | (MINOR << 12) | PATCH */
const char *brotli_version; /* human readable string. */
} curl_version_info_data;
#define CURL_VERSION_IPV6 (1<<0) /* IPv6-enabled */
@ -2658,6 +2665,7 @@ typedef struct {
for cookie domain verification */
#define CURL_VERSION_HTTPS_PROXY (1<<21) /* HTTPS-proxy support built-in */
#define CURL_VERSION_MULTI_SSL (1<<22) /* Multiple SSL backends available */
#define CURL_VERSION_BROTLI (1<<23) /* Brotli features are present. */
/*
* NAME curl_version_info()

138
lib/content_encoding.c
View File

@ -37,14 +37,15 @@
#ifndef CURL_DISABLE_HTTP
#define DSIZ CURL_MAX_WRITE_SIZE /* buffer size for decompressed data */
#ifdef HAVE_LIBZ
/* Comment this out if zlib is always going to be at least ver. 1.2.0.4
(doing so will reduce code size slightly). */
#define OLD_ZLIB_SUPPORT 1
#define DSIZ CURL_MAX_WRITE_SIZE /* buffer size for decompressed data */
#define GZIP_MAGIC_0 0x1f
#define GZIP_MAGIC_1 0x8b
@ -505,6 +506,136 @@ static const content_encoding gzip_encoding = {
#endif /* HAVE_LIBZ */
#ifdef HAVE_BROTLI
/* Writer parameters. */
typedef struct {
BrotliDecoderState *br; /* State structure for brotli. */
} brotli_params;
static CURLcode brotli_map_error(BrotliDecoderErrorCode be)
{
switch(be) {
case BROTLI_DECODER_ERROR_FORMAT_EXUBERANT_NIBBLE:
case BROTLI_DECODER_ERROR_FORMAT_EXUBERANT_META_NIBBLE:
case BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_ALPHABET:
case BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_SAME:
case BROTLI_DECODER_ERROR_FORMAT_CL_SPACE:
case BROTLI_DECODER_ERROR_FORMAT_HUFFMAN_SPACE:
case BROTLI_DECODER_ERROR_FORMAT_CONTEXT_MAP_REPEAT:
case BROTLI_DECODER_ERROR_FORMAT_BLOCK_LENGTH_1:
case BROTLI_DECODER_ERROR_FORMAT_BLOCK_LENGTH_2:
case BROTLI_DECODER_ERROR_FORMAT_TRANSFORM:
case BROTLI_DECODER_ERROR_FORMAT_DICTIONARY:
case BROTLI_DECODER_ERROR_FORMAT_WINDOW_BITS:
case BROTLI_DECODER_ERROR_FORMAT_PADDING_1:
case BROTLI_DECODER_ERROR_FORMAT_PADDING_2:
case BROTLI_DECODER_ERROR_COMPOUND_DICTIONARY:
case BROTLI_DECODER_ERROR_DICTIONARY_NOT_SET:
case BROTLI_DECODER_ERROR_INVALID_ARGUMENTS:
return CURLE_BAD_CONTENT_ENCODING;
case BROTLI_DECODER_ERROR_ALLOC_CONTEXT_MODES:
case BROTLI_DECODER_ERROR_ALLOC_TREE_GROUPS:
case BROTLI_DECODER_ERROR_ALLOC_CONTEXT_MAP:
case BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_1:
case BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_2:
case BROTLI_DECODER_ERROR_ALLOC_BLOCK_TYPE_TREES:
return CURLE_OUT_OF_MEMORY;
default:
break;
}
return CURLE_WRITE_ERROR;
}
static CURLcode brotli_init_writer(struct connectdata *conn,
contenc_writer *writer)
{
brotli_params *bp = (brotli_params *) &writer->params;
(void) conn;
if(!writer->downstream)
return CURLE_WRITE_ERROR;
bp->br = BrotliDecoderCreateInstance(NULL, NULL, NULL);
return bp->br? CURLE_OK: CURLE_OUT_OF_MEMORY;
}
static CURLcode brotli_unencode_write(struct connectdata *conn,
contenc_writer *writer,
const char *buf, size_t nbytes)
{
brotli_params *bp = (brotli_params *) &writer->params;
const uint8_t *src = (const uint8_t *) buf;
char *decomp;
uint8_t *dst;
size_t dstleft;
CURLcode result = CURLE_OK;
if(!nbytes)
return CURLE_OK;
if(!bp->br)
return CURLE_WRITE_ERROR; /* Stream already ended. */
decomp = malloc(DSIZ);
if(!decomp)
return CURLE_OUT_OF_MEMORY;
while(nbytes && result == CURLE_OK) {
BrotliDecoderResult r;
dst = (uint8_t *) decomp;
dstleft = DSIZ;
r = BrotliDecoderDecompressStream(bp->br,
&nbytes, &src, &dstleft, &dst, NULL);
result = Curl_unencode_write(conn, writer->downstream,
decomp, DSIZ - dstleft);
if(result)
break;
switch(r) {
case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
break;
case BROTLI_DECODER_RESULT_SUCCESS:
BrotliDecoderDestroyInstance(bp->br);
bp->br = NULL;
if(nbytes)
result = CURLE_WRITE_ERROR;
break;
default:
result = brotli_map_error(BrotliDecoderGetErrorCode(bp->br));
break;
}
}
free(decomp);
return result;
}
static void brotli_close_writer(struct connectdata *conn,
contenc_writer *writer)
{
brotli_params *bp = (brotli_params *) &writer->params;
(void) conn;
if(bp->br) {
BrotliDecoderDestroyInstance(bp->br);
bp->br = NULL;
}
}
static const content_encoding brotli_encoding = {
"br",
NULL,
brotli_init_writer,
brotli_unencode_write,
brotli_close_writer,
sizeof(brotli_params)
};
#endif
/* Identity handler. */
static CURLcode identity_init_writer(struct connectdata *conn,
contenc_writer *writer)
@ -543,6 +674,9 @@ static const content_encoding * const encodings[] = {
#ifdef HAVE_LIBZ
&deflate_encoding,
&gzip_encoding,
#endif
#ifdef HAVE_BROTLI
&brotli_encoding,
#endif
NULL
};

4
lib/urldata.h
View File

@ -96,6 +96,10 @@
#endif
#endif
#ifdef HAVE_BROTLI
#include <brotli/decode.h>
#endif
#include <curl/curl.h>
#include "http_chunks.h" /* for the structs and enum stuff */

36
lib/version.c
View File

@ -5,7 +5,7 @@
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
* Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel@haxx.se>, et al.
* Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al.
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
@ -74,6 +74,18 @@ void Curl_version_init(void)
curl_version_info(CURLVERSION_NOW);
}
#ifdef HAVE_BROTLI
static size_t brotli_version(char *buf, size_t bufsz)
{
uint32_t brotli_version = BrotliDecoderVersion();
unsigned int major = brotli_version >> 24;
unsigned int minor = (brotli_version & 0x00FFFFFF) >> 12;
unsigned int patch = brotli_version & 0x00000FFF;
return snprintf(buf, bufsz, "%u.%u.%u", major, minor, patch);
}
#endif
char *curl_version(void)
{
static bool initialized;
@ -105,6 +117,14 @@ char *curl_version(void)
left -= len;
ptr += len;
#endif
#ifdef HAVE_BROTLI
len = snprintf(ptr, left, "%s", " brotli/");
left -= len;
ptr += len;
len = brotli_version(ptr, left);
left -= len;
ptr += len;
#endif
#ifdef USE_ARES
/* this function is only present in c-ares, not in the original ares */
len = snprintf(ptr, left, " c-ares/%s", ares_version(NULL));
@ -326,6 +346,9 @@ static curl_version_info_data version_info = {
#endif
#if defined(CURL_WITH_MULTI_SSL)
| CURL_VERSION_MULTI_SSL
#endif
#if defined(HAVE_BROTLI)
| CURL_VERSION_BROTLI
#endif
,
NULL, /* ssl_version */
@ -337,6 +360,8 @@ static curl_version_info_data version_info = {
NULL, /* libidn version */
0, /* iconv version */
NULL, /* ssh lib version */
0, /* brotli_ver_num */
NULL, /* brotli version */
};
curl_version_info_data *curl_version_info(CURLversion stamp)
@ -348,6 +373,9 @@ curl_version_info_data *curl_version_info(CURLversion stamp)
#ifdef USE_SSL
static char ssl_buffer[80];
#endif
#ifdef HAVE_BROTLI
static char brotli_buffer[80];
#endif
if(initialized)
return &version_info;
@ -396,6 +424,12 @@ curl_version_info_data *curl_version_info(CURLversion stamp)
version_info.libssh_version = ssh_buffer;
#endif
#ifdef HAVE_BROTLI
version_info.brotli_ver_num = BrotliDecoderVersion();
brotli_version(brotli_buffer, sizeof brotli_buffer);
version_info.brotli_version = brotli_buffer;
#endif
(void)stamp; /* avoid compiler warnings, we don't use this */
initialized = true;

1
src/tool_help.c
View File

@ -499,6 +499,7 @@ static const struct feat feats[] = {
{"NTLM_WB", CURL_VERSION_NTLM_WB},
{"SSL", CURL_VERSION_SSL},
{"libz", CURL_VERSION_LIBZ},
{"brotli", CURL_VERSION_BROTLI},
{"CharConv", CURL_VERSION_CONV},
{"TLS-SRP", CURL_VERSION_TLSAUTH_SRP},
{"HTTP2", CURL_VERSION_HTTP2},

2
tests/data/Makefile.inc
View File

@ -54,7 +54,7 @@ test271 test272 test273 test274 test275 test276 test277 test278 test279 \
test280 test281 test282 test283 test284 test285 test286 test287 test288 \
test289 test290 test291 test292 test293 test294 test295 test296 test297 \
test298 test299 test300 test301 test302 test303 test304 test305 test306 \
test307 test308 test309 test310 test311 test312 test313 \
test307 test308 test309 test310 test311 test312 test313 test314 test315 \
test320 test321 test322 test323 test324 \
test325 \
test350 test351 test352 test353 test354 \

5
tests/data/test220
View File

@ -57,11 +57,14 @@ http://%HOSTIP:%HTTPPORT/220 --compressed
<strip>
^User-Agent:.*
</strip>
<strippart>
s/^Accept-Encoding: .*/Accept-Encoding: xxx/
</strippart>
<protocol>
GET /220 HTTP/1.1
Host: %HOSTIP:%HTTPPORT
Accept: */*
Accept-Encoding: deflate, gzip
Accept-Encoding: xxx
</protocol>
</verify>

5
tests/data/test221
View File

@ -57,11 +57,14 @@ http://%HOSTIP:%HTTPPORT/221 --compressed
<strip>
^User-Agent:.*
</strip>
<strippart>
s/^Accept-Encoding: .*/Accept-Encoding: xxx/
</strippart>
<protocol>
GET /221 HTTP/1.1
Host: %HOSTIP:%HTTPPORT
Accept: */*
Accept-Encoding: deflate, gzip
Accept-Encoding: xxx
</protocol>
<errorcode>

5
tests/data/test222
View File

@ -188,11 +188,14 @@ http://%HOSTIP:%HTTPPORT/222 --compressed
<strip>
^User-Agent:.*
</strip>
<strippart>
s/^Accept-Encoding: .*/Accept-Encoding: xxx/
</strippart>
<protocol>
GET /222 HTTP/1.1
Host: %HOSTIP:%HTTPPORT
Accept: */*
Accept-Encoding: deflate, gzip
Accept-Encoding: xxx
</protocol>
</verify>

5
tests/data/test223
View File

@ -78,11 +78,14 @@ http://%HOSTIP:%HTTPPORT/223 --compressed
<strip>
^User-Agent:.*
</strip>
<strippart>
s/^Accept-Encoding: .*/Accept-Encoding: xxx/
</strippart>
<protocol>
GET /223 HTTP/1.1
Host: %HOSTIP:%HTTPPORT
Accept: */*
Accept-Encoding: deflate, gzip
Accept-Encoding: xxx
</protocol>
<errorcode>

5
tests/data/test224
View File

@ -93,11 +93,14 @@ http://%HOSTIP:%HTTPPORT/224 --compressed
<strip>
^User-Agent:.*
</strip>
<strippart>
s/^Accept-Encoding: .*/Accept-Encoding: xxx/
</strippart>
<protocol>
GET /224 HTTP/1.1
Host: %HOSTIP:%HTTPPORT
Accept: */*
Accept-Encoding: deflate, gzip
Accept-Encoding: xxx
</protocol>
</verify>

5
tests/data/test230
View File

@ -189,11 +189,14 @@ http://%HOSTIP:%HTTPPORT/230 --compressed
<strip>
^User-Agent:.*
</strip>
<strippart>
s/^Accept-Encoding: .*/Accept-Encoding: xxx/
</strippart>
<protocol>
GET /230 HTTP/1.1
Host: %HOSTIP:%HTTPPORT
Accept: */*
Accept-Encoding: deflate, gzip
Accept-Encoding: xxx
</protocol>
</verify>

198
tests/data/test314 Normal file
View File

@ -0,0 +1,198 @@
<testcase>
<info>
<keywords>
HTTP
HTTP GET
compressed
</keywords>
</info>
#
# Server-side
<reply>
<data base64="yes">
SFRUUC8xLjEgMjAwIE9LDQpEYXRlOiBNb24sIDI5IE5vdiAyMDA0IDIxOjU2OjUzIEdNVA0KU2Vy
dmVyOiBBcGFjaGUvMS4zLjMxIChEZWJpYW4gR05VL0xpbnV4KSBtb2RfZ3ppcC8xLjMuMjYuMWEg
UEhQLzQuMy45LTEgbW9kX3NzbC8yLjguMjAgT3BlblNTTC8wLjkuN2QgbW9kX3BlcmwvMS4yOQ0K
VmFyeTogQWNjZXB0LUVuY29kaW5nDQpDb250ZW50LVR5cGU6IHRleHQvaHRtbDsgY2hhcnNldD1J
U08tODg1OS0xDQpDb250ZW50LUVuY29kaW5nOiBicg0KQ29udGVudC1MZW5ndGg6IDEwNTYNCg0K
G7ATAJwFdhtdgaQ8i+mZBoO/lwogPKuqHpeP38jV5TDITTB7/oJVCS69FFDKWDVtMk8y4SfMSu/a
9vvLxWPweDCKePH/2y9VIkbF+EgCYSNs9v53J8QTIHT4ZucHCCRQiXRdT6XdE60KSlbIvobr5rJQ
sRn7ipIjMVMq3Go+/UXtY2d0yP1qaaGSxCn8nZuUNGh74KOI7EEkgFl1tjYytkpc9mJJy9J+wTTI
+HroUQP2VR2DYkNoUECqgtOLlGcVEln4+eVzEWcrb8fNrcrVxLArJBpSd8FX8eZs8ebJUO7aBZ5e
pHz6zel7lhLlfHoQIkGh34riaSVr7VTGDmmO6HjSCzKO27LybZ9I3CtMSD2Il4mB131Tlcbut1Bd
zL4XU4DZYMLBN4jwVZEoHpjzHX+vQ3prnrNw4oB7OWOr/fBzjvfjDuO24WxwzPqPo+V6VNcthz1p
fF1+sMK4yWY7He33m32EuQgQFSZ3a5Wu4FyQcAb45Z+wUxM5XCmX52YmdUR2YTs+W+bNw2EZSfMR
cP3CinyJI/cTT+JubL3T4COkhz0Rffeoh/3E4c/6ugma1ubhokYecXp8HBwmeDL48d62H26u69DO
yMhg1PFj+oVDWnK4K+L5AlRr0mpJLqoGHrzflMLQ6qL2oIo9hN6qCeZEEqXM+/KunVYpWVeTY+ht
hA0y5p5RLLTTS4cehaJOpbFyAVxZOardIkJAVx0NshOZY4hDbts9BXsXzFEOgsFhrIQYgh04StZz
llIRMVDptYlwGmpZCHHmVECdGiFIfEhkQ2INSwMCuuKpaycgSOO9hJA9UFKDBdzTiLJBP9oUVkKL
bHjwicICCi3k0HcppcvQaW27AMI06kuQU4WUGizgnkaUDcZqCgsotMgG528UFlBo8SFpb05OAjJq
2gEI0UgN93KS1OvAOYSLN5IaLOCeRnQpJXuLUwcm7urpg6lYxAk26uEoADdsRytHGkSWjOKP6T07
wiceuNo7CXyu7ohtUZXoEWawRHGVkPDVJYqH+xa0DDRKSSgM4K3efLVPSTaUPvBGIZgnn2JBFFWa
MsKZguUuUnz6qaSGqnmGAYiupdC1EFye58V4CLbWVjJU4NF2jrOUYR/Dv04zYwVQtQcFzgmK6H4N
HAhmb0a6pQRKxZaZ+x2vCC7sCuIu4dNCATwqzk12ue6oEsxzYybLPNGJd084M43O9W8E+5/drd/F
QVB2X4jlFlCuHuWeQxQo+w73Tb9swW692v3BlfQTP1ClWzuJ+RwuSb9m4V3QVa4MEL+0Xzc5FX9P
+YX1cgaL+6oMHw7L+IOjOt+n1BOloyqk35lLHX7RZmu8SckMnGP95XjWc4FRKP9x/iXrKaeCnut/
zstyZdJS5FRmBT/wb5KK9YWBGnqPLO8isN2HS8gA
</data>
<datacheck>
HTTP/1.1 200 OK
Date: Mon, 29 Nov 2004 21:56:53 GMT
Server: Apache/1.3.31 (Debian GNU/Linux) mod_gzip/1.3.26.1a PHP/4.3.9-1 mod_ssl/2.8.20 OpenSSL/0.9.7d mod_perl/1.29
Vary: Accept-Encoding
Content-Type: text/html; charset=ISO-8859-1
Content-Encoding: br
Content-Length: 1056
<?xml version="1.0" encoding="ISO-8859-1"?>
<!DOCTYPE project-listing SYSTEM "http://freshmeat.net/backend/fm-projects-0.4.dtd">
<project-listing>
<project>
<project_id>1612</project_id>
<date_added>1998-08-21 04:01:29</date_added>
<date_updated>2004-10-18 02:22:23</date_updated>
<projectname_short>curl</projectname_short>
<projectname_full>curl and libcurl</projectname_full>
<desc_short>Command line tool and library for client-side URL transfers.</desc_short>
<desc_full>curl and libcurl is a tool for transferring files
using URL syntax. It supports HTTP, HTTPS, FTP,
FTPS, DICT, TELNET, LDAP, FILE, and GOPHER, as
well as HTTP-post, HTTP-put, cookies, FTP upload,
resumed transfers, passwords, portnumbers, SSL
certificates, Kerberos, and proxies. It is powered
by libcurl, the client-side URL transfer library.
There are bindings to libcurl for over 20
languages and environments.
</desc_full>
<vitality_score>5784.57</vitality_score>
<vitality_percent>3.16</vitality_percent>
<vitality_rank>169</vitality_rank>
<popularity_score>6594.54</popularity_score>
<popularity_percent>13.81</popularity_percent>
<popularity_rank>105</popularity_rank>
<rating>8.50</rating>
<rating_count>21</rating_count>
<rating_rank>183</rating_rank>
<subscriptions>323</subscriptions>
<branch_name>Default</branch_name>
<url_project_page>http://freshmeat.net/projects/curl/</url_project_page>
<url_homepage>http://freshmeat.net/redir/curl/1612/url_homepage/</url_homepage>
<url_tgz>http://freshmeat.net/redir/curl/1612/url_tgz/</url_tgz>
<url_bz2>http://freshmeat.net/redir/curl/1612/url_bz2/</url_bz2>
<url_zip>http://freshmeat.net/redir/curl/1612/url_zip/</url_zip>
<url_changelog>http://freshmeat.net/redir/curl/1612/url_changelog/</url_changelog>
<url_rpm>http://freshmeat.net/redir/curl/1612/url_rpm/</url_rpm>
<url_deb>http://freshmeat.net/redir/curl/1612/url_deb/</url_deb>
<url_osx>http://freshmeat.net/redir/curl/1612/url_osx/</url_osx>
<url_bsdport>http://freshmeat.net/redir/curl/1612/url_bsdport/</url_bsdport>
<url_purchase></url_purchase>
<url_cvs>http://freshmeat.net/redir/curl/1612/url_cvs/</url_cvs>
<url_list>http://freshmeat.net/redir/curl/1612/url_list/</url_list>
<url_mirror>http://freshmeat.net/redir/curl/1612/url_mirror/</url_mirror>
<url_demo></url_demo>
<license>MIT/X Consortium License</license>
<latest_release>
<latest_release_version>7.12.2</latest_release_version>
<latest_release_id>176085</latest_release_id>
<latest_release_date>2004-10-18 02:22:23</latest_release_date>
</latest_release>
<screenshot_thumb></screenshot_thumb>
<authors>
<author>
<author_name>Daniel Stenberg</author_name>
<author_url>http://freshmeat.net/~bagder/</author_url>
<author_role>Owner</author_role>
</author>
</authors>
<descriminators>
<trove_id>12</trove_id>
<trove_id>226</trove_id>
<trove_id>3</trove_id>
<trove_id>2</trove_id>
<trove_id>188</trove_id>
<trove_id>216</trove_id>
<trove_id>200</trove_id>
<trove_id>220</trove_id>
<trove_id>164</trove_id>
<trove_id>90</trove_id>
<trove_id>89</trove_id>
<trove_id>809</trove_id>
<trove_id>150</trove_id>
<trove_id>224</trove_id>
<trove_id>900</trove_id>
<trove_id>839</trove_id>
</descriminators>
<dependencies>
<dependency type="recommended">
<dependency_release_id>0</dependency_release_id>
<dependency_branch_id>7464</dependency_branch_id>
<dependency_project_id>7464</dependency_project_id>
<dependency_project_title>OpenSSL (Default)</dependency_project_title>
</dependency>
<dependency type="optional">
<dependency_release_id>0</dependency_release_id>
<dependency_branch_id>0</dependency_branch_id>
<dependency_project_id>7443</dependency_project_id>
<dependency_project_title>OpenLDAP</dependency_project_title>
</dependency>
<dependency type="optional">
<dependency_release_id>0</dependency_release_id>
<dependency_branch_id>0</dependency_branch_id>
<dependency_project_id>12351</dependency_project_id>
<dependency_project_title>zlib</dependency_project_title>
</dependency>
<dependency type="optional">
<dependency_release_id>0</dependency_release_id>
<dependency_branch_id>0</dependency_branch_id>
<dependency_project_id>32047</dependency_project_id>
<dependency_project_title>Heimdal</dependency_project_title>
</dependency>
<dependency type="optional">
<dependency_release_id>0</dependency_release_id>
<dependency_branch_id>0</dependency_branch_id>
<dependency_project_id>44532</dependency_project_id>
<dependency_project_title>c-ares</dependency_project_title>
</dependency>
</dependencies>
</project>
</project-listing>
</datacheck>
</reply>
#
# Client-side
<client>
<features>
brotli
</features>
<server>
http
</server>
<name>
HTTP GET brotli compressed content
</name>
<command>
http://%HOSTIP:%HTTPPORT/314 --compressed
</command>
</client>
#
# Verify data after the test has been "shot"
<verify>
<strip>
^User-Agent:.*
</strip>
<strippart>
s/^Accept-Encoding: .*/Accept-Encoding: xxx/
</strippart>
<protocol>
GET /314 HTTP/1.1
Host: %HOSTIP:%HTTPPORT
Accept: */*
Accept-Encoding: xxx
</protocol>
</verify>
</testcase>

91
tests/data/test315 Normal file
View File

@ -0,0 +1,91 @@
<testcase>
<info>
<keywords>
HTTP
HTTP GET
compressed
FAILURE
</keywords>
</info>
#
# Server-side
<reply>
# this brotli chunk has three bytes removed from the beginning
<data base64="yes">
SFRUUC8xLjEgMjAwIE9LDQpEYXRlOiBNb24sIDI5IE5vdiAyMDA0IDIxOjU2OjUzIEdNVA0KU2Vy
dmVyOiBBcGFjaGUvMS4zLjMxIChEZWJpYW4gR05VL0xpbnV4KSBtb2RfZ3ppcC8xLjMuMjYuMWEg
UEhQLzQuMy45LTEgbW9kX3NzbC8yLjguMjAgT3BlblNTTC8wLjkuN2QgbW9kX3BlcmwvMS4yOQ0K
VmFyeTogQWNjZXB0LUVuY29kaW5nDQpDb250ZW50LVR5cGU6IHRleHQvaHRtbDsgY2hhcnNldD1J
U08tODg1OS0xDQpDb250ZW50LUVuY29kaW5nOiBicg0KQ29udGVudC1MZW5ndGg6IDEwNTYNCg0K
AJwFdhtdgaQ8i+mZBoO/lwogPKuqHpeP38jV5TDITTB7/oJVCS69FFDKWDVtMk8y4SfMSu/a9vvL
xWPweDCKePH/2y9VIkbF+EgCYSNs9v53J8QTIHT4ZucHCCRQiXRdT6XdE60KSlbIvobr5rJQsRn7
ipIjMVMq3Go+/UXtY2d0yP1qaaGSxCn8nZuUNGh74KOI7EEkgFl1tjYytkpc9mJJy9J+wTTI+Hro
UQP2VR2DYkNoUECqgtOLlGcVEln4+eVzEWcrb8fNrcrVxLArJBpSd8FX8eZs8ebJUO7aBZ5epHz6
zel7lhLlfHoQIkGh34riaSVr7VTGDmmO6HjSCzKO27LybZ9I3CtMSD2Il4mB131Tlcbut1BdzL4X
U4DZYMLBN4jwVZEoHpjzHX+vQ3prnrNw4oB7OWOr/fBzjvfjDuO24WxwzPqPo+V6VNcthz1pfF1+
sMK4yWY7He33m32EuQgQFSZ3a5Wu4FyQcAb45Z+wUxM5XCmX52YmdUR2YTs+W+bNw2EZSfMRcP3C
inyJI/cTT+JubL3T4COkhz0Rffeoh/3E4c/6ugma1ubhokYecXp8HBwmeDL48d62H26u69DOyMhg
1PFj+oVDWnK4K+L5AlRr0mpJLqoGHrzflMLQ6qL2oIo9hN6qCeZEEqXM+/KunVYpWVeTY+hthA0y
5p5RLLTTS4cehaJOpbFyAVxZOardIkJAVx0NshOZY4hDbts9BXsXzFEOgsFhrIQYgh04StZzllIR
MVDptYlwGmpZCHHmVECdGiFIfEhkQ2INSwMCuuKpaycgSOO9hJA9UFKDBdzTiLJBP9oUVkKLbHjw
icICCi3k0HcppcvQaW27AMI06kuQU4WUGizgnkaUDcZqCgsotMgG528UFlBo8SFpb05OAjJq2gEI
0UgN93KS1OvAOYSLN5IaLOCeRnQpJXuLUwcm7urpg6lYxAk26uEoADdsRytHGkSWjOKP6T07wice
uNo7CXyu7ohtUZXoEWawRHGVkPDVJYqH+xa0DDRKSSgM4K3efLVPSTaUPvBGIZgnn2JBFFWaMsKZ
guUuUnz6qaSGqnmGAYiupdC1EFye58V4CLbWVjJU4NF2jrOUYR/Dv04zYwVQtQcFzgmK6H4NHAhm
b0a6pQRKxZaZ+x2vCC7sCuIu4dNCATwqzk12ue6oEsxzYybLPNGJd084M43O9W8E+5/drd/FQVB2
X4jlFlCuHuWeQxQo+w73Tb9swW692v3BlfQTP1ClWzuJ+RwuSb9m4V3QVa4MEL+0Xzc5FX9P+YX1
cgaL+6oMHw7L+IOjOt+n1BOloyqk35lLHX7RZmu8SckMnGP95XjWc4FRKP9x/iXrKaeCnut/zsty
ZdJS5FRmBT/wb5KK9YWBGnqPLO8isN2HS8gA
</data>
<datacheck>
HTTP/1.1 200 OK
Date: Mon, 29 Nov 2004 21:56:53 GMT
Server: Apache/1.3.31 (Debian GNU/Linux) mod_gzip/1.3.26.1a PHP/4.3.9-1 mod_ssl/2.8.20 OpenSSL/0.9.7d mod_perl/1.29
Vary: Accept-Encoding
Content-Type: text/html; charset=ISO-8859-1
Content-Encoding: br
Content-Length: 1056
</datacheck>
</reply>
#
# Client-side
<client>
<features>
brotli
</features>
<server>
http
</server>
<name>
HTTP GET brotli compressed content with broken header
</name>
<command>
http://%HOSTIP:%HTTPPORT/315 --compressed
</command>
</client>
#
# Verify data after the test has been "shot"
<verify>
<strip>
^User-Agent:.*
</strip>
<strippart>
s/^Accept-Encoding: .*/Accept-Encoding: xxx/
</strippart>
<protocol>
GET /315 HTTP/1.1
Host: %HOSTIP:%HTTPPORT
Accept: */*
Accept-Encoding: xxx
</protocol>
<errorcode>
61
</errorcode>
</verify>
</testcase>

14
tests/runtests.pl
View File

@ -217,6 +217,7 @@ my $gopher_ipv6; # set if Gopher server has IPv6 support
my $has_ipv6; # set if libcurl is built with IPv6 support
my $has_unix; # set if libcurl is built with Unix sockets support
my $has_libz; # set if libcurl is built with libz support
my $has_brotli; # set if libcurl is built with brotli support
my $has_getrlimit; # set if system has getrlimit()
my $has_ntlm; # set if libcurl is built with NTLM support
my $has_ntlm_wb; # set if libcurl is built with NTLM delegation to winbind
@ -2880,6 +2881,9 @@ sub checksystem {
if($feat =~ /libz/i) {
$has_libz = 1;
}
if($feat =~ /brotli/i) {
$has_brotli = 1;
}
if($feat =~ /NTLM/i) {
# NTLM enabled
$has_ntlm=1;
@ -3396,6 +3400,11 @@ sub singletest {
next;
}
}
elsif($1 eq "brotli") {
if($has_brotli) {
next;
}
}
elsif($1 eq "NTLM") {
if($has_ntlm) {
next;
@ -3552,6 +3561,11 @@ sub singletest {
next;
}
}
elsif($1 eq "brotli") {
if(!$has_brotli) {
next;
}
}
elsif($1 eq "NTLM") {
if(!$has_ntlm) {
next;