moparisthebest/wget
1
0
Fork 0
mirror of https://github.com/moparisthebest/wget synced 2024-07-03 16:38:41 -04:00
Code Issues Releases Wiki Activity

fixed IRI misbehaviour(s)

Browse Source
This commit is contained in:
Tim Rühsen 2014-10-06 16:32:37 +02:00 committed by Giuseppe Scrivano
parent 6fc11e46ec
commit bc347cc36f
9 changed files with 94 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/ChangeLog
View File

@ -1,5 +1,11 @@
2014-10-16 Tim Ruehsen <tim.ruehsen@gmx.de>
* url.c (url_parse): little code cleanup
* html-url.c (get_urls_html): HTTP header Content-Type charset preceeds http-equiv
* iri.c (do_conversion): moved iconv code completely into the function
* iri.c (do_conversion): call url_unescape to fix charset conversion
* iri.c (remote_to_utf8): use strcasecmp to compare encoding
* gnutls.c (ssl_connect_wget): do not use SSLv3 except explicitely requested
* openssl.c (ssl_init): do not use SSLv3 except explicitely requested

5
src/html-url.c
View File

@ -748,8 +748,9 @@ get_urls_html (const char *file, const char *url, bool *meta_disallow_follow,
map_html_tags (fm->content, fm->length, collect_tags_mapper, &ctx, flags,
NULL, interesting_attributes);
/* If meta charset isn't null, override content encoding */
if (iri && meta_charset)
/* Meta charset is only valid if there was no HTTP header Content-Type charset. */
/* This is true for HTTP 1.0 and 1.1. */
if (iri && !iri->content_encoding && meta_charset)
set_content_encoding (iri, meta_charset);
DEBUGP (("no-follow in %s: %d\n", file, ctx.nofollow));

102
src/iri.c
View File

@ -38,15 +38,13 @@ as that of the covered work. */
#include <errno.h>
#include "utils.h"
#include "url.h"
/* RFC3987 section 3.1 mandates STD3 ASCII RULES */
#define IDNA_FLAGS IDNA_USE_STD3_ASCII_RULES
/* Note: locale encoding is kept in options struct (opt.locale) */
static bool do_conversion (iconv_t cd, char *in, size_t inlen, char **out);
/* Given a string containing "charset=XXX", return the encoding if found,
or NULL otherwise */
char *
@ -110,52 +108,34 @@ check_encoding_name (char *encoding)
return true;
}
/* Try converting string str from locale to UTF-8. Return a new string
on success, or str on error or if conversion isn't needed. */
const char *
locale_to_utf8 (const char *str)
{
iconv_t l2u;
char *new;
/* That shouldn't happen, just in case */
if (!opt.locale)
{
logprintf (LOG_VERBOSE, _("locale_to_utf8: locale is unset\n"));
opt.locale = find_locale ();
}
if (!opt.locale || !strcasecmp (opt.locale, "utf-8"))
return str;
l2u = iconv_open ("UTF-8", opt.locale);
if (l2u == (iconv_t)(-1))
{
logprintf (LOG_VERBOSE, _("Conversion from %s to %s isn't supported\n"),
quote (opt.locale), quote ("UTF-8"));
return str;
}
if (do_conversion (l2u, (char *) str, strlen ((char *) str), &new))
return (const char *) new;
return str;
}
/* Do the conversion according to the passed conversion descriptor cd. *out
will contain the transcoded string on success. *out content is
unspecified otherwise. */
static bool
do_conversion (iconv_t cd, char *in, size_t inlen, char **out)
do_conversion (const char *tocode, const char *fromcode, char *in, size_t inlen, char **out)
{
iconv_t cd;
/* sXXXav : hummm hard to guess... */
size_t len, done, outlen = inlen * 2;
size_t len, done, outlen;
int invalid = 0, tooshort = 0;
char *s;
char *s, *in_org, *in_save;
s = xmalloc (outlen + 1);
*out = s;
len = outlen;
cd = iconv_open (tocode, fromcode);
if (cd == (iconv_t)(-1))
{
logprintf (LOG_VERBOSE, _("Conversion from %s to %s isn't supported\n"),
quote (opt.locale), quote ("UTF-8"));
return false;
}
/* iconv() has to work on an unescaped string */
in_org = in;
in_save = in = strndup(in, inlen);
url_unescape(in);
inlen = strlen(in);
len = outlen = inlen * 2;
*out = s = xmalloc (outlen + 1);
done = 0;
for (;;)
@ -164,6 +144,9 @@ do_conversion (iconv_t cd, char *in, size_t inlen, char **out)
{
*out = s;
*(s + len - outlen - done) = '\0';
xfree(in_save);
iconv_close(cd);
logprintf (LOG_VERBOSE, _("converted '%s' (%s) -> '%s' (%s)\n"), in_org, fromcode, *out, tocode);
return true;
}
@ -202,9 +185,35 @@ do_conversion (iconv_t cd, char *in, size_t inlen, char **out)
}
}
xfree(in_save);
iconv_close(cd);
logprintf (LOG_VERBOSE, _("converted '%s' (%s) -> '%s' (%s)\n"), in_org, fromcode, *out, tocode);
return false;
}
/* Try converting string str from locale to UTF-8. Return a new string
on success, or str on error or if conversion isn't needed. */
const char *
locale_to_utf8 (const char *str)
{
char *new;
/* That shouldn't happen, just in case */
if (!opt.locale)
{
logprintf (LOG_VERBOSE, _("locale_to_utf8: locale is unset\n"));
opt.locale = find_locale ();
}
if (!opt.locale || !strcasecmp (opt.locale, "utf-8"))
return str;
if (do_conversion ("UTF-8", opt.locale, (char *) str, strlen ((char *) str), &new))
return (const char *) new;
return str;
}
/* Try to "ASCII encode" UTF-8 host. Return the new domain on success or NULL
on error. */
char *
@ -258,7 +267,6 @@ idn_decode (char *host)
bool
remote_to_utf8 (struct iri *iri, const char *str, const char **new)
{
iconv_t cd;
bool ret = false;
if (!iri->uri_encoding)
@ -267,7 +275,7 @@ remote_to_utf8 (struct iri *iri, const char *str, const char **new)
/* When `i->uri_encoding' == "UTF-8" there is nothing to convert. But we must
test for non-ASCII symbols for correct hostname processing in `idn_encode'
function. */
if (!strcmp (iri->uri_encoding, "UTF-8"))
if (!strcasecmp (iri->uri_encoding, "UTF-8"))
{
const char *p = str;
for (p = str; *p; p++)
@ -279,15 +287,9 @@ remote_to_utf8 (struct iri *iri, const char *str, const char **new)
return false;
}
cd = iconv_open ("UTF-8", iri->uri_encoding);
if (cd == (iconv_t)(-1))
return false;
if (do_conversion (cd, (char *) str, strlen ((char *) str), (char **) new))
if (do_conversion ("UTF-8", iri->uri_encoding, (char *) str, strlen (str), (char **) new))
ret = true;
iconv_close (cd);
/* Test if something was converted */
if (!strcmp (str, *new))
{

19
src/url.c
View File

@ -681,7 +681,6 @@ url_parse (const char *url, int *error, struct iri *iri, bool percent_encode)
char *user = NULL, *passwd = NULL;
const char *url_encoded = NULL;
char *new_url = NULL;
int error_code;
@ -695,29 +694,29 @@ url_parse (const char *url, int *error, struct iri *iri, bool percent_encode)
goto error;
}
url_encoded = url;
if (iri && iri->utf8_encode)
{
char *new_url = NULL;
iri->utf8_encode = remote_to_utf8 (iri, iri->orig_url ? iri->orig_url : url, (const char **) &new_url);
if (!iri->utf8_encode)
new_url = NULL;
else
{
iri->orig_url = xstrdup (url);
percent_encode = true;
url_encoded = reencode_escapes (new_url);
if (url_encoded != new_url)
xfree (new_url);
percent_encode = false;
}
}
/* XXX XXX Could that change introduce (security) bugs ??? XXX XXX*/
if (percent_encode)
url_encoded = reencode_escapes (new_url ? new_url : url);
else
url_encoded = new_url ? new_url : url;
url_encoded = reencode_escapes (url);
p = url_encoded;
if (new_url && url_encoded != new_url)
xfree (new_url);
p += strlen (supported_schemes[scheme].leading_string);
uname_b = p;
p = url_skip_credentials (p);

7
tests/ChangeLog
View File

@ -1,3 +1,10 @@
2014-10-06 Tim Ruehsen <tim.ruehsen@gmx.de>
* Test-iri.px: fixed encodings
* Test-iri-forced-remote.px: fixed encodings
* Test-iri-percent.px: fixed encodings
* Test-idn-meta.px: fixed encodings
2014-10-02 Tim Ruehsen <tim.ruehsen@gmx.de>
* tests/WgetTests.pm: use filename as default test name

3
tests/Test-idn-meta.px
View File

@ -27,7 +27,8 @@ my %urls = (
code => "200",
msg => "You want fries with that?",
headers => {
'Content-Type' => 'text/html; charset=UTF-8',
# HTTP header preceeds http-equiv, simply just omit it here
#'Content-Type' => 'text/html; charset=UTF-8',
},
content => $starter_file,
},

31
tests/Test-iri-forced-remote.px
View File

@ -48,7 +48,7 @@ my $pagefrancais = <<EOF;
<html>
<head>
<title>La seule page en français</title>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1"/>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
</head>
<body>
<p>
@ -117,17 +117,10 @@ my %urls = (
content => "",
},
'/p1_fran%C3%A7ais.html' => { # UTF-8 encoded
code => "404",
msg => "File not found",
headers => {
"Content-type" => "text/html; charset=UTF-8",
},
content => $page404,
},
'/p1_fran%E7ais.html' => {
code => "200",
msg => "Ok",
headers => {
# wrong charset here, overridden by --remote-encoding=iso-8859-1
"Content-type" => "text/html; charset=UTF-8",
},
content => $pagefrancais,
@ -140,14 +133,6 @@ my %urls = (
},
content => $pageeen,
},
'/p2_%E9%E9n.html' => {
code => "200",
msg => "Ok",
headers => {
"Content-type" => "text/html; charset=ISO-8859-1",
},
content => $pageeen,
},
'/p3_%E2%82%AC%E2%82%AC%E2%82%AC.html' => { # UTF-8 encoded
code => "200",
msg => "Ok",
@ -156,14 +141,6 @@ my %urls = (
},
content => $pageeuro,
},
'/p3_%A4%A4%A4.html' => {
code => "200",
msg => "Ok",
headers => {
"Content-type" => "text/plain",
},
content => $pageeuro,
},
'/p3_%C2%A4%C2%A4%C2%A4.html' => { # UTF-8 encoded
code => "200",
msg => "Ok",
@ -174,7 +151,7 @@ my %urls = (
},
);
my $cmdline = $WgetTest::WGETPATH . " --iri --trust-server-names --remote-encoding=iso-8859-1 -nH -r http://localhost:{{port}}/";
my $cmdline = $WgetTest::WGETPATH . " --iri -e robots=on --trust-server-names --remote-encoding=iso-8859-1 -nH -r http://localhost:{{port}}/";
my $expected_error_code = 0;
@ -185,7 +162,7 @@ my %expected_downloaded_files = (
'robots.txt' => {
content => "",
},
"p1_fran${ccedilla_l15}ais.html" => {
"p1_fran${ccedilla_u8}ais.html" => {
content => $pagefrancais,
},
"p2_${eacute_u8}${eacute_u8}n.html" => {

6
tests/Test-iri-percent.px
View File

@ -11,6 +11,8 @@ use HTTPTest;
my $ccedilla_l15 = "\xE7";
my $ccedilla_l15_pct = "%E7";
my $ccedilla_u8 = "\xC3\xA7";
my $ccedilla_u8_pct = "%C3%A7";
my $eacute_l1 = "\xE9";
my $eacute_u8 = "\xC3\xA9";
my $eacute_u8_pct = "%C3%A9";
@ -52,7 +54,7 @@ my %urls = (
},
content => $pageindex,
},
"/hello_${ccedilla_l15_pct}${eacute_u8_pct}.html" => {
"/hello_${ccedilla_u8_pct}${eacute_u8_pct}.html" => {
code => "200",
msg => "Ok",
headers => {
@ -70,7 +72,7 @@ my %expected_downloaded_files = (
'index.html' => {
content => $pageindex,
},
"hello_${ccedilla_l15}${eacute_u8}.html" => {
"hello_${ccedilla_u8}${eacute_u8}.html" => {
content => $pagefrancais,
},
);

28
tests/Test-iri.px
View File

@ -42,11 +42,12 @@ my $pageindex = <<EOF;
</html>
EOF
# specifying a wrong charset in http-equiv - it will be overridden by Content-Type HTTP header
my $pagefrancais = <<EOF;
<html>
<head>
<title>La seule page en français</title>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1"/>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
</head>
<body>
<p>
@ -131,18 +132,11 @@ my %urls = (
content => "",
},
'/p1_fran%C3%A7ais.html' => { # UTF-8 encoded
code => "404",
msg => "File not found",
headers => {
"Content-type" => "text/html; charset=UTF-8",
},
content => $page404,
},
'/p1_fran%E7ais.html' => {
code => "200",
msg => "Ok",
headers => {
"Content-type" => "text/html; charset=UTF-8",
# Content-Type header overrides http-equiv Content-Type
"Content-type" => "text/html; charset=ISO-8859-15",
},
content => $pagefrancais,
},
@ -150,10 +144,10 @@ my %urls = (
code => "200",
msg => "Ok",
request_headers => {
"Referer" => qr|http://localhost:[0-9]+/p1_fran%E7ais.html|,
"Referer" => qr|http://localhost:[0-9]+/p1_fran%C3%A7ais.html|,
},
headers => {
"Content-type" => "text/html; charset=ISO-8859-1",
"Content-type" => "text/html; charset=UTF-8",
},
content => $pageeen,
},
@ -165,14 +159,6 @@ my %urls = (
},
content => $pageeuro,
},
'/p3_%A4%A4%A4.html' => {
code => "200",
msg => "Ok",
headers => {
"Content-type" => "text/plain; charset=ISO-8859-1",
},
content => $pageeuro,
},
'/p4_m%C3%A9%C3%A9r.html' => {
code => "200",
msg => "Ok",
@ -197,7 +183,7 @@ my %expected_downloaded_files = (
'robots.txt' => {
content => "",
},
"p1_fran${ccedilla_l15}ais.html" => {
"p1_fran${ccedilla_u8}ais.html" => {
content => $pagefrancais,
},
"p2_${eacute_u8}${eacute_u8}n.html" => {