moparisthebest/wget
1
0
Fork 0
mirror of https://github.com/moparisthebest/wget synced 2024-07-03 16:38:41 -04:00
Code Issues Releases Wiki Activity

Correct iri handling while fetching a remote file list with -i and provide a test

Browse Source
This commit is contained in:
Xavier Saint 2008-08-14 18:26:53 +02:00
parent e6b4e761d1
commit 723dbfc818
6 changed files with 195 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main.c
View File

@ -1196,7 +1196,7 @@ WARNING: Can't reopen standard output in binary mode;\n\
if (url_scheme (*t) == SCHEME_FTP) if (url_scheme (*t) == SCHEME_FTP)
opt.follow_ftp = 1; opt.follow_ftp = 1;
status = retrieve_tree (*t); status = retrieve_tree (*t, NULL);
opt.follow_ftp = old_follow_ftp; opt.follow_ftp = old_follow_ftp;
} }

15
src/recur.c
View File

@ -187,7 +187,7 @@ static bool descend_redirect_p (const char *, const char *, int,
options, add it to the queue. */ options, add it to the queue. */
uerr_t uerr_t
retrieve_tree (const char *start_url) retrieve_tree (const char *start_url, struct iri *pi)
{ {
uerr_t status = RETROK; uerr_t status = RETROK;
@ -201,7 +201,18 @@ retrieve_tree (const char *start_url)
int up_error_code; int up_error_code;
struct url *start_url_parsed; struct url *start_url_parsed;
struct iri *i = iri_new (); struct iri *i = iri_new ();
set_uri_encoding (i, opt.locale, true);
#define COPYSTR(x) (x) ? xstrdup(x) : NULL;
/* Duplicate pi struct if not NULL */
if (pi)
{
i->uri_encoding = COPYSTR (pi->uri_encoding);
i->content_encoding = COPYSTR (pi->content_encoding);
i->utf8_encode = pi->utf8_encode;
}
else
set_uri_encoding (i, opt.locale, true);
#undef COPYSTR
start_url_parsed = url_parse (start_url, &up_error_code, i); start_url_parsed = url_parse (start_url, &up_error_code, i);
if (!start_url_parsed) if (!start_url_parsed)

2
src/recur.h
View File

@ -42,6 +42,6 @@ as that of the covered work. */
struct urlpos; struct urlpos;
void recursive_cleanup (void); void recursive_cleanup (void);
uerr_t retrieve_tree (const char *); uerr_t retrieve_tree (const char *, struct iri *);
#endif /* RECUR_H */ #endif /* RECUR_H */

8
src/retr.c
View File

@ -651,7 +651,6 @@ retrieve_url (const char *origurl, char **file, char **newloc,
proxy = getproxy (u); proxy = getproxy (u);
if (proxy) if (proxy)
{ {
/* sXXXav : could a proxy include a path ??? */
struct iri *pi = iri_new (); struct iri *pi = iri_new ();
set_uri_encoding (pi, opt.locale, true); set_uri_encoding (pi, opt.locale, true);
pi->utf8_encode = false; pi->utf8_encode = false;
@ -858,6 +857,7 @@ retrieve_from_file (const char *file, bool html, int *count)
*count = 0; /* Reset the URL count. */ *count = 0; /* Reset the URL count. */
/* sXXXav : Assume filename and links in the file are in the locale */ /* sXXXav : Assume filename and links in the file are in the locale */
set_uri_encoding (iri, opt.locale, true);
set_content_encoding (iri, opt.locale); set_content_encoding (iri, opt.locale);
if (url_has_scheme (url)) if (url_has_scheme (url))
@ -894,6 +894,10 @@ retrieve_from_file (const char *file, bool html, int *count)
status = QUOTEXC; status = QUOTEXC;
break; break;
} }
/* Reset UTF-8 encode status */
iri->utf8_encode = opt.enable_iri;
if ((opt.recursive || opt.page_requisites) if ((opt.recursive || opt.page_requisites)
&& (cur_url->url->scheme != SCHEME_FTP || getproxy (cur_url->url))) && (cur_url->url->scheme != SCHEME_FTP || getproxy (cur_url->url)))
{ {
@ -903,7 +907,7 @@ retrieve_from_file (const char *file, bool html, int *count)
if (cur_url->url->scheme == SCHEME_FTP) if (cur_url->url->scheme == SCHEME_FTP)
opt.follow_ftp = 1; opt.follow_ftp = 1;
status = retrieve_tree (cur_url->url->url); status = retrieve_tree (cur_url->url->url, iri);
opt.follow_ftp = old_follow_ftp; opt.follow_ftp = old_follow_ftp;
} }

173
tests/Test-iri-list.px Executable file
View File

@ -0,0 +1,173 @@
#!/usr/bin/perl -w
use strict;
use HTTPTest;
# cf. http://en.wikipedia.org/wiki/Latin1
# http://en.wikipedia.org/wiki/ISO-8859-15
###############################################################################
#
# mime : charset found in Content-Type HTTP MIME header
# meta : charset found in Content-Type meta tag
#
# index.html mime + file = iso-8859-15
# p1_français.html meta + file = iso-8859-1, mime = utf-8
# p2_één.html meta + file = utf-8, mime =iso-8859-1
#
my $ccedilla_l1 = "\xE7";
my $ccedilla_u8 = "\xC3\xA7";
my $eacute_l1 = "\xE9";
my $eacute_u8 = "\xC3\xA9";
my $urllist = <<EOF;
http://localhost:{{port}}/
http://localhost:{{port}}/p1_fran${ccedilla_l1}ais.html
http://localhost:{{port}}/p2_${eacute_l1}${eacute_l1}n.html
EOF
my $pageindex = <<EOF;
<html>
<head>
<title>Main Page</title>
</head>
<body>
<p>
Main page.
</p>
</body>
</html>
EOF
my $pagefrancais = <<EOF;
<html>
<head>
<title>La seule page en français</title>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1"/>
</head>
<body>
<p>
French page.
</p>
</body>
</html>
EOF
my $pageeen = <<EOF;
<html>
<head>
<title>Die enkele nederlandstalige pagina</title>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
</head>
<body>
<p>
Dutch page.
</p>
</body>
</html>
EOF
my $page404 = <<EOF;
<html>
<head>
<title>404</title>
</head>
<body>
<p>
Nop nop nop...
</p>
</body>
</html>
EOF
# code, msg, headers, content
my %urls = (
'/index.html' => {
code => "200",
msg => "Ok",
headers => {
"Content-type" => "text/html; charset=ISO-8859-15",
},
content => $pageindex,
},
'/robots.txt' => {
code => "200",
msg => "Ok",
headers => {
"Content-type" => "text/plain",
},
content => "",
},
'/p1_fran%C3%A7ais.html' => { # UTF-8 encoded
code => "404",
msg => "File not found",
headers => {
"Content-type" => "text/html; charset=UTF-8",
},
content => $page404,
},
'/p1_fran%E7ais.html' => {
code => "200",
msg => "Ok",
headers => {
"Content-type" => "text/html; charset=UTF-8",
},
content => $pagefrancais,
},
'/p2_%C3%A9%C3%A9n.html' => { # UTF-8 encoded
code => "200",
msg => "Ok",
headers => {
"Content-type" => "text/html; charset=ISO-8859-1",
},
content => $pageeen,
},
'/p2_%E9%E9n.html' => {
code => "200",
msg => "Ok",
headers => {
"Content-type" => "text/html; charset=ISO-8859-1",
},
content => $pageeen,
},
'/url_list.txt' => {
code => "200",
msg => "Ok",
headers => {
"Content-type" => "text/plain; charset=ISO-8859-1",
},
content => $urllist,
},
);
my $cmdline = $WgetTest::WGETPATH . " --iri -d -i http://localhost:{{port}}/url_list.txt";
my $expected_error_code = 0;
my %expected_downloaded_files = (
'url_list.txt' => {
content => $urllist,
},
'index.html' => {
content => $pageindex,
},
"p1_fran${ccedilla_l1}ais.html" => {
content => $pagefrancais,
},
"p2_${eacute_u8}${eacute_u8}n.html" => {
content => $pageeen,
},
);
###############################################################################
my $the_test = HTTPTest->new (name => "Test-iri-list",
input => \%urls,
cmdline => $cmdline,
errcode => $expected_error_code,
output => \%expected_downloaded_files);
exit $the_test->run();
# vim: et ts=4 sw=4

1
tests/run-px
View File

@ -25,6 +25,7 @@ my @tests = (
'Test-iri.px', 'Test-iri.px',
'Test-iri-disabled.px', 'Test-iri-disabled.px',
'Test-iri-forced-remote.px', 'Test-iri-forced-remote.px',
'Test-iri-list.px',
'Test-N-current.px', 'Test-N-current.px',
'Test-N-smaller.px', 'Test-N-smaller.px',
'Test-N-no-info.px', 'Test-N-no-info.px',