From bfd8a73f004b95d044741f4cb78ecad9de92bddc Mon Sep 17 00:00:00 2001 From: Xavier Saint Date: Sat, 2 Aug 2008 11:22:14 +0200 Subject: [PATCH 1/3] quote*() functions don't like that much NULL arg --- src/iri.c | 9 +++++---- src/recur.c | 2 +- src/retr.c | 3 ++- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/iri.c b/src/iri.c index 3ee99871..9050e858 100644 --- a/src/iri.c +++ b/src/iri.c @@ -318,6 +318,7 @@ iri_new (void) i->uri_encoding = opt.encoding_remote ? xstrdup (opt.encoding_remote) : NULL; i->content_encoding = NULL; i->utf8_encode = opt.enable_iri; + return i; } void @@ -331,12 +332,12 @@ iri_free (struct iri *i) void set_uri_encoding (struct iri *i, char *charset, bool force) { - DEBUGP (("[IRI uri = `%s'\n", quote (charset))); + DEBUGP (("[IRI uri = `%s'\n", charset ? quote (charset) : "None")); if (!force && opt.encoding_remote) return; if (i->uri_encoding) { - if (!strcasecmp (i->uri_encoding, charset)) + if (charset && !strcasecmp (i->uri_encoding, charset)) return; xfree (i->uri_encoding); } @@ -347,12 +348,12 @@ set_uri_encoding (struct iri *i, char *charset, bool force) void set_content_encoding (struct iri *i, char *charset) { - DEBUGP (("[IRI content = %s\n", quote (charset))); + DEBUGP (("[IRI content = %s\n", charset ? quote (charset) : "None")); if (opt.encoding_remote) return; if (i->content_encoding) { - if (!strcasecmp (i->content_encoding, charset)) + if (charset && !strcasecmp (i->content_encoding, charset)) return; xfree (i->content_encoding); } diff --git a/src/recur.c b/src/recur.c index 19ef8f1c..baeaed58 100644 --- a/src/recur.c +++ b/src/recur.c @@ -117,7 +117,7 @@ url_enqueue (struct url_queue *queue, struct iri *i, if (i) DEBUGP (("[IRI Enqueuing %s with %s\n", quote (url), - quote (i->uri_encoding))); + i->uri_encoding ? quote (i->uri_encoding) : "None")); if (queue->tail) queue->tail->next = qel; diff --git a/src/retr.c b/src/retr.c index 111b745a..fa7f762d 100644 --- a/src/retr.c +++ b/src/retr.c @@ -636,7 +636,8 @@ retrieve_url (const char *origurl, char **file, char **newloc, } DEBUGP (("[IRI Retrieving %s with %s (UTF-8=%d)\n", quote (url), - quote (iri->uri_encoding), iri->utf8_encode)); + iri->uri_encoding ? quote (iri->uri_encoding) : "None", + iri->utf8_encode)); if (!refurl) refurl = opt.referer; From da7adbaef4bb2c47a19db3e83620aed06ba9456e Mon Sep 17 00:00:00 2001 From: Xavier Saint Date: Sat, 2 Aug 2008 12:17:03 +0200 Subject: [PATCH 2/3] Functional tests for IRI and HTTP --- tests/Test-iri-disabled.px | 197 ++++++++++++++++++++++++++++ tests/Test-iri-forced-remote.px | 208 +++++++++++++++++++++++++++++ tests/Test-iri.px | 225 ++++++++++++++++++++++++++++++++ tests/run-px | 3 + 4 files changed, 633 insertions(+) create mode 100755 tests/Test-iri-disabled.px create mode 100755 tests/Test-iri-forced-remote.px create mode 100755 tests/Test-iri.px diff --git a/tests/Test-iri-disabled.px b/tests/Test-iri-disabled.px new file mode 100755 index 00000000..122537ff --- /dev/null +++ b/tests/Test-iri-disabled.px @@ -0,0 +1,197 @@ +#!/usr/bin/perl -w + +use strict; + +use HTTPTest; + +# cf. http://en.wikipedia.org/wiki/Latin1 +# http://en.wikipedia.org/wiki/ISO-8859-15 + +############################################################################### +# +# mime : charset found in Content-Type HTTP MIME header +# meta : charset found in Content-Type meta tag +# +# index.html mime + file = iso-8859-15 +# p1_français.html meta + file = iso-8859-1, mime = utf-8 +# p2_één.html mime + file = iso-8859-1 +# p3_€€€.html meta + file = utf-8, mime = iso-8859-1 +# + +my $ccedilla_l15 = "\xE7"; +my $ccedilla_u8 = "\xC3\xA7"; +my $eacute_l1 = "\xE9"; +my $eacute_u8 = "\xC3\xA9"; +my $eurosign_l15 = "\xA4"; +my $eurosign_u8 = "\xE2\x82\xAC"; +my $eurosign2_u8 = "\xE2%82\xAC"; # version wget use... sXXXav + +my $pageindex = < + + Main Page + + +

+ Link to page 1 La seule page en français. + Link to page 3 My tailor is rich. +

+ + +EOF + +my $pagefrancais = < + + La seule page en français + + + +

+ Link to page 2 Die enkele nerderlangstalige pagina. +

+ + +EOF + +my $pageeen = < + + Die enkele nederlandstalige pagina + + +

+ Één is niet veel maar toch meer dan nul.
+ Nerdelands is een mooie taal... dit zin stuckje spreekt vanzelf, of niet :) +

+ + +EOF + +my $pageeuro = < + + Euro page + + +

+ My tailor isn't rich anymore. +

+ + +EOF + +my $page404 = < + + 404 + + +

+ Nop nop nop... +

+ + +EOF + +# code, msg, headers, content +my %urls = ( + '/index.html' => { + code => "200", + msg => "Ok", + headers => { + "Content-type" => "text/html; charset=ISO-8859-15", + }, + content => $pageindex, + }, + '/robots.txt' => { + code => "200", + msg => "Ok", + headers => { + "Content-type" => "text/plain", + }, + content => "", + }, + '/p1_fran%C3%A7ais.html' => { # UTF-8 encoded + code => "200", + msg => "File not found", + headers => { + "Content-type" => "text/html; charset=UTF-8", + }, + content => $pagefrancais, + }, + '/p1_fran%E7ais.html' => { + code => "200", + msg => "Ok", + headers => { + "Content-type" => "text/html; charset=UTF-8", + }, + content => $pagefrancais, + }, + '/p2_%C3%A9%C3%A9n.html' => { # UTF-8 encoded + code => "200", + msg => "Ok", + headers => { + "Content-type" => "text/html; charset=UTF-8", + }, + content => $pageeen, + }, + '/p2_%E9%E9n.html' => { + code => "200", + msg => "Ok", + headers => { + "Content-type" => "text/html; charset=ISO-8859-1", + }, + content => $pageeen, + }, + '/p3_%E2%82%AC%E2%82%AC%E2%82%AC.html' => { # UTF-8 encoded + code => "200", + msg => "Ok", + headers => { + "Content-type" => "text/plain", + }, + content => $pageeuro, + }, + '/p3_%A4%A4%A4.html' => { + code => "200", + msg => "Ok", + headers => { + "Content-type" => "text/plain", + }, + content => $pageeuro, + }, +); + +my $cmdline = $WgetTest::WGETPATH . " --iri=no -nH -r http://localhost:{{port}}/"; + +my $expected_error_code = 0; + +my %expected_downloaded_files = ( + 'index.html' => { + content => $pageindex, + }, + 'robots.txt' => { + content => "", + }, + "p1_fran${ccedilla_l15}ais.html" => { + content => $pagefrancais, + }, + "p2_${eacute_l1}${eacute_l1}n.html" => { + content => $pageeen, + }, + "p3_${eurosign_l15}${eurosign_l15}${eurosign_l15}.html" => { + content => $pageeuro, + }, +); + +############################################################################### + +my $the_test = HTTPTest->new (name => "Test-iri-disabled", + input => \%urls, + cmdline => $cmdline, + errcode => $expected_error_code, + output => \%expected_downloaded_files); +exit $the_test->run(); + +# vim: et ts=4 sw=4 + diff --git a/tests/Test-iri-forced-remote.px b/tests/Test-iri-forced-remote.px new file mode 100755 index 00000000..0d116d8f --- /dev/null +++ b/tests/Test-iri-forced-remote.px @@ -0,0 +1,208 @@ +#!/usr/bin/perl -w + +use strict; + +use HTTPTest; + +# cf. http://en.wikipedia.org/wiki/Latin1 +# http://en.wikipedia.org/wiki/ISO-8859-15 + +############################################################################### +# Force remote encoding to ISO-8859-1 +# +# mime : charset found in Content-Type HTTP MIME header +# meta : charset found in Content-Type meta tag +# +# index.html mime + file = iso-8859-15 +# p1_français.html meta + file = iso-8859-1, mime = utf-8 +# p2_één.html mime + file = iso-8859-1 +# p3_€€€.html meta + file = utf-8, mime = iso-8859-1 +# + +my $ccedilla_l15 = "\xE7"; +my $ccedilla_u8 = "\xC3\xA7"; +my $eacute_l1 = "\xE9"; +my $eacute_u8 = "\xC3\xA9"; +my $eurosign_l15 = "\xA4"; +my $eurosign_u8 = "\xE2\x82\xAC"; +my $eurosign2_u8 = "\xE2%82\xAC"; # version wget use... sXXXav +my $currency_l1 = "\xA4"; +my $currency_u8 = "\xC2\xA4"; + +my $pageindex = < + + Main Page + + +

+ Link to page 1 La seule page en français. + Link to page 3 My tailor is rich. +

+ + +EOF + +my $pagefrancais = < + + La seule page en français + + + +

+ Link to page 2 Die enkele nerderlangstalige pagina. +

+ + +EOF + +my $pageeen = < + + Die enkele nederlandstalige pagina + + +

+ Één is niet veel maar toch meer dan nul.
+ Nerdelands is een mooie taal... dit zin stuckje spreekt vanzelf, of niet :) +

+ + +EOF + +my $pageeuro = < + + Euro page + + +

+ My tailor isn't rich anymore. +

+ + +EOF + +my $page404 = < + + 404 + + +

+ Nop nop nop... +

+ + +EOF + +# code, msg, headers, content +my %urls = ( + '/index.html' => { + code => "200", + msg => "Ok", + headers => { + "Content-type" => "text/html; charset=ISO-8859-15", + }, + content => $pageindex, + }, + '/robots.txt' => { + code => "200", + msg => "Ok", + headers => { + "Content-type" => "text/plain", + }, + content => "", + }, + '/p1_fran%C3%A7ais.html' => { # UTF-8 encoded + code => "404", + msg => "File not found", + headers => { + "Content-type" => "text/html; charset=UTF-8", + }, + content => $page404, + }, + '/p1_fran%E7ais.html' => { + code => "200", + msg => "Ok", + headers => { + "Content-type" => "text/html; charset=UTF-8", + }, + content => $pagefrancais, + }, + '/p2_%C3%A9%C3%A9n.html' => { # UTF-8 encoded + code => "200", + msg => "Ok", + headers => { + "Content-type" => "text/html; charset=UTF-8", + }, + content => $pageeen, + }, + '/p2_%E9%E9n.html' => { + code => "200", + msg => "Ok", + headers => { + "Content-type" => "text/html; charset=ISO-8859-1", + }, + content => $pageeen, + }, + '/p3_%E2%82%AC%E2%82%AC%E2%82%AC.html' => { # UTF-8 encoded + code => "200", + msg => "Ok", + headers => { + "Content-type" => "text/plain", + }, + content => $pageeuro, + }, + '/p3_%A4%A4%A4.html' => { + code => "200", + msg => "Ok", + headers => { + "Content-type" => "text/plain", + }, + content => $pageeuro, + }, + '/p3_%C2%A4%C2%A4%C2%A4.html' => { # UTF-8 encoded + code => "200", + msg => "Ok", + headers => { + "Content-type" => "text/plain", + }, + content => $pageeuro, + }, +); + +my $cmdline = $WgetTest::WGETPATH . " --iri --remote-encoding=iso-8859-1 -nH -r http://localhost:{{port}}/"; + +my $expected_error_code = 0; + +my %expected_downloaded_files = ( + 'index.html' => { + content => $pageindex, + }, + 'robots.txt' => { + content => "", + }, + "p1_fran${ccedilla_l15}ais.html" => { + content => $pagefrancais, + }, + "p2_${eacute_u8}${eacute_u8}n.html" => { + content => $pageeen, + }, + "p3_${currency_u8}${currency_u8}${currency_u8}.html" => { + content => $pageeuro, + }, +); + +############################################################################### + +my $the_test = HTTPTest->new (name => "Test-iri-forced-remote", + input => \%urls, + cmdline => $cmdline, + errcode => $expected_error_code, + output => \%expected_downloaded_files); +exit $the_test->run(); + +# vim: et ts=4 sw=4 + diff --git a/tests/Test-iri.px b/tests/Test-iri.px new file mode 100755 index 00000000..3f4cf3fd --- /dev/null +++ b/tests/Test-iri.px @@ -0,0 +1,225 @@ +#!/usr/bin/perl -w + +use strict; + +use HTTPTest; + +# cf. http://en.wikipedia.org/wiki/Latin1 +# http://en.wikipedia.org/wiki/ISO-8859-15 + +############################################################################### +# +# mime : charset found in Content-Type HTTP MIME header +# meta : charset found in Content-Type meta tag +# +# index.html mime + file = iso-8859-15 +# p1_français.html meta + file = iso-8859-1, mime = utf-8 +# p2_één.html meta + file = utf-8, mime =iso-8859-1 +# p3_€€€.html meta + file = utf-8, mime = iso-8859-1 +# p4_méér.html mime + file = utf-8 +# + +my $ccedilla_l15 = "\xE7"; +my $ccedilla_u8 = "\xC3\xA7"; +my $eacute_l1 = "\xE9"; +my $eacute_u8 = "\xC3\xA9"; +my $eurosign_l15 = "\xA4"; +my $eurosign_u8 = "\xE2\x82\xAC"; +my $eurosign2_u8 = "\xE2%82\xAC"; # version wget use... sXXXav + +my $pageindex = < + + Main Page + + +

+ Link to page 1 La seule page en français. + Link to page 3 My tailor is rich. +

+ + +EOF + +my $pagefrancais = < + + La seule page en français + + + +

+ Link to page 2 Die enkele nerderlangstalige pagina. +

+ + +EOF + +my $pageeen = < + + Die enkele nederlandstalige pagina + + + +

+ Één is niet veel maar toch meer dan nul.
+ Nerdelands is een mooie taal... dit zin stuckje spreekt vanzelf, of niet :)
+ Méér +

+ + +EOF + +my $pageeuro = < + + Euro page + + + +

+ My tailor isn't rich anymore. +

+ + +EOF + +my $pagemeer = < + + Bekende supermarkt + + +

+ Ik ben toch niet gek ! +

+ + +EOF + +my $page404 = < + + 404 + + +

+ Nop nop nop... +

+ + +EOF + +# code, msg, headers, content +my %urls = ( + '/index.html' => { + code => "200", + msg => "Ok", + headers => { + "Content-type" => "text/html; charset=ISO-8859-15", + }, + content => $pageindex, + }, + '/robots.txt' => { + code => "200", + msg => "Ok", + headers => { + "Content-type" => "text/plain", + }, + content => "", + }, + '/p1_fran%C3%A7ais.html' => { # UTF-8 encoded + code => "404", + msg => "File not found", + headers => { + "Content-type" => "text/html; charset=UTF-8", + }, + content => $page404, + }, + '/p1_fran%E7ais.html' => { + code => "200", + msg => "Ok", + headers => { + "Content-type" => "text/html; charset=UTF-8", + }, + content => $pagefrancais, + }, + '/p2_%C3%A9%C3%A9n.html' => { # UTF-8 encoded + code => "200", + msg => "Ok", + headers => { + "Content-type" => "text/html; charset=ISO-8859-1", + }, + content => $pageeen, + }, + '/p2_%E9%E9n.html' => { + code => "200", + msg => "Ok", + headers => { + "Content-type" => "text/html; charset=ISO-8859-1", + }, + content => $pageeen, + }, + '/p3_%E2%82%AC%E2%82%AC%E2%82%AC.html' => { # UTF-8 encoded + code => "200", + msg => "Ok", + headers => { + "Content-type" => "text/plain; charset=ISO-8859-1", + }, + content => $pageeuro, + }, + '/p3_%A4%A4%A4.html' => { + code => "200", + msg => "Ok", + headers => { + "Content-type" => "text/plain; charset=ISO-8859-1", + }, + content => $pageeuro, + }, + '/p4_m%C3%A9%C3%A9r.html' => { + code => "200", + msg => "Ok", + headers => { + "Content-type" => "text/plain; charset=UTF-8", + }, + content => $pagemeer, + }, +); + +my $cmdline = $WgetTest::WGETPATH . " --iri -nH -r http://localhost:{{port}}/"; + +my $expected_error_code = 0; + +my %expected_downloaded_files = ( + 'index.html' => { + content => $pageindex, + }, + 'robots.txt' => { + content => "", + }, + "p1_fran${ccedilla_l15}ais.html" => { + content => $pagefrancais, + }, + "p2_${eacute_u8}${eacute_u8}n.html" => { + content => $pageeen, + }, + "p3_${eurosign2_u8}${eurosign2_u8}${eurosign2_u8}.html" => { + content => $pageeuro, + }, + "p4_m${eacute_u8}${eacute_u8}r.html" => { + content => $pagemeer, + }, +); + +############################################################################### + +my $the_test = HTTPTest->new (name => "Test-iri", + input => \%urls, + cmdline => $cmdline, + errcode => $expected_error_code, + output => \%expected_downloaded_files); +exit $the_test->run(); + +# vim: et ts=4 sw=4 + diff --git a/tests/run-px b/tests/run-px index 37f14324..865246e3 100755 --- a/tests/run-px +++ b/tests/run-px @@ -19,6 +19,9 @@ my @tests = ( 'Test-HTTP-Content-Disposition-1.px', 'Test-HTTP-Content-Disposition-2.px', 'Test-HTTP-Content-Disposition.px', + 'Test-iri.px', + 'Test-iri-disabled.px', + 'Test-iri-forced-remote.px', 'Test-N-current.px', 'Test-N-smaller.px', 'Test-N-no-info.px', From 8d7c2219d1965fb1bda16d46bb45e8fe7dc60501 Mon Sep 17 00:00:00 2001 From: Xavier Saint Date: Sat, 2 Aug 2008 13:47:10 +0200 Subject: [PATCH 3/3] Test FTP IRI support --- tests/Test-ftp-iri-disabled.px | 50 ++++++++++++++++++++++++++++++++++ tests/Test-ftp-iri-fallback.px | 46 +++++++++++++++++++++++++++++++ tests/Test-ftp-iri.px | 47 ++++++++++++++++++++++++++++++++ tests/run-px | 3 ++ 4 files changed, 146 insertions(+) create mode 100755 tests/Test-ftp-iri-disabled.px create mode 100755 tests/Test-ftp-iri-fallback.px create mode 100755 tests/Test-ftp-iri.px diff --git a/tests/Test-ftp-iri-disabled.px b/tests/Test-ftp-iri-disabled.px new file mode 100755 index 00000000..14d849da --- /dev/null +++ b/tests/Test-ftp-iri-disabled.px @@ -0,0 +1,50 @@ +#!/usr/bin/perl -w + +use strict; + +use FTPTest; + + +############################################################################### + +my $ccedilla_l1 = "\xE7"; +my $ccedilla_u8 = "\xC3\xA7"; + +my $francais = < { + content => $francais, + }, + "/fran${ccedilla_l1}ais.txt" => { + content => $francais, + }, +); + +my $cmdline = $WgetTest::WGETPATH . " --iri=no --locale=iso-8859-1 -S ftp://localhost:{{port}}/fran${ccedilla_l1}ais.txt"; + +my $expected_error_code = 0; + +my %expected_downloaded_files = ( + "fran${ccedilla_l1}ais.txt" => { + content => $francais, + }, +); + +############################################################################### + +my $the_test = FTPTest->new (name => "Test-ftp-iri", + input => \%urls, + cmdline => $cmdline, + errcode => $expected_error_code, + output => \%expected_downloaded_files); +exit $the_test->run(); + +# vim: et ts=4 sw=4 + diff --git a/tests/Test-ftp-iri-fallback.px b/tests/Test-ftp-iri-fallback.px new file mode 100755 index 00000000..8902e0f9 --- /dev/null +++ b/tests/Test-ftp-iri-fallback.px @@ -0,0 +1,46 @@ +#!/usr/bin/perl -w + +use strict; + +use FTPTest; + + +############################################################################### + +my $ccedilla_l1 = "\xE7"; +my $ccedilla_u8 = "\xC3\xA7"; + +my $francais = < { + content => $francais, + }, +); + +my $cmdline = $WgetTest::WGETPATH . " --locale=iso-8859-1 -S ftp://localhost:{{port}}/fran${ccedilla_l1}ais.txt"; + +my $expected_error_code = 0; + +my %expected_downloaded_files = ( + "fran${ccedilla_l1}ais.txt" => { + content => $francais, + }, +); + +############################################################################### + +my $the_test = FTPTest->new (name => "Test-ftp-iri", + input => \%urls, + cmdline => $cmdline, + errcode => $expected_error_code, + output => \%expected_downloaded_files); +exit $the_test->run(); + +# vim: et ts=4 sw=4 + diff --git a/tests/Test-ftp-iri.px b/tests/Test-ftp-iri.px new file mode 100755 index 00000000..d453669c --- /dev/null +++ b/tests/Test-ftp-iri.px @@ -0,0 +1,47 @@ +#!/usr/bin/perl -w + +use strict; + +use FTPTest; + + +############################################################################### + +my $ccedilla_l1 = "\xE7"; +my $ccedilla_u8 = "\xC3\xA7"; + +my $francais = < { + content => $francais, + }, +); + +my $cmdline = $WgetTest::WGETPATH . " --locale=iso-8859-1 -S ftp://localhost:{{port}}/fran${ccedilla_l1}ais.txt"; + +my $expected_error_code = 0; + +my %expected_downloaded_files = ( + "fran${ccedilla_u8}ais.txt" => { + content => $francais, + }, +); + +############################################################################### + +my $the_test = FTPTest->new (name => "Test-ftp-iri", + input => \%urls, + cmdline => $cmdline, + errcode => $expected_error_code, + output => \%expected_downloaded_files); +exit $the_test->run(); + +# vim: et ts=4 sw=4 + diff --git a/tests/run-px b/tests/run-px index 865246e3..172adcd7 100755 --- a/tests/run-px +++ b/tests/run-px @@ -16,6 +16,9 @@ my @tests = ( 'Test-E-k-K.px', 'Test-E-k.px', 'Test-ftp.px', + 'Test-ftp-iri.px', + 'Test-ftp-iri-fallback.px', + 'Test-ftp-iri-disabled.px', 'Test-HTTP-Content-Disposition-1.px', 'Test-HTTP-Content-Disposition-2.px', 'Test-HTTP-Content-Disposition.px',