moparisthebest/wget
1
0
Fork 0
mirror of https://github.com/moparisthebest/wget synced 2024-07-03 16:38:41 -04:00
Code Issues Releases Wiki Activity

Add a generic --method command to set a method in HTTP Requests.

Browse Source 
Add supplementary --body-data and --body-file commands to send BODY Data.

Signed-off-by: Darshit Shah <darnir@gmail.com>
This commit is contained in:
Darshit Shah 2013-04-12 23:44:32 +05:30 committed by Giuseppe Scrivano
parent edbeaa0c4e
commit 6c30653a1a
8 changed files with 162 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
doc/ChangeLog
View File

@ -1,3 +1,10 @@
2013-04-05 Darshit Shah <darnir@gmail.com>
* doc/wget.texi: Fix ambiguous wording in --post-data section. Make it
clear that wget does not check for the format of the post-data.
* doc/wget.texi: Add documentation for --method, --body-data and
--body-file.
2012-10-08 Stefano Lattarini <stefano.lattarini@gmail.com> (tiny change)
docs: fix errors and warnings with Texinfo 5

33
doc/wget.texi
View File

@ -1457,6 +1457,11 @@ like everything else. Wget does not currently support
@code{application/x-www-form-urlencoded}. Only one of
@samp{--post-data} and @samp{--post-file} should be specified.
Please note that wget does not require the content to be of the form
@code{key1=value1&key2=value2}, and neither does it test for it. Wget will
simply transmit whatever data is provided to it. Most servers however expect
the POST data to be in the above format when processing HTML Forms.
Please be aware that Wget needs to know the size of the POST data in
advance. Therefore the argument to @code{--post-file} must be a regular
file; specifying a FIFO or something like @file{/dev/stdin} won't work.
@ -1497,6 +1502,34 @@ them (and neither will browsers) and the @file{cookies.txt} file will
be empty. In that case use @samp{--keep-session-cookies} along with
@samp{--save-cookies} to force saving of session cookies.
@cindex Other HTTP Methods
@item --method=@var{HTTP-Method}
For the purpose of RESTful scripting, Wget allows sending of other HTTP Methods
without the need to explicitly set them using @samp{--header=Header-Line}.
Wget will use whatever string is passed to it after @samp{--method} as the HTTP
Method to the server.
@item --body-data=@var{Data-String}
@itemx --body-file=@var{Data-File}
Must be set when additional data needs to be sent to the server along with the
Method specified using @samp{--method}. @samp{--post-data} sends @var{string} as
data, whereas @samp{--post-file} sends the contents of @var{file}. Other than that,
they work in exactly the same way.
Currently, @samp{--body-file} is @emph{not} for transmitting files as a whole.
Wget does not currently support @code{multipart/form-data} for transmitting data;
only @code{application/x-www-form-urlencoded}. In the future, this may be changed
so that wget sends the @samp{--body-file} as a complete file instead of sending its
contents to the server. Please be aware that Wget needs to know the contents of
BODY Data in advance, and hence the argument to @samp{--body-file} should be a
regular file. See @samp{--post-file} for a more detailed explanation.
Only one of @samp{--body-data} and @samp{--body-file} should be specified.
Wget handles these requests in the same way that it handles @samp{--post-data}
and @samp{--post-file}. If you invoke Wget with @samp{--method=POST} and the server
responds with a redirect request, then Wget will revert to a GET request during the
redirection as is explained in @samp{--post-data}.
@cindex Content-Disposition
@item --content-disposition

16
src/ChangeLog
View File

@ -1,3 +1,19 @@
2013-03-15 Darshit Shah <darnir@gmail.com>
* http.c (post_file): Rename function to body_file_send to more
accurately reflect its use.
* http.c (gethttp): Add support for --method, --body-data and
--body-file
* init.c (commands): Same.
* options.h (options): Same.
* main.c (option_data): Same.
* main.c (print_help): Add --method command.
* main.c (main): Make old --post-{data,file} commands aliases to
--method.
Add sanity checks for --method, --body-data and --body-file.
* retr.c (SUSPEND_POST_DATA): Edit Macro Definition to use body_data.
* retr.c (RESTORE_POST_DATA): Same.
2013-03-31 Gijs van Tulder <gvtulder@gmail.com>
* warc.c: Correctly write the field length in the skip length field

64
src/http.c
View File

@ -459,14 +459,14 @@ register_basic_auth_host (const char *hostname)
also be written to that file. */
static int
post_file (int sock, const char *file_name, wgint promised_size, FILE *warc_tmp)
body_file_send (int sock, const char *file_name, wgint promised_size, FILE *warc_tmp)
{
static char chunk[8192];
wgint written = 0;
int write_error;
FILE *fp;
DEBUGP (("[writing POST file %s ... ", file_name));
DEBUGP (("[writing BODY file %s ... ", file_name));
fp = fopen (file_name, "rb");
if (!fp)
@ -1726,7 +1726,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
!opt.http_keep_alive || opt.ignore_length;
/* Headers sent when using POST. */
wgint post_data_size = 0;
wgint body_data_size = 0;
bool host_lookup_failed = false;
@ -1767,6 +1767,13 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
meth = "HEAD";
else if (opt.post_file_name || opt.post_data)
meth = "POST";
else if (opt.method)
{
char *q;
for (q = opt.method; *q; ++q)
*q = c_toupper (*q);
meth = opt.method;
}
/* Use the full path, i.e. one that includes the leading slash and
the query string. E.g. if u->path is "foo/bar" and u->query is
"param=value", full_path will be "/foo/bar?param=value". */
@ -1852,25 +1859,30 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
}
}
if (opt.post_data || opt.post_file_name)
if (opt.method)
{
request_set_header (req, "Content-Type",
"application/x-www-form-urlencoded", rel_none);
if (opt.post_data)
post_data_size = strlen (opt.post_data);
else
if (opt.body_data || opt.body_file)
{
post_data_size = file_size (opt.post_file_name);
if (post_data_size == -1)
request_set_header (req, "Content-Type",
"application/x-www-form-urlencoded", rel_none);
if (opt.body_data)
body_data_size = strlen (opt.body_data);
else
{
logprintf (LOG_NOTQUIET, _("POST data file %s missing: %s\n"),
quote (opt.post_file_name), strerror (errno));
return FILEBADFILE;
body_data_size = file_size (opt.body_file);
if (body_data_size == -1)
{
logprintf (LOG_NOTQUIET, _("BODY data file %s missing: %s\n"),
quote (opt.body_file), strerror (errno));
return FILEBADFILE;
}
}
request_set_header (req, "Content-Length",
xstrdup (number_to_static_string (body_data_size)),
rel_value);
}
request_set_header (req, "Content-Length",
xstrdup (number_to_static_string (post_data_size)),
rel_value);
}
retry_with_auth:
@ -2128,28 +2140,28 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
if (write_error >= 0)
{
if (opt.post_data)
if (opt.body_data)
{
DEBUGP (("[POST data: %s]\n", opt.post_data));
write_error = fd_write (sock, opt.post_data, post_data_size, -1);
DEBUGP (("[BODY data: %s]\n", opt.body_data));
write_error = fd_write (sock, opt.body_data, body_data_size, -1);
if (write_error >= 0 && warc_tmp != NULL)
{
/* Remember end of headers / start of payload. */
warc_payload_offset = ftello (warc_tmp);
/* Write a copy of the data to the WARC record. */
int warc_tmp_written = fwrite (opt.post_data, 1, post_data_size, warc_tmp);
if (warc_tmp_written != post_data_size)
int warc_tmp_written = fwrite (opt.post_data, 1, body_data_size, warc_tmp);
if (warc_tmp_written != body_data_size)
write_error = -2;
}
}
else if (opt.post_file_name && post_data_size != 0)
}
else if (opt.body_file && body_data_size != 0)
{
if (warc_tmp != NULL)
/* Remember end of headers / start of payload. */
/* Remember end of headers / start of payload */
warc_payload_offset = ftello (warc_tmp);
write_error = post_file (sock, opt.post_file_name, post_data_size, warc_tmp);
write_error = body_file_send (sock, opt.body_file, body_data_size, warc_tmp);
}
}

6
src/init.c
View File

@ -136,6 +136,8 @@ static const struct {
{ "backups", &opt.backups, cmd_number },
{ "base", &opt.base_href, cmd_string },
{ "bindaddress", &opt.bind_address, cmd_string },
{ "bodydata", &opt.body_data, cmd_string },
{ "bodyfile", &opt.body_file, cmd_string },
#ifdef HAVE_SSL
{ "cacertificate", &opt.ca_cert, cmd_file },
#endif
@ -157,7 +159,7 @@ static const struct {
#ifdef ENABLE_DEBUG
{ "debug", &opt.debug, cmd_boolean },
#endif
{ "defaultpage", &opt.default_page, cmd_string},
{ "defaultpage", &opt.default_page, cmd_string },
{ "deleteafter", &opt.delete_after, cmd_boolean },
{ "dirprefix", &opt.dir_prefix, cmd_directory },
{ "dirstruct", NULL, cmd_spec_dirstruct },
@ -210,6 +212,7 @@ static const struct {
{ "logfile", &opt.lfilename, cmd_file },
{ "login", &opt.ftp_user, cmd_string },/* deprecated*/
{ "maxredirect", &opt.max_redirect, cmd_number },
{ "method", &opt.method, cmd_string },
{ "mirror", NULL, cmd_spec_mirror },
{ "netrc", &opt.netrc, cmd_boolean },
{ "noclobber", &opt.noclobber, cmd_boolean },
@ -1754,6 +1757,7 @@ cleanup (void)
xfree_null (opt.user);
xfree_null (opt.passwd);
xfree_null (opt.base_href);
xfree_null (opt.method);
#endif /* DEBUG_MALLOC */
}

52
src/main.c
View File

@ -167,6 +167,8 @@ static struct cmdline_option option_data[] =
{ "backups", 0, OPT_BOOLEAN, "backups", -1 },
{ "base", 'B', OPT_VALUE, "base", -1 },
{ "bind-address", 0, OPT_VALUE, "bindaddress", -1 },
{ "body-data", 0, OPT_VALUE, "bodydata", -1 },
{ "body-file", 0, OPT_VALUE, "bodyfile", -1 },
{ IF_SSL ("ca-certificate"), 0, OPT_VALUE, "cacertificate", -1 },
{ IF_SSL ("ca-directory"), 0, OPT_VALUE, "cadirectory", -1 },
{ "cache", 0, OPT_BOOLEAN, "cache", -1 },
@ -231,6 +233,7 @@ static struct cmdline_option option_data[] =
{ "load-cookies", 0, OPT_VALUE, "loadcookies", -1 },
{ "local-encoding", 0, OPT_VALUE, "localencoding", -1 },
{ "max-redirect", 0, OPT_VALUE, "maxredirect", -1 },
{ "method", 0, OPT_VALUE, "method", -1 },
{ "mirror", 'm', OPT_BOOLEAN, "mirror", -1 },
{ "no", 'n', OPT__NO, NULL, required_argument },
{ "no-clobber", 0, OPT_BOOLEAN, "noclobber", -1 },
@ -609,6 +612,12 @@ HTTP options:\n"),
--post-data=STRING use the POST method; send STRING as the data.\n"),
N_("\
--post-file=FILE use the POST method; send contents of FILE.\n"),
N_("\
--method=HTTPMethod use method \"HTTPMethod\" in the header.\n"),
N_("\
--body-data=STRING Send STRING as data. --method MUST be set.\n"),
N_("\
--body-file=FILE Send contents of FILE. --method MUST be set.\n"),
N_("\
--content-disposition honor the Content-Disposition header when\n\
choosing local file names (EXPERIMENTAL).\n"),
@ -1211,6 +1220,21 @@ main (int argc, char **argv)
if (opt.verbose == -1)
opt.verbose = !opt.quiet;
if (opt.post_data || opt.post_file_name)
{
setoptval ("method", "POST", "method");
if (opt.post_data)
{
setoptval ("bodydata", opt.post_data, "body-data");
opt.post_data = NULL;
}
else
{
setoptval ("bodyfile", opt.post_file_name, "body-file");
opt.post_file_name = NULL;
}
}
/* Sanity checks. */
if (opt.verbose && opt.quiet)
{
@ -1358,6 +1382,34 @@ for details.\n\n"));
if (!opt.rejectregex)
exit (1);
}
if (opt.post_data || opt.post_file_name)
{
if (opt.post_data && opt.post_file_name)
{
fprintf (stderr, _("You cannot specify both --post-data and --post-file.\n"));
exit (1);
}
else if (opt.method)
{
fprintf (stderr, _("You cannot use --post-data or --post-file along with --method. "
"--method expects data through --body-data and --body-file options"));
exit (1);
}
}
if (opt.body_data || opt.body_file)
{
if (!opt.method)
{
fprintf (stderr, _("You must specify a method through --method=HTTPMethod "
"to use with --body-data or --body-file.\n"));
exit (1);
}
else if (opt.body_data && opt.body_file)
{
fprintf (stderr, _("You cannot specify both --body-data and --body-file.\n"));
exit (1);
}
}
#ifdef ENABLE_IRI
if (opt.enable_iri)

3
src/options.h
View File

@ -228,6 +228,9 @@ struct options
char *post_data; /* POST query string */
char *post_file_name; /* File to post */
char *method; /* HTTP Method to use in Header */
char *body_data; /* HTTP Method Data String */
char *body_file; /* HTTP Method File */
enum {
restrict_unix,

14
src/retr.c
View File

@ -679,18 +679,20 @@ calc_rate (wgint bytes, double secs, int *units)
#define SUSPEND_POST_DATA do { \
post_data_suspended = true; \
saved_post_data = opt.post_data; \
saved_post_file_name = opt.post_file_name; \
opt.post_data = NULL; \
opt.post_file_name = NULL; \
saved_post_data = opt.body_data; \
saved_post_file_name = opt.body_file; \
opt.body_data = NULL; \
opt.body_file = NULL; \
opt.method = NULL; \
} while (0)
#define RESTORE_POST_DATA do { \
if (post_data_suspended) \
{ \
opt.post_data = saved_post_data; \
opt.post_file_name = saved_post_file_name; \
opt.body_data = saved_post_data; \
opt.body_file = saved_post_file_name; \
post_data_suspended = false; \
opt.method = "POST"; \
} \
} while (0)