Add a generic --method command to set a method in HTTP Requests.

Add supplementary --body-data and --body-file commands to send BODY Data.

Signed-off-by: Darshit Shah <darnir@gmail.com>

doc/ChangeLog

@@ -1,3 +1,10 @@
+2013-04-05  Darshit Shah <darnir@gmail.com>
+
+	* doc/wget.texi: Fix ambiguous wording in --post-data section. Make it
+	clear that wget does not check for the format of the post-data.
+	* doc/wget.texi: Add documentation for --method, --body-data and
+	--body-file.
+
 2012-10-08  Stefano Lattarini <stefano.lattarini@gmail.com>  (tiny change)
 
 	docs: fix errors and warnings with Texinfo 5

doc/wget.texi

@@ -1457,6 +1457,11 @@ like everything else. Wget does not currently support
 @code{application/x-www-form-urlencoded}. Only one of
 @samp{--post-data} and @samp{--post-file} should be specified.
 
+Please note that wget does not require the content to be of the form
+@code{key1=value1&key2=value2}, and neither does it test for it. Wget will
+simply transmit whatever data is provided to it. Most servers however expect
+the POST data to be in the above format when processing HTML Forms.
+
 Please be aware that Wget needs to know the size of the POST data in
 advance.  Therefore the argument to @code{--post-file} must be a regular
 file; specifying a FIFO or something like @file{/dev/stdin} won't work.
@@ -1497,6 +1502,34 @@ them (and neither will browsers) and the @file{cookies.txt} file will
 be empty.  In that case use @samp{--keep-session-cookies} along with
 @samp{--save-cookies} to force saving of session cookies.
 
+@cindex Other HTTP Methods
+@item --method=@var{HTTP-Method}
+For the purpose of RESTful scripting, Wget allows sending of other HTTP Methods
+without the need to explicitly set them using @samp{--header=Header-Line}.
+Wget will use whatever string is passed to it after @samp{--method} as the HTTP
+Method to the server.
+
+@item --body-data=@var{Data-String}
+@itemx --body-file=@var{Data-File}
+Must be set when additional data needs to be sent to the server along with the
+Method specified using @samp{--method}. @samp{--post-data} sends @var{string} as
+data, whereas @samp{--post-file} sends the contents of @var{file}. Other than that,
+they work in exactly the same way.
+
+Currently, @samp{--body-file} is @emph{not} for transmitting files as a whole.
+Wget does not currently support @code{multipart/form-data} for transmitting data;
+only @code{application/x-www-form-urlencoded}. In the future, this may be changed
+so that wget sends the @samp{--body-file} as a complete file instead of sending its
+contents to the server. Please be aware that Wget needs to know the contents of
+BODY Data in advance, and hence the argument to @samp{--body-file} should be a
+regular file. See @samp{--post-file} for a more detailed explanation.
+Only one of @samp{--body-data} and @samp{--body-file} should be specified.
+
+Wget handles these requests in the same way that it handles @samp{--post-data}
+and @samp{--post-file}. If you invoke Wget with @samp{--method=POST} and the server
+responds with a redirect request, then Wget will revert to a GET request during the
+redirection as is explained in @samp{--post-data}.
+
 @cindex Content-Disposition
 @item --content-disposition
 

src/ChangeLog

@@ -1,3 +1,19 @@
+2013-03-15  Darshit Shah <darnir@gmail.com>
+
+	* http.c (post_file): Rename function to body_file_send to more
+	accurately reflect its use.
+	* http.c (gethttp): Add support for --method, --body-data and
+	--body-file
+	* init.c (commands): Same.
+	* options.h (options): Same.
+	* main.c (option_data): Same.
+	* main.c (print_help): Add --method command.
+	* main.c (main): Make old --post-{data,file} commands aliases to
+	--method.
+	Add sanity checks for --method, --body-data and --body-file.
+	* retr.c (SUSPEND_POST_DATA): Edit Macro Definition to use body_data.
+	* retr.c (RESTORE_POST_DATA): Same.
+
 2013-03-31  Gijs van Tulder  <gvtulder@gmail.com>
 
 	* warc.c: Correctly write the field length in the skip length field

src/http.c

@@ -459,14 +459,14 @@ register_basic_auth_host (const char *hostname)
    also be written to that file.  */
 
 static int
-post_file (int sock, const char *file_name, wgint promised_size, FILE *warc_tmp)
+body_file_send (int sock, const char *file_name, wgint promised_size, FILE *warc_tmp)
 {
   static char chunk[8192];
   wgint written = 0;
   int write_error;
   FILE *fp;
 
-  DEBUGP (("[writing POST file %s ... ", file_name));
+  DEBUGP (("[writing BODY file %s ... ", file_name));
 
   fp = fopen (file_name, "rb");
   if (!fp)
@@ -1726,7 +1726,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
     !opt.http_keep_alive || opt.ignore_length;
 
   /* Headers sent when using POST. */
-  wgint post_data_size = 0;
+  wgint body_data_size = 0;
 
   bool host_lookup_failed = false;
 
@@ -1767,6 +1767,13 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
       meth = "HEAD";
     else if (opt.post_file_name || opt.post_data)
       meth = "POST";
+    else if (opt.method)
+      {
+        char *q;
+        for (q = opt.method; *q; ++q)
+          *q = c_toupper (*q);
+        meth = opt.method;
+      }
     /* Use the full path, i.e. one that includes the leading slash and
        the query string.  E.g. if u->path is "foo/bar" and u->query is
        "param=value", full_path will be "/foo/bar?param=value".  */
@@ -1852,26 +1859,31 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
         }
     }
 
-  if (opt.post_data || opt.post_file_name)
+  if (opt.method)
+    {
+
+      if (opt.body_data || opt.body_file)
         {
           request_set_header (req, "Content-Type",
                               "application/x-www-form-urlencoded", rel_none);
-      if (opt.post_data)
+
-        post_data_size = strlen (opt.post_data);
+          if (opt.body_data)
+            body_data_size = strlen (opt.body_data);
           else
             {
-          post_data_size = file_size (opt.post_file_name);
+              body_data_size = file_size (opt.body_file);
-          if (post_data_size == -1)
+              if (body_data_size == -1)
                 {
-              logprintf (LOG_NOTQUIET, _("POST data file %s missing: %s\n"),
+                  logprintf (LOG_NOTQUIET, _("BODY data file %s missing: %s\n"),
-                         quote (opt.post_file_name), strerror (errno));
+                             quote (opt.body_file), strerror (errno));
                   return FILEBADFILE;
                 }
             }
           request_set_header (req, "Content-Length",
-                          xstrdup (number_to_static_string (post_data_size)),
+                              xstrdup (number_to_static_string (body_data_size)),
                               rel_value);
         }
+    }
 
  retry_with_auth:
   /* We need to come back here when the initial attempt to retrieve
@@ -2128,28 +2140,28 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy,
 
   if (write_error >= 0)
     {
-      if (opt.post_data)
+      if (opt.body_data)
         {
-          DEBUGP (("[POST data: %s]\n", opt.post_data));
+          DEBUGP (("[BODY data: %s]\n", opt.body_data));
-          write_error = fd_write (sock, opt.post_data, post_data_size, -1);
+          write_error = fd_write (sock, opt.body_data, body_data_size, -1);
           if (write_error >= 0 && warc_tmp != NULL)
             {
               /* Remember end of headers / start of payload. */
               warc_payload_offset = ftello (warc_tmp);
 
               /* Write a copy of the data to the WARC record. */
-              int warc_tmp_written = fwrite (opt.post_data, 1, post_data_size, warc_tmp);
+              int warc_tmp_written = fwrite (opt.post_data, 1, body_data_size, warc_tmp);
-              if (warc_tmp_written != post_data_size)
+              if (warc_tmp_written != body_data_size)
                 write_error = -2;
             }
          }
-      else if (opt.post_file_name && post_data_size != 0)
+      else if (opt.body_file && body_data_size != 0)
         {
           if (warc_tmp != NULL)
-            /* Remember end of headers / start of payload. */
+            /* Remember end of headers / start of payload */
             warc_payload_offset = ftello (warc_tmp);
 
-          write_error = post_file (sock, opt.post_file_name, post_data_size, warc_tmp);
+          write_error = body_file_send (sock, opt.body_file, body_data_size, warc_tmp);
         }
     }
 

src/init.c

@@ -136,6 +136,8 @@ static const struct {
   { "backups",          &opt.backups,           cmd_number },
   { "base",             &opt.base_href,         cmd_string },
   { "bindaddress",      &opt.bind_address,      cmd_string },
+  { "bodydata",         &opt.body_data,         cmd_string },
+  { "bodyfile",         &opt.body_file,         cmd_string },
 #ifdef HAVE_SSL
   { "cacertificate",    &opt.ca_cert,           cmd_file },
 #endif
@@ -210,6 +212,7 @@ static const struct {
   { "logfile",          &opt.lfilename,         cmd_file },
   { "login",            &opt.ftp_user,          cmd_string },/* deprecated*/
   { "maxredirect",      &opt.max_redirect,      cmd_number },
+  { "method",           &opt.method,            cmd_string },
   { "mirror",           NULL,                   cmd_spec_mirror },
   { "netrc",            &opt.netrc,             cmd_boolean },
   { "noclobber",        &opt.noclobber,         cmd_boolean },
@@ -1754,6 +1757,7 @@ cleanup (void)
   xfree_null (opt.user);
   xfree_null (opt.passwd);
   xfree_null (opt.base_href);
+  xfree_null (opt.method);
 
 #endif /* DEBUG_MALLOC */
 }

src/main.c

@@ -167,6 +167,8 @@ static struct cmdline_option option_data[] =
     { "backups", 0, OPT_BOOLEAN, "backups", -1 },
     { "base", 'B', OPT_VALUE, "base", -1 },
     { "bind-address", 0, OPT_VALUE, "bindaddress", -1 },
+    { "body-data", 0, OPT_VALUE, "bodydata", -1 },
+    { "body-file", 0, OPT_VALUE, "bodyfile", -1 },
     { IF_SSL ("ca-certificate"), 0, OPT_VALUE, "cacertificate", -1 },
     { IF_SSL ("ca-directory"), 0, OPT_VALUE, "cadirectory", -1 },
     { "cache", 0, OPT_BOOLEAN, "cache", -1 },
@@ -231,6 +233,7 @@ static struct cmdline_option option_data[] =
     { "load-cookies", 0, OPT_VALUE, "loadcookies", -1 },
     { "local-encoding", 0, OPT_VALUE, "localencoding", -1 },
     { "max-redirect", 0, OPT_VALUE, "maxredirect", -1 },
+    { "method", 0, OPT_VALUE, "method", -1 },
     { "mirror", 'm', OPT_BOOLEAN, "mirror", -1 },
     { "no", 'n', OPT__NO, NULL, required_argument },
     { "no-clobber", 0, OPT_BOOLEAN, "noclobber", -1 },
@@ -609,6 +612,12 @@ HTTP options:\n"),
        --post-data=STRING      use the POST method; send STRING as the data.\n"),
     N_("\
        --post-file=FILE        use the POST method; send contents of FILE.\n"),
+    N_("\
+       --method=HTTPMethod     use method \"HTTPMethod\" in the header.\n"),
+    N_("\
+       --body-data=STRING      Send STRING as data. --method MUST be set.\n"),
+    N_("\
+       --body-file=FILE        Send contents of FILE. --method MUST be set.\n"),
     N_("\
        --content-disposition   honor the Content-Disposition header when\n\
                                choosing local file names (EXPERIMENTAL).\n"),
@@ -1211,6 +1220,21 @@ main (int argc, char **argv)
   if (opt.verbose == -1)
     opt.verbose = !opt.quiet;
 
+  if (opt.post_data || opt.post_file_name)
+    {
+        setoptval ("method", "POST", "method");
+        if (opt.post_data)
+          {
+            setoptval ("bodydata", opt.post_data, "body-data");
+            opt.post_data = NULL;
+          }
+        else
+          {
+            setoptval ("bodyfile", opt.post_file_name, "body-file");
+            opt.post_file_name = NULL;
+          }
+    }
+
   /* Sanity checks.  */
   if (opt.verbose && opt.quiet)
     {
@@ -1358,6 +1382,34 @@ for details.\n\n"));
       if (!opt.rejectregex)
         exit (1);
     }
+  if (opt.post_data || opt.post_file_name)
+    {
+      if (opt.post_data && opt.post_file_name)
+        {
+          fprintf (stderr, _("You cannot specify both --post-data and --post-file.\n"));
+          exit (1);
+        }
+      else if (opt.method)
+        {
+          fprintf (stderr, _("You cannot use --post-data or --post-file along with --method. "
+                             "--method expects data through --body-data and --body-file options"));
+          exit (1);
+        }
+    }
+  if (opt.body_data || opt.body_file)
+    {
+      if (!opt.method)
+        {
+          fprintf (stderr, _("You must specify a method through --method=HTTPMethod "
+                              "to use with --body-data or --body-file.\n"));
+          exit (1);
+        }
+      else if (opt.body_data && opt.body_file)
+        {
+          fprintf (stderr, _("You cannot specify both --body-data and --body-file.\n"));
+          exit (1);
+        }
+    }
 
 #ifdef ENABLE_IRI
   if (opt.enable_iri)

src/options.h

@@ -228,6 +228,9 @@ struct options
 
   char *post_data;		/* POST query string */
   char *post_file_name;		/* File to post */
+  char *method;                 /* HTTP Method to use in Header */
+  char *body_data;              /* HTTP Method Data String */
+  char *body_file;              /* HTTP Method File */
 
   enum {
     restrict_unix,

src/retr.c

@@ -679,18 +679,20 @@ calc_rate (wgint bytes, double secs, int *units)
 
 #define SUSPEND_POST_DATA do {                  \
   post_data_suspended = true;                   \
-  saved_post_data = opt.post_data;              \
+  saved_post_data = opt.body_data;              \
-  saved_post_file_name = opt.post_file_name;    \
+  saved_post_file_name = opt.body_file;         \
-  opt.post_data = NULL;                         \
+  opt.body_data = NULL;                         \
-  opt.post_file_name = NULL;                    \
+  opt.body_file = NULL;                         \
+  opt.method = NULL;                            \
 } while (0)
 
 #define RESTORE_POST_DATA do {                          \
   if (post_data_suspended)                              \
     {                                                   \
-      opt.post_data = saved_post_data;                  \
+      opt.body_data = saved_post_data;                  \
-      opt.post_file_name = saved_post_file_name;        \
+      opt.body_file = saved_post_file_name;             \
       post_data_suspended = false;                      \
+      opt.method = "POST";                              \
     }                                                   \
 } while (0)