Make 504 Gateway Timeout non fatal

src/ChangeLog

@@ -1,3 +1,10 @@
+2014-11-19  Darshit Shah  <darnir@gmail.com>
+
+	* exits.c (get_status_for_err): GATEWAYTIMEOUT is a Server Error and Wget's
+	exit code should reflect that
+	* wget.h: Add GATEWAYTIMEOUT as an error
+	* http.c (http_loop): Gateway timeout errors should be non-fatal
+
 2014-11-19  Tim Ruehsen  <tim.ruehsen@gmx.de>
 
 	* openssl.c (ssl_check_certificate): Fix memory leak
@@ -21,7 +28,7 @@
 2014-11-17  Tim Ruehsen  <tim.ruehsen@gmx.de>
 
 	* http.c: use c_strncasecmp() in BEGINS_WITH macro
-          and in STARTS macro
+	and in STARTS macro
 
 2014-11-17  Tim Ruehsen  <tim.ruehsen@gmx.de>
 

src/exits.c

@@ -56,7 +56,7 @@ get_status_for_err (uerr_t err)
     case FTPNSFOD: case FTPUNKNOWNTYPE: case FTPSRVERR:
     case FTPRETRINT: case FTPRESTFAIL: case FTPNOPASV:
     case CONTNOTSUPPORTED: case RANGEERR: case RETRBADPATTERN:
-    case PROXERR:
+    case PROXERR: case GATEWAYTIMEOUT:
       return WGET_EXIT_SERVER_ERROR;
     case URLERROR: case QUOTEXC: case SSLINITFAILED: case UNKNOWNATTR:
     default:

src/http.c

@@ -130,6 +130,7 @@ static struct cookie_jar *wget_cookie_jar;
 #define HTTP_STATUS_NOT_IMPLEMENTED       501
 #define HTTP_STATUS_BAD_GATEWAY           502
 #define HTTP_STATUS_UNAVAILABLE           503
+#define HTTP_STATUS_GATEWAY_TIMEOUT       504
 
 enum rp {
   rel_none, rel_name, rel_value, rel_both
@@ -2461,6 +2462,24 @@ read_header:
         pconn.authorized = true;
     }
 
+  if (statcode == HTTP_STATUS_GATEWAY_TIMEOUT)
+    {
+      hs->len = 0;
+      hs->res = 0;
+      hs->restval = 0;
+
+      CLOSE_FINISH (sock);
+      request_free (req);
+      xfree (hs->message);
+      hs->message = NULL;
+      xfree_null (message);
+      resp_free (resp);
+      xfree (head);
+
+      return GATEWAYTIMEOUT;
+    }
+
+
   /* Determine the local filename if needed. Notice that if -O is used
    * hstat.local_file is set by http_loop to the argument of -O. */
   if (!hs->local_file)
@@ -3184,7 +3203,7 @@ Spider mode enabled. Check if remote file exists.\n"));
         {
         case HERR: case HEOF: case CONSOCKERR:
         case CONERROR: case READERR: case WRITEFAILED:
-        case RANGEERR: case FOPEN_EXCL_ERR:
+        case RANGEERR: case FOPEN_EXCL_ERR: case GATEWAYTIMEOUT:
           /* Non-fatal errors continue executing the loop, which will
              bring them to "while" statement at the end, to judge
              whether the number of tries was exceeded.  */

src/wget.h

@@ -337,7 +337,7 @@ typedef enum
   FTPOK, FTPLOGINC, FTPLOGREFUSED, FTPPORTERR, FTPSYSERR,
   FTPNSFOD, FTPUNKNOWNTYPE, FTPRERR,
   FTPSRVERR, FTPRETRINT, FTPRESTFAIL, URLERROR, FOPENERR,
-  FOPEN_EXCL_ERR, FWRITEERR, HEOF,
+  FOPEN_EXCL_ERR, FWRITEERR, HEOF, GATEWAYTIMEOUT,
   HERR, RETROK, RECLEVELEXC, WRONGCODE,
   FTPINVPASV, FTPNOPASV, CONTNOTSUPPORTED, RETRUNNEEDED, RETRFINISHED,
   READERR, TRYLIMEXC, FILEBADFILE, RANGEERR,

testenv/ChangeLog

@@ -1,7 +1,13 @@
+2014-11-19  Darshit Shah  <darnir@gmail.com>
+
+	* Test-504.py: Add new test case to show how Wget handles 504 Gateway
+	Timeouts
+	* Makefile.am: Add Test-504.py to TESTS
+
 2014-11-17  Tim Ruehsen <tim.ruehsen@gmx.de>
 
 	* server/http/http_server.py: allow case-insensitive auth-type,
-          send BasIc and DIgest to provoke Wget failures with turkish locales
+	send BasIc and DIgest to provoke Wget failures with turkish locales
 
 2014-11-15  Darshit Shah  <darnir@gmail.com>
 

testenv/Makefile.am

@@ -50,6 +50,7 @@ if HAVE_PYTHON3
     Test--https-crl.py                      \
     Test-O.py                               \
     Test-Post.py                            \
+    Test-504.py                             \
     Test--spider-r.py
 
   XFAIL_TESTS = Test-auth-both.py

testenv/Test-504.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+from sys import exit
+from test.http_test import HTTPTest
+from misc.wget_file import WgetFile
+
+"""
+    This test ensures that Wget handles a 504 Gateway Timeout response
+    correctly.
+    Since, we do not have a direct mechanism for conditionally sending responses
+    via the HTTP Server, I've used a workaround.
+    The server will always respond to a request for File1 with a 504 Gateway
+    Timeout. Using the --tries=2 option, we ensure that Wget attempts the file
+    only twice and then move on to the next file. Finally, check the exact
+    requests that the Server received and compare them, in order, to the
+    expected sequence of requests.
+
+    In this case, we expect Wget to attempt File1 twice and File2 once. If Wget
+    considered 504 as a general Server Error, it would be a fatal failure and
+    Wget would request File1 only once.
+"""
+TEST_NAME = "504 Gateway Timeouts"
+############# File Definitions ###############################################
+File1 = """All happy families are alike;
+Each unhappy family is unhappy in its own way"""
+File2 = "Anyone for chocochip cookies?"
+
+File1_rules = {
+    "Response"          : 504
+}
+
+A_File = WgetFile ("File1", File1, rules=File1_rules)
+B_File = WgetFile ("File2", File2)
+
+Request_List = [
+    [
+        "GET /File1",
+        "GET /File1",
+        "GET /File2",
+    ]
+]
+
+
+WGET_OPTIONS = "--tries=2"
+WGET_URLS = [["File1", "File2"]]
+
+Files = [[A_File, B_File]]
+
+ExpectedReturnCode = 4
+ExpectedDownloadedFiles = [B_File]
+
+################ Pre and Post Test Hooks #####################################
+pre_test = {
+    "ServerFiles"       : Files
+}
+test_options = {
+    "WgetCommands"      : WGET_OPTIONS,
+    "Urls"              : WGET_URLS
+}
+post_test = {
+    "ExpectedFiles"     : ExpectedDownloadedFiles,
+    "ExpectedRetcode"   : ExpectedReturnCode,
+    "FilesCrawled"      : Request_List
+}
+
+err = HTTPTest (
+                name=TEST_NAME,
+                pre_hook=pre_test,
+                test_params=test_options,
+                post_hook=post_test
+).begin ()
+
+exit (err)