tool: Add option --retry-all-errors to retry on any error

The "sledgehammer" of retrying.

Closes https://github.com/curl/curl/pull/5185

docs/cmdline-opts/Makefile.inc

@@ -180,6 +180,7 @@ DPAGES =					\
   request-target.d				\
   request.d					\
   resolve.d					\
+  retry-all-errors.d				\
   retry-connrefused.d				\
   retry-delay.d					\
   retry-max-time.d				\

docs/cmdline-opts/retry-all-errors.d

@@ -0,0 +1,19 @@
+Long: retry-all-errors
+Help: Retry all errors (use with --retry) (read manpage, don't use by default)
+Added: 7.71.0
+---
+Retry on any error. This option is used together with --retry.
+
+This option is the "sledgehammer" of retrying. Do not use this option by
+default (eg in curlrc), there may be unintended consequences such as sending or
+receiving duplicate data. Do not use with redirected input or output. You'd be
+much better off handling your unique problems in shell script. Please read the
+example below.
+
+Warning: For server compatibility curl attempts to retry failed flaky transfers
+as close as possible to how they were started, but this is not possible with
+redirected input or output. For example, before retrying it removes output data
+from a failed partial transfer that was written to an output file. However this
+is not true of data redirected to a | pipe or > file, which are not reset. We
+strongly suggest don't parse or record output via redirect in combination with
+this option, since you may receive duplicate data.

src/tool_cfgable.h

@@ -223,6 +223,7 @@ struct OperationConfig {
   bool tcp_nodelay;
   bool tcp_fastopen;
   long req_retry;           /* number of retries */
+  bool retry_all_errors;    /* retry on any error */
   bool retry_connrefused;   /* set connection refused as a transient error */
   long retry_delay;         /* delay between retries (in seconds) */
   long retry_maxtime;       /* maximum time to keep retrying */

src/tool_getparam.c

@@ -197,6 +197,7 @@ static const struct LongShort aliases[]= {
   {"$Y", "suppress-connect-headers", ARG_BOOL},
   {"$Z", "compressed-ssh",           ARG_BOOL},
   {"$~", "happy-eyeballs-timeout-ms", ARG_STRING},
+  {"$!", "retry-all-errors",         ARG_BOOL},
   {"0",   "http1.0",                 ARG_NONE},
   {"01",  "http1.1",                 ARG_NONE},
   {"02",  "http2",                   ARG_NONE},
@@ -927,6 +928,9 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
         if(err)
           return err;
         break;
+      case '!': /* --retry-all-errors */
+        config->retry_all_errors = toggle;
+        break;
 
       case 'k': /* --proxy-negotiate */
         if(curlinfo->features & CURL_VERSION_SPNEGO)

src/tool_help.c

@@ -399,6 +399,8 @@ static const struct helptxt helptext[] = {
    "Retry on connection refused (use with --retry)"},
   {"    --retry-delay <seconds>",
    "Wait time between retries"},
+  {"    --retry-all-errors",
+   "Retry all errors (use with --retry) (read manpage, don't use by default)"},
   {"    --retry-max-time <seconds>",
    "Retry only within this period"},
   {"    --sasl-authzid <identity> ",

src/tool_operate.c

@@ -437,6 +437,7 @@ static CURLcode post_per_transfer(struct GlobalConfig *global,
        config->retry_maxtime*1000L)) ) {
     enum {
       RETRY_NO,
+      RETRY_ALL_ERRORS,
       RETRY_TIMEOUT,
       RETRY_CONNREFUSED,
       RETRY_HTTP,
@@ -506,11 +507,15 @@ static CURLcode post_per_transfer(struct GlobalConfig *global,
         retry = RETRY_FTP;
     }
 
+    if(result && !retry && config->retry_all_errors)
+      retry = RETRY_ALL_ERRORS;
+
     if(retry) {
       long sleeptime = 0;
       curl_off_t retry_after = 0;
       static const char * const m[]={
         NULL,
+        "(retrying all errors)",
         "timeout",
         "connection refused",
         "HTTP error",

tests/data/Makefile.inc

@@ -201,7 +201,7 @@ test1700 test1701 test1702 \
 test1800 test1801 \
 \
 test1900 test1901 test1902 test1903 test1904 test1905 test1906 test1907 \
-test1908 \
+test1908 test1909 \
 \
 test2000 test2001 test2002 test2003 test2004 test2005 test2006 test2007 \
 test2008 test2009 test2010 test2011 test2012 test2013 test2014 test2015 \

tests/data/test1909

@@ -0,0 +1,65 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+retry
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data nocheck="yes">
+HTTP/1.1 200 OK swsclose swsbounce
+Content-Length: 5
+Connection: close
+
+bbb
+</data>
+<data1>
+HTTP/1.1 200 OK
+Content-Length: 5
+Connection: close
+
+data
+</data1>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+ <name>
+HTTP GET --retry-all-errors to overcome partial transfer
+ </name>
+ <command option="no-output,no-include">
+--retry 1 --retry-all-errors -o log/outfile1909 http://%HOSTIP:%HTTPPORT/1909
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<strip>
+^User-Agent:.*
+</strip>
+<protocol>
+GET /1909 HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+Accept: */*
+
+GET /1909 HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+Accept: */*
+
+</protocol>
+
+<file1 name="log/outfile1909">
+data
+</file1>
+
+</verify>
+</testcase>