moparisthebest/curl
1
0
Fork 0
mirror of https://github.com/moparisthebest/curl synced 2024-12-21 23:58:49 -05:00
Code Issues Releases Wiki Activity

Added --retry and --retry-delay first attempt with four related test cases.

Browse Source
This commit is contained in:
Daniel Stenberg 2004-10-27 21:29:55 +00:00
parent e9b3e1d031
commit 96cf615e9d
7 changed files with 338 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
CHANGES
View File

@ -6,6 +6,16 @@
Changelog Changelog
Daniel (27 October 2004)
- Added a --retry option to curl that takes a numerical option for the number
of times the operation should be retried. It is retried if a transient error
is detected or if a timeout occurred. By default, it will first wait one
second between the retries and then double the delay time between each retry
until the delay time is ten minutes which then will be the delay time
between all forthcoming retries. You can set a static delay time with
"--retry-delay [num]" where [num] is the number of seconds to wait between
each retry.
Daniel (25 October 2004) Daniel (25 October 2004)
- Tomas Pospisek filed bug report #1053287 that proved -C - and --fail on a - Tomas Pospisek filed bug report #1053287 that proved -C - and --fail on a
file that was already completely downloaded caused an error, while it file that was already completely downloaded caused an error, while it

146
src/main.c
View File

@ -362,6 +362,8 @@ static void help(void)
" -r/--range <range> Retrieve a byte range from a HTTP/1.1 or FTP server", " -r/--range <range> Retrieve a byte range from a HTTP/1.1 or FTP server",
" --random-file <file> File for reading random data from (SSL)", " --random-file <file> File for reading random data from (SSL)",
" -R/--remote-time Set the remote file's time on the local output", " -R/--remote-time Set the remote file's time on the local output",
" --retry <num> Retry request <num> times if transient problems occur",
" --retry-delay <seconds> When retrying, wait this many seconds between each",
" -s/--silent Silent mode. Don't output anything", " -s/--silent Silent mode. Don't output anything",
" -S/--show-error Show error. With -s, make curl show errors when they occur", " -S/--show-error Show error. With -s, make curl show errors when they occur",
" --socks <host[:port]> Use SOCKS5 proxy on given host + port", " --socks <host[:port]> Use SOCKS5 proxy on given host + port",
@ -445,13 +447,10 @@ struct Configurable {
char *proxy; char *proxy;
bool proxytunnel; bool proxytunnel;
long conf; long conf;
struct getout *url_list; /* point to the first node */ struct getout *url_list; /* point to the first node */
struct getout *url_last; /* point to the last/current node */ struct getout *url_last; /* point to the last/current node */
struct getout *url_get; /* point to the node to fill in URL */ struct getout *url_get; /* point to the node to fill in URL */
struct getout *url_out; /* point to the node to fill in outfile */ struct getout *url_out; /* point to the node to fill in outfile */
char *cipher_list; char *cipher_list;
char *cert; char *cert;
char *cert_type; char *cert_type;
@ -468,7 +467,6 @@ struct Configurable {
FILE *trace_stream; FILE *trace_stream;
bool trace_fopened; bool trace_fopened;
bool trace_ascii; bool trace_ascii;
long httpversion; long httpversion;
bool progressmode; bool progressmode;
bool nobuffer; bool nobuffer;
@ -480,29 +478,21 @@ struct Configurable {
bool proxyntlm; bool proxyntlm;
bool proxydigest; bool proxydigest;
bool proxybasic; bool proxybasic;
char *writeout; /* %-styled format string to output */ char *writeout; /* %-styled format string to output */
bool writeenv; /* write results to environment, if available */ bool writeenv; /* write results to environment, if available */
FILE *errors; /* if stderr redirect is requested */ FILE *errors; /* if stderr redirect is requested */
bool errors_fopened; bool errors_fopened;
struct curl_slist *quote; struct curl_slist *quote;
struct curl_slist *postquote; struct curl_slist *postquote;
struct curl_slist *prequote; struct curl_slist *prequote;
long ssl_version; long ssl_version;
long ip_version; long ip_version;
curl_TimeCond timecond; curl_TimeCond timecond;
time_t condtime; time_t condtime;
struct curl_slist *headers; struct curl_slist *headers;
struct curl_httppost *httppost; struct curl_httppost *httppost;
struct curl_httppost *last_post; struct curl_httppost *last_post;
struct curl_slist *telnet_options; struct curl_slist *telnet_options;
HttpReq httpreq; HttpReq httpreq;
/* for bandwidth limiting features: */ /* for bandwidth limiting features: */
@ -512,10 +502,11 @@ struct Configurable {
size_t lastsendsize; size_t lastsendsize;
struct timeval lastrecvtime; struct timeval lastrecvtime;
size_t lastrecvsize; size_t lastrecvsize;
bool ftp_ssl; bool ftp_ssl;
char *socks5proxy; char *socks5proxy;
bool tcp_nodelay; bool tcp_nodelay;
long req_retry; /* number of retries */
long retry_delay; /* delay between retries (in seconds) */
}; };
/* global variable to hold info about libcurl */ /* global variable to hold info about libcurl */
@ -1197,6 +1188,8 @@ static ParameterError getparameter(char *flag, /* f or -long-flag */
{"$d", "tcp-nodelay",FALSE}, {"$d", "tcp-nodelay",FALSE},
{"$e", "proxy-digest", FALSE}, {"$e", "proxy-digest", FALSE},
{"$f", "proxy-basic", FALSE}, {"$f", "proxy-basic", FALSE},
{"$g", "retry", TRUE},
{"$h", "retry-delay", TRUE},
{"0", "http1.0", FALSE}, {"0", "http1.0", FALSE},
{"1", "tlsv1", FALSE}, {"1", "tlsv1", FALSE},
{"2", "sslv2", FALSE}, {"2", "sslv2", FALSE},
@ -1532,6 +1525,14 @@ static ParameterError getparameter(char *flag, /* f or -long-flag */
case 'f': /* --proxy-basic */ case 'f': /* --proxy-basic */
config->proxybasic ^= TRUE; config->proxybasic ^= TRUE;
break; break;
case 'g': /* --retry */
if(str2num(&config->req_retry, nextarg))
return PARAM_BAD_NUMERIC;
break;
case 'h': /* --retry-delay */
if(str2num(&config->retry_delay, nextarg))
return PARAM_BAD_NUMERIC;
break;
} }
break; break;
case '#': /* added 19990617 larsa */ case '#': /* added 19990617 larsa */
@ -2299,6 +2300,8 @@ struct OutStruct {
char *filename; char *filename;
FILE *stream; FILE *stream;
struct Configurable *config; struct Configurable *config;
curl_off_t bytes; /* amount written so far */
curl_off_t init; /* original size (non-zero when appending) */
}; };
static int my_fwrite(void *buffer, size_t sz, size_t nmemb, void *stream) static int my_fwrite(void *buffer, size_t sz, size_t nmemb, void *stream)
@ -2367,6 +2370,11 @@ static int my_fwrite(void *buffer, size_t sz, size_t nmemb, void *stream)
rc = fwrite(buffer, sz, nmemb, out->stream); rc = fwrite(buffer, sz, nmemb, out->stream);
if((int)(sz * nmemb) == rc) {
/* we added this amount of data to the output */
out->bytes += (sz * nmemb);
}
if(config->nobuffer) if(config->nobuffer)
/* disable output buffering */ /* disable output buffering */
fflush(out->stream); fflush(out->stream);
@ -2731,6 +2739,9 @@ static void FindWin32CACert(struct Configurable *config,
#endif #endif
#define RETRY_SLEEP_DEFAULT 1000 /* ms */
#define RETRY_SLEEP_MAX 600000 /* ms == 10 minutes */
static int static int
operate(struct Configurable *config, int argc, char *argv[]) operate(struct Configurable *config, int argc, char *argv[])
{ {
@ -2771,6 +2782,11 @@ operate(struct Configurable *config, int argc, char *argv[])
int res = 0; int res = 0;
int i; int i;
int up; /* upload file counter within a single upload glob */ int up; /* upload file counter within a single upload glob */
int retry_sleep_default = config->retry_delay?
config->retry_delay*1000:RETRY_SLEEP_DEFAULT; /* ms */
int retry_numretries;
int retry_sleep = retry_sleep_default;
long response;
char *env; char *env;
#ifdef CURLDEBUG #ifdef CURLDEBUG
@ -2974,6 +2990,7 @@ operate(struct Configurable *config, int argc, char *argv[])
/* default output stream is stdout */ /* default output stream is stdout */
outs.stream = stdout; outs.stream = stdout;
outs.config = config; outs.config = config;
outs.bytes = 0; /* nothing written yet */
/* save outfile pattern before expansion */ /* save outfile pattern before expansion */
outfiles = urlnode->outfile?strdup(urlnode->outfile):NULL; outfiles = urlnode->outfile?strdup(urlnode->outfile):NULL;
@ -3105,6 +3122,7 @@ operate(struct Configurable *config, int argc, char *argv[])
outs.filename = outfile; outs.filename = outfile;
if(config->resume_from) { if(config->resume_from) {
outs.init = config->resume_from;
/* open file for output: */ /* open file for output: */
outs.stream=(FILE *) fopen(outfile, config->resume_from?"ab":"wb"); outs.stream=(FILE *) fopen(outfile, config->resume_from?"ab":"wb");
if (!outs.stream) { if (!outs.stream) {
@ -3471,7 +3489,107 @@ operate(struct Configurable *config, int argc, char *argv[])
curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5); curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5);
} }
res = curl_easy_perform(curl); retry_numretries = config->req_retry;
do {
res = curl_easy_perform(curl);
if(retry_numretries) {
enum {
RETRY_NO,
RETRY_TIMEOUT,
RETRY_HTTP,
RETRY_FTP,
RETRY_LAST /* not used */
} retry = RETRY_NO;
if(CURLE_OPERATION_TIMEDOUT == res)
/* retry timeout always */
retry = RETRY_TIMEOUT;
else if(CURLE_OK == res) {
/* Check for HTTP transient errors */
char *url=NULL;
curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &url);
if(url &&
curlx_strnequal(url, "http", 4)) {
/* This was HTTP(S) */
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response);
switch(response) {
case 500: /* Internal Server Error */
case 502: /* Bad Gateway */
case 503: /* Service Unavailable */
case 504: /* Gateway Timeout */
retry = RETRY_HTTP;
/*
* At this point, we have already written data to the output
* file (or terminal). If we write to a file, we must rewind
* or close/re-open the file so that the next attempt starts
* over from the beginning.
*
* TODO: similar action for the upload case. We might need
* to start over reading from a previous point if we have
* uploaded something when this was returned.
*/
break;
}
}
} /* if CURLE_OK */
else if(CURLE_FTP_USER_PASSWORD_INCORRECT == res) {
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response);
if(response/100 == 5)
/*
* This is typically when the FTP server only allows a certain
* amount of users and we are not one of them. It mostly
* returns 530 in this case, but all 5xx codes are transient.
*/
retry = RETRY_FTP;
}
if(retry) {
if(!(config->conf&CONF_MUTE)) {
static const char *m[]={NULL,
"timeout",
"HTTP error",
"FTP error"
};
fprintf(stderr, "Transient problem: %s\n"
"Will retry in %d seconds. "
"%d retries left.\n",
m[retry],
retry_sleep/1000,
retry_numretries);
}
go_sleep(retry_sleep);
retry_numretries--;
if(!config->retry_delay) {
retry_sleep *= 2;
if(retry_sleep > RETRY_SLEEP_MAX)
retry_sleep = RETRY_SLEEP_MAX;
}
if(outs.bytes && outs.filename) {
/* We have written data to a output file, we truncate file
*/
if(!(config->conf&CONF_MUTE))
fprintf(stderr, "Throwing away " CURL_FORMAT_OFF_T
" bytes\n", outs.bytes);
fflush(outs.stream);
/* truncate file at the position where we started appending */
ftruncate( fileno(outs.stream), outs.init);
/* now seek to the end of the file, the position where we
just truncated the file */
fseek(outs.stream, 0, SEEK_END);
outs.bytes = 0; /* clear for next round */
}
continue;
}
} /* if retry_numretries */
/* In all ordinary cases, just break out of loop here */
retry_sleep = retry_sleep_default;
break;
} while(1);
if((config->progressmode == CURL_PROGRESS_BAR) && if((config->progressmode == CURL_PROGRESS_BAR) &&
progressbar.calls) { progressbar.calls) {

2
tests/data/Makefile.am
View File

@ -27,7 +27,7 @@ EXTRA_DIST = test1 test108 test117 test127 test20 test27 test34 test46 \
test172 test204 test205 test173 test174 test175 test176 test177 \ test172 test204 test205 test173 test174 test175 test176 test177 \
test513 test514 test178 test179 test180 test181 test182 test183 \ test513 test514 test178 test179 test180 test181 test182 test183 \
test184 test185 test186 test187 test188 test189 test191 test192 \ test184 test185 test186 test187 test188 test189 test191 test192 \
test193 test194 test193 test194 test195 test196 test197 test198
# The following tests have been removed from the dist since they no longer # The following tests have been removed from the dist since they no longer
# work. We need to fix the test suite's FTPS server first, then bring them # work. We need to fix the test suite's FTPS server first, then bring them

30
tests/data/test195 Normal file
View File

@ -0,0 +1,30 @@
# Server-side
<reply>
</reply>
# Client-side
<client>
<server>
ftp
</server>
<name>
FTP response 530 after PASS, temporarily not allowed access
</name>
<command>
ftp://%HOSTIP:%FTPPORT/195
</command>
<file name="log/ftpserver.cmd">
REPLY PASS 530 temporarily not available
</file>
</test>
# Verify data after the test has been "shot"
<verify>
<errorcode>
10
</errorcode>
<protocol>
USER anonymous
PASS curl_by_daniel@haxx.se
</protocol>
</verify>

32
tests/data/test196 Normal file
View File

@ -0,0 +1,32 @@
# Server-side
<reply>
</reply>
# Client-side
<client>
<server>
ftp
</server>
<name>
FTP transient error, retry request once
</name>
<command>
ftp://%HOSTIP:%FTPPORT/196 --retry 1
</command>
<file name="log/ftpserver.cmd">
REPLY PASS 530 temporarily not available
</file>
</test>
# Verify data after the test has been "shot"
<verify>
<errorcode>
10
</errorcode>
<protocol>
USER anonymous
PASS curl_by_daniel@haxx.se
USER anonymous
PASS curl_by_daniel@haxx.se
</protocol>
</verify>

69
tests/data/test197 Normal file
View File

@ -0,0 +1,69 @@
#
# Server-side
<reply>
<data nocheck=1>
HTTP/1.1 503 OK swsbounce
Date: Thu, 09 Nov 2010 14:49:00 GMT
Content-Length: 21
server not available
</data>
<data1 nocheck=1>
HTTP/1.1 200 OK
Date: Thu, 09 Nov 2010 14:49:00 GMT
Content-Length: 3
Connection: close
ok
</data1>
</reply>
#
# Client-side
<client>
<server>
http
</server>
<name>
HTTP GET --retry on 503 error with output to stdout
</name>
<command>
http://%HOSTIP:%HTTPPORT/197 --retry 1000
</command>
</client>
#
# Verify data after the test has been "shot"
<verify>
<strip>
^User-Agent:.*
</strip>
<protocol>
GET /197 HTTP/1.1
Host: 127.0.0.1:%HTTPPORT
Pragma: no-cache
Accept: */*
GET /197 HTTP/1.1
Host: 127.0.0.1:%HTTPPORT
Pragma: no-cache
Accept: */*
</protocol>
<stdout>
HTTP/1.1 503 OK swsbounce
Date: Thu, 09 Nov 2010 14:49:00 GMT
Content-Length: 21
server not available
HTTP/1.1 200 OK
Date: Thu, 09 Nov 2010 14:49:00 GMT
Content-Length: 3
Connection: close
ok
</stdout>
</verify>

64
tests/data/test198 Normal file
View File

@ -0,0 +1,64 @@
#
# Server-side
<reply>
<data>
HTTP/1.1 503 OK swsbounce
Date: Thu, 09 Nov 2010 14:49:00 GMT
Content-Length: 21
server not available
</data>
<data1>
HTTP/1.1 200 OK
Date: Thu, 09 Nov 2010 14:49:00 GMT
Content-Length: 3
Connection: close
ok
</data1>
<datacheck>
HTTP/1.1 200 OK
Date: Thu, 09 Nov 2010 14:49:00 GMT
Content-Length: 3
Connection: close
ok
</datacheck>
</reply>
#
# Client-side
<client>
<server>
http
</server>
<name>
HTTP GET --retry on 503 error with output to file
</name>
<command>
http://%HOSTIP:%HTTPPORT/198 --retry 1000
</command>
</client>
#
# Verify data after the test has been "shot"
<verify>
<strip>
^User-Agent:.*
</strip>
<protocol>
GET /198 HTTP/1.1
Host: 127.0.0.1:%HTTPPORT
Pragma: no-cache
Accept: */*
GET /198 HTTP/1.1
Host: 127.0.0.1:%HTTPPORT
Pragma: no-cache
Accept: */*
</protocol>
</verify>