moparisthebest/curl
1
0
Fork 0
mirror of https://github.com/moparisthebest/curl synced 2024-11-16 06:25:03 -05:00
Code Issues Releases Wiki Activity

curl: create easy handles on-demand and not ahead of time

Browse Source 
This should again enable crazy-large download ranges of the style
[1-10000000] that otherwise easily ran out of memory starting in 7.66.0
when this new handle allocating scheme was introduced.

Reported-by: Peter Sumatra
Fixes #4393
Closes #4438
This commit is contained in:
Daniel Stenberg 2019-09-23 17:11:22 +02:00
parent c124e6b3c0
commit e59371a493
No known key found for this signature in database
GPG Key ID: 5CC908FDB71E12C2
7 changed files with 307 additions and 218 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/tool_cfgable.h
View File

@ -22,11 +22,9 @@
* *
***************************************************************************/ ***************************************************************************/
#include "tool_setup.h" #include "tool_setup.h"
#include "tool_sdecls.h" #include "tool_sdecls.h"
#include "tool_metalink.h" #include "tool_metalink.h"
#include "tool_urlglob.h"
#include "tool_formparse.h" #include "tool_formparse.h"
typedef enum { typedef enum {
@ -37,6 +35,20 @@ typedef enum {
struct GlobalConfig; struct GlobalConfig;
struct State {
struct getout *urlnode;
URLGlob *inglob;
URLGlob *urls;
char *outfiles;
char *httpgetfields;
char *uploadfile;
unsigned long infilenum; /* number of files to upload */
unsigned long up; /* upload file counter within a single upload glob */
unsigned long urlnum; /* how many iterations this single URL has with ranges
etc */
unsigned long li;
};
struct OperationConfig { struct OperationConfig {
bool remote_time; bool remote_time;
char *random_file; char *random_file;
@ -262,6 +274,7 @@ struct OperationConfig {
struct GlobalConfig *global; struct GlobalConfig *global;
struct OperationConfig *prev; struct OperationConfig *prev;
struct OperationConfig *next; /* Always last in the struct */ struct OperationConfig *next; /* Always last in the struct */
struct State state; /* for create_transfer() */
}; };
struct GlobalConfig { struct GlobalConfig {

12
src/tool_metalink.c
View File

@ -984,12 +984,14 @@ void delete_metalinkfile(metalinkfile *mlfile)
void clean_metalink(struct OperationConfig *config) void clean_metalink(struct OperationConfig *config)
{ {
while(config->metalinkfile_list) { if(config) {
metalinkfile *mlfile = config->metalinkfile_list; while(config->metalinkfile_list) {
config->metalinkfile_list = config->metalinkfile_list->next; metalinkfile *mlfile = config->metalinkfile_list;
delete_metalinkfile(mlfile); config->metalinkfile_list = config->metalinkfile_list->next;
delete_metalinkfile(mlfile);
}
config->metalinkfile_last = 0;
} }
config->metalinkfile_last = 0;
} }
void metalink_cleanup(void) void metalink_cleanup(void)

463
src/tool_operate.c
View File

@ -103,10 +103,14 @@ CURLcode curl_easy_perform_ev(CURL *easy);
"this situation and\nhow to fix it, please visit the web page mentioned " \ "this situation and\nhow to fix it, please visit the web page mentioned " \
"above.\n" "above.\n"
static CURLcode create_transfers(struct GlobalConfig *global, static CURLcode single_transfer(struct GlobalConfig *global,
struct OperationConfig *config, struct OperationConfig *config,
CURLSH *share, CURLSH *share,
bool capath_from_env); bool capath_from_env,
bool *added);
static CURLcode get_transfer(struct GlobalConfig *global,
CURLSH *share,
bool *added);
static bool is_fatal_error(CURLcode code) static bool is_fatal_error(CURLcode code)
{ {
@ -316,7 +320,6 @@ static CURLcode pre_transfer(struct GlobalConfig *global,
my_setopt(per->curl, CURLOPT_INFILESIZE_LARGE, uploadfilesize); my_setopt(per->curl, CURLOPT_INFILESIZE_LARGE, uploadfilesize);
per->input.fd = per->infd; per->input.fd = per->infd;
} }
show_error:
return result; return result;
} }
@ -324,7 +327,6 @@ static CURLcode pre_transfer(struct GlobalConfig *global,
* Call this after a transfer has completed. * Call this after a transfer has completed.
*/ */
static CURLcode post_transfer(struct GlobalConfig *global, static CURLcode post_transfer(struct GlobalConfig *global,
CURLSH *share,
struct per_transfer *per, struct per_transfer *per,
CURLcode result, CURLcode result,
bool *retryp) bool *retryp)
@ -333,6 +335,9 @@ static CURLcode post_transfer(struct GlobalConfig *global,
CURL *curl = per->curl; CURL *curl = per->curl;
struct OperationConfig *config = per->config; struct OperationConfig *config = per->config;
if(!curl || !config)
return result;
*retryp = FALSE; *retryp = FALSE;
if(per->infdopen) if(per->infdopen)
@ -401,7 +406,6 @@ static CURLcode post_transfer(struct GlobalConfig *global,
else if(rv == -1) else if(rv == -1)
fprintf(config->global->errors, "Metalink: parsing (%s) FAILED\n", fprintf(config->global->errors, "Metalink: parsing (%s) FAILED\n",
per->this_url); per->this_url);
result = create_transfers(global, config, share, FALSE);
} }
else if(per->metalink && result == CURLE_OK && !per->metalink_next_res) { else if(per->metalink && result == CURLE_OK && !per->metalink_next_res) {
int rv; int rv;
@ -410,8 +414,6 @@ static CURLcode post_transfer(struct GlobalConfig *global,
if(!rv) if(!rv)
per->metalink_next_res = 1; per->metalink_next_res = 1;
} }
#else
(void)share;
#endif /* USE_METALINK */ #endif /* USE_METALINK */
#ifdef USE_METALINK #ifdef USE_METALINK
@ -651,60 +653,81 @@ static CURLcode post_transfer(struct GlobalConfig *global,
return CURLE_OK; return CURLE_OK;
} }
/* go through the list of URLs and configs and add transfers */ static void single_transfer_cleanup(struct OperationConfig *config)
{
if(config) {
struct State *state = &config->state;
if(state->urls) {
/* Free list of remaining URLs */
glob_cleanup(state->urls);
state->urls = NULL;
}
Curl_safefree(state->outfiles);
Curl_safefree(state->httpgetfields);
Curl_safefree(state->uploadfile);
if(state->inglob) {
/* Free list of globbed upload files */
glob_cleanup(state->inglob);
state->inglob = NULL;
}
}
}
static CURLcode create_transfers(struct GlobalConfig *global, /* create the next (singular) transfer */
struct OperationConfig *config,
CURLSH *share, static CURLcode single_transfer(struct GlobalConfig *global,
bool capath_from_env) struct OperationConfig *config,
CURLSH *share,
bool capath_from_env,
bool *added)
{ {
CURLcode result = CURLE_OK; CURLcode result = CURLE_OK;
struct getout *urlnode; struct getout *urlnode;
metalinkfile *mlfile_last = NULL; metalinkfile *mlfile_last = NULL;
bool orig_noprogress = global->noprogress; bool orig_noprogress = global->noprogress;
bool orig_isatty = global->isatty; bool orig_isatty = global->isatty;
char *httpgetfields = NULL; struct State *state = &config->state;
char *httpgetfields = state->httpgetfields;
*added = FALSE; /* not yet */
if(config->postfields) { if(config->postfields) {
if(config->use_httpget) { if(config->use_httpget) {
/* Use the postfields data for a http get */
httpgetfields = strdup(config->postfields);
Curl_safefree(config->postfields);
if(!httpgetfields) { if(!httpgetfields) {
helpf(global->errors, "out of memory\n"); /* Use the postfields data for a http get */
result = CURLE_OUT_OF_MEMORY; httpgetfields = state->httpgetfields = strdup(config->postfields);
goto quit_curl; Curl_safefree(config->postfields);
} if(!httpgetfields) {
if(SetHTTPrequest(config, helpf(global->errors, "out of memory\n");
(config->no_body?HTTPREQ_HEAD:HTTPREQ_GET), result = CURLE_OUT_OF_MEMORY;
&config->httpreq)) { }
result = CURLE_FAILED_INIT; else if(SetHTTPrequest(config,
goto quit_curl; (config->no_body?HTTPREQ_HEAD:HTTPREQ_GET),
&config->httpreq)) {
result = CURLE_FAILED_INIT;
}
} }
} }
else { else {
if(SetHTTPrequest(config, HTTPREQ_SIMPLEPOST, &config->httpreq)) { if(SetHTTPrequest(config, HTTPREQ_SIMPLEPOST, &config->httpreq))
result = CURLE_FAILED_INIT; result = CURLE_FAILED_INIT;
goto quit_curl;
}
} }
if(result)
return result;
}
if(!state->urlnode) {
/* first time caller, setup things */
state->urlnode = config->url_list;
state->infilenum = 1;
} }
for(urlnode = config->url_list; urlnode; urlnode = urlnode->next) { while(config->state.urlnode) {
unsigned long li;
unsigned long up; /* upload file counter within a single upload glob */
char *infiles; /* might be a glob pattern */ char *infiles; /* might be a glob pattern */
char *outfiles; URLGlob *inglob = state->inglob;
unsigned long infilenum;
URLGlob *inglob;
bool metalink = FALSE; /* metalink download? */ bool metalink = FALSE; /* metalink download? */
metalinkfile *mlfile; metalinkfile *mlfile;
metalink_resource *mlres; metalink_resource *mlres;
outfiles = NULL; urlnode = config->state.urlnode;
infilenum = 1;
inglob = NULL;
if(urlnode->flags & GETOUT_METALINK) { if(urlnode->flags & GETOUT_METALINK) {
metalink = 1; metalink = 1;
if(mlfile_last == NULL) { if(mlfile_last == NULL) {
@ -727,13 +750,15 @@ static CURLcode create_transfers(struct GlobalConfig *global,
Curl_safefree(urlnode->outfile); Curl_safefree(urlnode->outfile);
Curl_safefree(urlnode->infile); Curl_safefree(urlnode->infile);
urlnode->flags = 0; urlnode->flags = 0;
config->state.urlnode = urlnode->next;
state->up = 0;
continue; /* next URL please */ continue; /* next URL please */
} }
/* save outfile pattern before expansion */ /* save outfile pattern before expansion */
if(urlnode->outfile) { if(urlnode->outfile && !state->outfiles) {
outfiles = strdup(urlnode->outfile); state->outfiles = strdup(urlnode->outfile);
if(!outfiles) { if(!state->outfiles) {
helpf(global->errors, "out of memory\n"); helpf(global->errors, "out of memory\n");
result = CURLE_OUT_OF_MEMORY; result = CURLE_OUT_OF_MEMORY;
break; break;
@ -742,88 +767,95 @@ static CURLcode create_transfers(struct GlobalConfig *global,
infiles = urlnode->infile; infiles = urlnode->infile;
if(!config->globoff && infiles) { if(!config->globoff && infiles && !inglob) {
/* Unless explicitly shut off */ /* Unless explicitly shut off */
result = glob_url(&inglob, infiles, &infilenum, result = glob_url(&inglob, infiles, &state->infilenum,
global->showerror?global->errors:NULL); global->showerror?global->errors:NULL);
if(result) { if(result)
Curl_safefree(outfiles);
break; break;
} config->state.inglob = inglob;
} }
/* Here's the loop for uploading multiple files within the same {
single globbed string. If no upload, we enter the loop once anyway. */
for(up = 0 ; up < infilenum; up++) {
char *uploadfile; /* a single file, never a glob */
int separator; int separator;
URLGlob *urls;
unsigned long urlnum; unsigned long urlnum;
uploadfile = NULL; if(!state->up && !infiles)
urls = NULL;
urlnum = 0;
if(!up && !infiles)
Curl_nop_stmt; Curl_nop_stmt;
else { else {
if(inglob) { if(!state->uploadfile) {
result = glob_next_url(&uploadfile, inglob); if(inglob) {
if(result == CURLE_OUT_OF_MEMORY) result = glob_next_url(&state->uploadfile, inglob);
helpf(global->errors, "out of memory\n"); if(result == CURLE_OUT_OF_MEMORY)
} helpf(global->errors, "out of memory\n");
else if(!up) { }
uploadfile = strdup(infiles); else if(!state->up) {
if(!uploadfile) { state->uploadfile = strdup(infiles);
helpf(global->errors, "out of memory\n"); if(!state->uploadfile) {
result = CURLE_OUT_OF_MEMORY; helpf(global->errors, "out of memory\n");
result = CURLE_OUT_OF_MEMORY;
}
} }
} }
if(!uploadfile) if(result)
break; break;
} }
if(metalink) { if(!state->urlnum) {
/* For Metalink download, we don't use glob. Instead we use if(metalink) {
the number of resources as urlnum. */ /* For Metalink download, we don't use glob. Instead we use
urlnum = count_next_metalink_resource(mlfile); the number of resources as urlnum. */
} urlnum = count_next_metalink_resource(mlfile);
else if(!config->globoff) {
/* Unless explicitly shut off, we expand '{...}' and '[...]'
expressions and return total number of URLs in pattern set */
result = glob_url(&urls, urlnode->url, &urlnum,
global->showerror?global->errors:NULL);
if(result) {
Curl_safefree(uploadfile);
break;
} }
else if(!config->globoff) {
/* Unless explicitly shut off, we expand '{...}' and '[...]'
expressions and return total number of URLs in pattern set */
result = glob_url(&state->urls, urlnode->url, &state->urlnum,
global->showerror?global->errors:NULL);
if(result)
break;
urlnum = state->urlnum;
}
else
urlnum = 1; /* without globbing, this is a single URL */
} }
else else
urlnum = 1; /* without globbing, this is a single URL */ urlnum = state->urlnum;
/* if multiple files extracted to stdout, insert separators! */ /* if multiple files extracted to stdout, insert separators! */
separator = ((!outfiles || !strcmp(outfiles, "-")) && urlnum > 1); separator = ((!state->outfiles ||
!strcmp(state->outfiles, "-")) && urlnum > 1);
/* Here's looping around each globbed URL */ /* Here's looping around each globbed URL */
for(li = 0 ; li < urlnum; li++) {
if(state->li >= urlnum) {
state->li = 0;
state->up++;
}
if(state->up < state->infilenum) {
struct per_transfer *per; struct per_transfer *per;
struct OutStruct *outs; struct OutStruct *outs;
struct InStruct *input; struct InStruct *input;
struct OutStruct *heads; struct OutStruct *heads;
struct HdrCbData *hdrcbdata = NULL; struct HdrCbData *hdrcbdata = NULL;
CURL *curl = curl_easy_init(); CURL *curl = curl_easy_init();
result = add_transfer(&per); result = add_transfer(&per);
if(result || !curl) { if(result || !curl) {
free(uploadfile);
curl_easy_cleanup(curl); curl_easy_cleanup(curl);
result = CURLE_OUT_OF_MEMORY; result = CURLE_OUT_OF_MEMORY;
goto show_error; break;
} }
if(state->uploadfile) {
per->uploadfile = strdup(state->uploadfile);
if(!per->uploadfile) {
curl_easy_cleanup(curl);
result = CURLE_OUT_OF_MEMORY;
break;
}
}
*added = TRUE;
per->config = config; per->config = config;
per->curl = curl; per->curl = curl;
per->uploadfile = uploadfile;
/* default headers output stream is stdout */ /* default headers output stream is stdout */
heads = &per->heads; heads = &per->heads;
@ -838,7 +870,7 @@ static CURLcode create_transfers(struct GlobalConfig *global,
if(!newfile) { if(!newfile) {
warnf(config->global, "Failed to open %s\n", config->headerfile); warnf(config->global, "Failed to open %s\n", config->headerfile);
result = CURLE_WRITE_ERROR; result = CURLE_WRITE_ERROR;
goto quit_curl; break;
} }
else { else {
heads->filename = config->headerfile; heads->filename = config->headerfile;
@ -873,26 +905,26 @@ static CURLcode create_transfers(struct GlobalConfig *global,
per->outfile = strdup(mlfile->filename); per->outfile = strdup(mlfile->filename);
if(!per->outfile) { if(!per->outfile) {
result = CURLE_OUT_OF_MEMORY; result = CURLE_OUT_OF_MEMORY;
goto show_error; break;
} }
per->this_url = strdup(mlres->url); per->this_url = strdup(mlres->url);
if(!per->this_url) { if(!per->this_url) {
result = CURLE_OUT_OF_MEMORY; result = CURLE_OUT_OF_MEMORY;
goto show_error; break;
} }
per->mlfile = mlfile; per->mlfile = mlfile;
} }
else { else {
if(urls) { if(state->urls) {
result = glob_next_url(&per->this_url, urls); result = glob_next_url(&per->this_url, state->urls);
if(result) if(result)
goto show_error; break;
} }
else if(!li) { else if(!state->li) {
per->this_url = strdup(urlnode->url); per->this_url = strdup(urlnode->url);
if(!per->this_url) { if(!per->this_url) {
result = CURLE_OUT_OF_MEMORY; result = CURLE_OUT_OF_MEMORY;
goto show_error; break;
} }
} }
else else
@ -900,11 +932,11 @@ static CURLcode create_transfers(struct GlobalConfig *global,
if(!per->this_url) if(!per->this_url)
break; break;
if(outfiles) { if(state->outfiles) {
per->outfile = strdup(outfiles); per->outfile = strdup(state->outfiles);
if(!per->outfile) { if(!per->outfile) {
result = CURLE_OUT_OF_MEMORY; result = CURLE_OUT_OF_MEMORY;
goto show_error; break;
} }
} }
} }
@ -922,22 +954,22 @@ static CURLcode create_transfers(struct GlobalConfig *global,
/* extract the file name from the URL */ /* extract the file name from the URL */
result = get_url_file_name(&per->outfile, per->this_url); result = get_url_file_name(&per->outfile, per->this_url);
if(result) if(result)
goto show_error; break;
if(!*per->outfile && !config->content_disposition) { if(!*per->outfile && !config->content_disposition) {
helpf(global->errors, "Remote file name has no length!\n"); helpf(global->errors, "Remote file name has no length!\n");
result = CURLE_WRITE_ERROR; result = CURLE_WRITE_ERROR;
goto quit_urls; break;
} }
} }
else if(urls) { else if(state->urls) {
/* fill '#1' ... '#9' terms from URL pattern */ /* fill '#1' ... '#9' terms from URL pattern */
char *storefile = per->outfile; char *storefile = per->outfile;
result = glob_match_url(&per->outfile, storefile, urls); result = glob_match_url(&per->outfile, storefile, state->urls);
Curl_safefree(storefile); Curl_safefree(storefile);
if(result) { if(result) {
/* bad globbing */ /* bad globbing */
warnf(config->global, "bad output glob!\n"); warnf(config->global, "bad output glob!\n");
goto quit_urls; break;
} }
} }
@ -947,11 +979,8 @@ static CURLcode create_transfers(struct GlobalConfig *global,
if(config->create_dirs || metalink) { if(config->create_dirs || metalink) {
result = create_dir_hierarchy(per->outfile, global->errors); result = create_dir_hierarchy(per->outfile, global->errors);
/* create_dir_hierarchy shows error upon CURLE_WRITE_ERROR */ /* create_dir_hierarchy shows error upon CURLE_WRITE_ERROR */
if(result == CURLE_WRITE_ERROR) if(result)
goto quit_urls; break;
if(result) {
goto show_error;
}
} }
if((urlnode->flags & GETOUT_USEREMOTE) if((urlnode->flags & GETOUT_USEREMOTE)
@ -986,7 +1015,7 @@ static CURLcode create_transfers(struct GlobalConfig *global,
if(!file) { if(!file) {
helpf(global->errors, "Can't open '%s'!\n", per->outfile); helpf(global->errors, "Can't open '%s'!\n", per->outfile);
result = CURLE_WRITE_ERROR; result = CURLE_WRITE_ERROR;
goto quit_urls; break;
} }
outs->fopened = TRUE; outs->fopened = TRUE;
outs->stream = file; outs->stream = file;
@ -1006,7 +1035,7 @@ static CURLcode create_transfers(struct GlobalConfig *global,
char *nurl = add_file_name_to_url(per->this_url, per->uploadfile); char *nurl = add_file_name_to_url(per->this_url, per->uploadfile);
if(!nurl) { if(!nurl) {
result = CURLE_OUT_OF_MEMORY; result = CURLE_OUT_OF_MEMORY;
goto show_error; break;
} }
per->this_url = nurl; per->this_url = nurl;
} }
@ -1065,7 +1094,7 @@ static CURLcode create_transfers(struct GlobalConfig *global,
if(urlnum > 1 && !global->mute) { if(urlnum > 1 && !global->mute) {
per->separator_err = per->separator_err =
aprintf("\n[%lu/%lu]: %s --> %s", aprintf("\n[%lu/%lu]: %s --> %s",
li + 1, urlnum, per->this_url, state->li + 1, urlnum, per->this_url,
per->outfile ? per->outfile : "<stdout>"); per->outfile ? per->outfile : "<stdout>");
if(separator) if(separator)
per->separator = aprintf("%s%s", CURLseparator, per->this_url); per->separator = aprintf("%s%s", CURLseparator, per->this_url);
@ -1103,7 +1132,7 @@ static CURLcode create_transfers(struct GlobalConfig *global,
if(!urlbuffer) { if(!urlbuffer) {
result = CURLE_OUT_OF_MEMORY; result = CURLE_OUT_OF_MEMORY;
goto show_error; break;
} }
Curl_safefree(per->this_url); /* free previous URL */ Curl_safefree(per->this_url); /* free previous URL */
@ -1124,11 +1153,10 @@ static CURLcode create_transfers(struct GlobalConfig *global,
config->terminal_binary_ok = config->terminal_binary_ok =
(per->outfile && !strcmp(per->outfile, "-")); (per->outfile && !strcmp(per->outfile, "-"));
/* avoid having this setopt added to the --libcurl source /* Avoid having this setopt added to the --libcurl source output. */
output */
result = curl_easy_setopt(curl, CURLOPT_SHARE, share); result = curl_easy_setopt(curl, CURLOPT_SHARE, share);
if(result) if(result)
goto show_error; break;
if(!config->tcp_nodelay) if(!config->tcp_nodelay)
my_setopt(curl, CURLOPT_TCP_NODELAY, 0L); my_setopt(curl, CURLOPT_TCP_NODELAY, 0L);
@ -1256,12 +1284,14 @@ static CURLcode create_transfers(struct GlobalConfig *global,
case HTTPREQ_MIMEPOST: case HTTPREQ_MIMEPOST:
result = tool2curlmime(curl, config->mimeroot, &config->mimepost); result = tool2curlmime(curl, config->mimeroot, &config->mimepost);
if(result) if(result)
goto show_error; break;
my_setopt_mimepost(curl, CURLOPT_MIMEPOST, config->mimepost); my_setopt_mimepost(curl, CURLOPT_MIMEPOST, config->mimepost);
break; break;
default: default:
break; break;
} }
if(result)
break;
/* new in libcurl 7.10.6 (default is Basic) */ /* new in libcurl 7.10.6 (default is Basic) */
if(config->authtype) if(config->authtype)
@ -1371,7 +1401,7 @@ static CURLcode create_transfers(struct GlobalConfig *global,
"SSL_CERT_DIR environment variable":"--capath"); "SSL_CERT_DIR environment variable":"--capath");
} }
else if(result) else if(result)
goto show_error; break;
} }
/* For the time being if --proxy-capath is not set then we use the /* For the time being if --proxy-capath is not set then we use the
--capath value for it, if any. See #1257 */ --capath value for it, if any. See #1257 */
@ -1388,7 +1418,7 @@ static CURLcode create_transfers(struct GlobalConfig *global,
} }
} }
else if(result) else if(result)
goto show_error; break;
} }
if(config->crlfile) if(config->crlfile)
@ -1503,7 +1533,7 @@ static CURLcode create_transfers(struct GlobalConfig *global,
Curl_safefree(home); Curl_safefree(home);
} }
if(result) if(result)
goto show_error; break;
} }
} }
@ -1607,7 +1637,7 @@ static CURLcode create_transfers(struct GlobalConfig *global,
if(config->engine) { if(config->engine) {
result = res_setopt_str(curl, CURLOPT_SSLENGINE, config->engine); result = res_setopt_str(curl, CURLOPT_SSLENGINE, config->engine);
if(result) if(result)
goto show_error; break;
} }
/* new in curl 7.10.7, extended in 7.19.4. Modified to use /* new in curl 7.10.7, extended in 7.19.4. Modified to use
@ -1855,7 +1885,7 @@ static CURLcode create_transfers(struct GlobalConfig *global,
outs->metalink_parser = metalink_parser_context_new(); outs->metalink_parser = metalink_parser_context_new();
if(outs->metalink_parser == NULL) { if(outs->metalink_parser == NULL) {
result = CURLE_OUT_OF_MEMORY; result = CURLE_OUT_OF_MEMORY;
goto show_error; break;
} }
fprintf(config->global->errors, fprintf(config->global->errors,
"Metalink: parsing (%s) metalink/XML...\n", per->this_url); "Metalink: parsing (%s) metalink/XML...\n", per->this_url);
@ -1874,48 +1904,37 @@ static CURLcode create_transfers(struct GlobalConfig *global,
per->retry_sleep = per->retry_sleep_default; /* ms */ per->retry_sleep = per->retry_sleep_default; /* ms */
per->retrystart = tvnow(); per->retrystart = tvnow();
} /* loop to the next URL */ state->li++;
show_error:
quit_urls:
if(urls) {
/* Free list of remaining URLs */
glob_cleanup(urls);
urls = NULL;
} }
else {
/* Free this URL node data without destroying the
the node itself nor modifying next pointer. */
Curl_safefree(urlnode->outfile);
Curl_safefree(urlnode->infile);
urlnode->flags = 0;
glob_cleanup(state->urls);
state->urls = NULL;
state->urlnum = 0;
if(result) Curl_safefree(state->outfiles);
/* exit loop upon error */ Curl_safefree(state->uploadfile);
break; if(state->inglob) {
} /* loop to the next globbed upload file */ /* Free list of globbed upload files */
glob_cleanup(state->inglob);
/* Free loop-local allocated memory */ state->inglob = NULL;
}
Curl_safefree(outfiles); config->state.urlnode = urlnode->next;
state->up = 0;
if(inglob) { continue;
/* Free list of globbed upload files */ }
glob_cleanup(inglob);
inglob = NULL;
} }
break;
}
/* Free this URL node data without destroying the if(!*added || result) {
the node itself nor modifying next pointer. */ *added = FALSE;
Curl_safefree(urlnode->url); single_transfer_cleanup(config);
Curl_safefree(urlnode->outfile); }
Curl_safefree(urlnode->infile);
urlnode->flags = 0;
if(result)
/* exit loop upon error */
break;
} /* for-loop through all URLs */
quit_curl:
/* Free function-local referenced allocated memory */
Curl_safefree(httpgetfields);
return result; return result;
} }
@ -1926,18 +1945,23 @@ static long all_added; /* number of easy handles currently added */
* to add even after this call returns. sets 'addedp' to TRUE if one or more * to add even after this call returns. sets 'addedp' to TRUE if one or more
* transfers were added. * transfers were added.
*/ */
static int add_parallel_transfers(struct GlobalConfig *global, static CURLcode add_parallel_transfers(struct GlobalConfig *global,
CURLM *multi, CURLM *multi,
bool *morep, CURLSH *share,
bool *addedp) bool *morep,
bool *addedp)
{ {
struct per_transfer *per; struct per_transfer *per;
CURLcode result; CURLcode result = CURLE_OK;
CURLMcode mcode; CURLMcode mcode;
*addedp = FALSE; *addedp = FALSE;
*morep = FALSE; *morep = FALSE;
result = get_transfer(global, share, addedp);
if(result || !*addedp)
return result;
for(per = transfers; per && (all_added < global->parallel_max); for(per = transfers; per && (all_added < global->parallel_max);
per = per->next) { per = per->next) {
bool getadded = FALSE;
if(per->added) if(per->added)
/* already added */ /* already added */
continue; continue;
@ -1953,6 +1977,10 @@ static int add_parallel_transfers(struct GlobalConfig *global,
mcode = curl_multi_add_handle(multi, per->curl); mcode = curl_multi_add_handle(multi, per->curl);
if(mcode) if(mcode)
return CURLE_OUT_OF_MEMORY; return CURLE_OUT_OF_MEMORY;
/* get the next transfer created */
result = get_transfer(global, share, &getadded);
if(result)
return result;
per->added = TRUE; per->added = TRUE;
all_added++; all_added++;
*addedp = TRUE; *addedp = TRUE;
@ -1976,7 +2004,7 @@ static CURLcode parallel_transfers(struct GlobalConfig *global,
if(!multi) if(!multi)
return CURLE_OUT_OF_MEMORY; return CURLE_OUT_OF_MEMORY;
result = add_parallel_transfers(global, multi, result = add_parallel_transfers(global, multi, share,
&more_transfers, &added_transfers); &more_transfers, &added_transfers);
if(result) if(result)
return result; return result;
@ -2002,7 +2030,7 @@ static CURLcode parallel_transfers(struct GlobalConfig *global,
curl_easy_getinfo(easy, CURLINFO_PRIVATE, (void *)&ended); curl_easy_getinfo(easy, CURLINFO_PRIVATE, (void *)&ended);
curl_multi_remove_handle(multi, easy); curl_multi_remove_handle(multi, easy);
result = post_transfer(global, share, ended, result, &retry); result = post_transfer(global, ended, result, &retry);
if(retry) if(retry)
continue; continue;
progress_finalize(ended); /* before it goes away */ progress_finalize(ended); /* before it goes away */
@ -2013,7 +2041,8 @@ static CURLcode parallel_transfers(struct GlobalConfig *global,
} while(msg); } while(msg);
if(removed) { if(removed) {
/* one or more transfers completed, add more! */ /* one or more transfers completed, add more! */
(void)add_parallel_transfers(global, multi, &more_transfers, (void)add_parallel_transfers(global, multi, share,
&more_transfers,
&added_transfers); &added_transfers);
if(added_transfers) if(added_transfers)
/* we added new ones, make sure the loop doesn't exit yet */ /* we added new ones, make sure the loop doesn't exit yet */
@ -2043,8 +2072,14 @@ static CURLcode serial_transfers(struct GlobalConfig *global,
CURLcode returncode = CURLE_OK; CURLcode returncode = CURLE_OK;
CURLcode result = CURLE_OK; CURLcode result = CURLE_OK;
struct per_transfer *per; struct per_transfer *per;
bool added = FALSE;
result = get_transfer(global, share, &added);
if(result || !added)
return result;
for(per = transfers; per;) { for(per = transfers; per;) {
bool retry; bool retry;
bool bailout = FALSE;
result = pre_transfer(global, per); result = pre_transfer(global, per);
if(result) if(result)
break; break;
@ -2066,32 +2101,47 @@ static CURLcode serial_transfers(struct GlobalConfig *global,
/* store the result of the actual transfer */ /* store the result of the actual transfer */
returncode = result; returncode = result;
result = post_transfer(global, share, per, result, &retry); result = post_transfer(global, per, result, &retry);
if(retry) if(retry)
continue; continue;
/* Bail out upon critical errors or --fail-early */
if(result || is_fatal_error(returncode) ||
(returncode && global->fail_early))
bailout = TRUE;
else {
/* setup the next one just before we delete this */
result = get_transfer(global, share, &added);
if(result)
bailout = TRUE;
}
/* Release metalink related resources here */ /* Release metalink related resources here */
delete_metalinkfile(per->mlfile); delete_metalinkfile(per->mlfile);
per = del_transfer(per); per = del_transfer(per);
/* Bail out upon critical errors or --fail-early */ if(bailout)
if(result || is_fatal_error(returncode) ||
(returncode && global->fail_early))
break; break;
} }
if(returncode) if(returncode)
/* returncode errors have priority */ /* returncode errors have priority */
result = returncode; result = returncode;
if(result)
single_transfer_cleanup(global->current);
return result; return result;
} }
static CURLcode operate_do(struct GlobalConfig *global, static CURLcode operate_do(struct GlobalConfig *global,
struct OperationConfig *config, struct OperationConfig *config,
CURLSH *share) CURLSH *share,
bool *added)
{ {
CURLcode result = CURLE_OK; CURLcode result = CURLE_OK;
bool capath_from_env; bool capath_from_env;
*added = FALSE;
/* Check we have a url */ /* Check we have a url */
if(!config->url_list || !config->url_list->url) { if(!config->url_list || !config->url_list->url) {
@ -2180,12 +2230,33 @@ static CURLcode operate_do(struct GlobalConfig *global,
} }
if(!result) if(!result)
/* loop through the list of given URLs */ result = single_transfer(global, config, share, capath_from_env, added);
result = create_transfers(global, config, share, capath_from_env);
return result; return result;
} }
/*
* 'get_transfer' gets the details and sets up *one* new transfer if 'added'
* returns TRUE.
*/
static CURLcode get_transfer(struct GlobalConfig *global,
CURLSH *share,
bool *added)
{
CURLcode result = CURLE_OK;
*added = FALSE;
while(global->current) {
result = operate_do(global, global->current, share, added);
if(!result && !*added) {
/* when one set is drained, continue to next */
global->current = global->current->next;
continue;
}
break;
}
return result;
}
static CURLcode operate_transfers(struct GlobalConfig *global, static CURLcode operate_transfers(struct GlobalConfig *global,
CURLSH *share, CURLSH *share,
CURLcode result) CURLcode result)
@ -2206,7 +2277,11 @@ static CURLcode operate_transfers(struct GlobalConfig *global,
/* cleanup if there are any left */ /* cleanup if there are any left */
for(per = transfers; per;) { for(per = transfers; per;) {
bool retry; bool retry;
(void)post_transfer(global, share, per, result, &retry); CURLcode result2 = post_transfer(global, per, result, &retry);
if(!result)
/* don't overwrite the original error */
result = result2;
/* Free list of given URLs */ /* Free list of given URLs */
clean_getout(per->config); clean_getout(per->config);
@ -2223,7 +2298,7 @@ static CURLcode operate_transfers(struct GlobalConfig *global,
return result; return result;
} }
CURLcode operate(struct GlobalConfig *config, int argc, argv_item_t argv[]) CURLcode operate(struct GlobalConfig *global, int argc, argv_item_t argv[])
{ {
CURLcode result = CURLE_OK; CURLcode result = CURLE_OK;
@ -2236,18 +2311,18 @@ CURLcode operate(struct GlobalConfig *config, int argc, argv_item_t argv[])
if((argc == 1) || if((argc == 1) ||
(!curl_strequal(argv[1], "-q") && (!curl_strequal(argv[1], "-q") &&
!curl_strequal(argv[1], "--disable"))) { !curl_strequal(argv[1], "--disable"))) {
parseconfig(NULL, config); /* ignore possible failure */ parseconfig(NULL, global); /* ignore possible failure */
/* If we had no arguments then make sure a url was specified in .curlrc */ /* If we had no arguments then make sure a url was specified in .curlrc */
if((argc < 2) && (!config->first->url_list)) { if((argc < 2) && (!global->first->url_list)) {
helpf(config->errors, NULL); helpf(global->errors, NULL);
result = CURLE_FAILED_INIT; result = CURLE_FAILED_INIT;
} }
} }
if(!result) { if(!result) {
/* Parse the command line arguments */ /* Parse the command line arguments */
ParameterError res = parse_args(config, argc, argv); ParameterError res = parse_args(global, argc, argv);
if(res) { if(res) {
result = CURLE_OK; result = CURLE_OK;
@ -2270,7 +2345,7 @@ CURLcode operate(struct GlobalConfig *config, int argc, argv_item_t argv[])
} }
else { else {
#ifndef CURL_DISABLE_LIBCURL_OPTION #ifndef CURL_DISABLE_LIBCURL_OPTION
if(config->libcurl) { if(global->libcurl) {
/* Initialise the libcurl source output */ /* Initialise the libcurl source output */
result = easysrc_init(); result = easysrc_init();
} }
@ -2279,11 +2354,11 @@ CURLcode operate(struct GlobalConfig *config, int argc, argv_item_t argv[])
/* Perform the main operations */ /* Perform the main operations */
if(!result) { if(!result) {
size_t count = 0; size_t count = 0;
struct OperationConfig *operation = config->first; struct OperationConfig *operation = global->first;
CURLSH *share = curl_share_init(); CURLSH *share = curl_share_init();
if(!share) { if(!share) {
#ifndef CURL_DISABLE_LIBCURL_OPTION #ifndef CURL_DISABLE_LIBCURL_OPTION
if(config->libcurl) { if(global->libcurl) {
/* Cleanup the libcurl source output */ /* Cleanup the libcurl source output */
easysrc_cleanup(); easysrc_cleanup();
} }
@ -2305,30 +2380,24 @@ CURLcode operate(struct GlobalConfig *config, int argc, argv_item_t argv[])
} while(!result && operation); } while(!result && operation);
/* Set the current operation pointer */ /* Set the current operation pointer */
config->current = config->first; global->current = global->first;
/* Setup all transfers */
while(!result && config->current) {
result = operate_do(config, config->current, share);
config->current = config->current->next;
}
/* now run! */ /* now run! */
result = operate_transfers(config, share, result); result = operate_transfers(global, share, result);
curl_share_cleanup(share); curl_share_cleanup(share);
#ifndef CURL_DISABLE_LIBCURL_OPTION #ifndef CURL_DISABLE_LIBCURL_OPTION
if(config->libcurl) { if(global->libcurl) {
/* Cleanup the libcurl source output */ /* Cleanup the libcurl source output */
easysrc_cleanup(); easysrc_cleanup();
/* Dump the libcurl code if previously enabled */ /* Dump the libcurl code if previously enabled */
dumpeasysrc(config); dumpeasysrc(global);
} }
#endif #endif
} }
else else
helpf(config->errors, "out of memory\n"); helpf(global->errors, "out of memory\n");
} }
} }

22
src/tool_operhlp.c
View File

@ -37,18 +37,20 @@
void clean_getout(struct OperationConfig *config) void clean_getout(struct OperationConfig *config)
{ {
struct getout *next; if(config) {
struct getout *node = config->url_list; struct getout *next;
struct getout *node = config->url_list;
while(node) { while(node) {
next = node->next; next = node->next;
Curl_safefree(node->url); Curl_safefree(node->url);
Curl_safefree(node->outfile); Curl_safefree(node->outfile);
Curl_safefree(node->infile); Curl_safefree(node->infile);
Curl_safefree(node); Curl_safefree(node);
node = next; node = next;
}
config->url_list = NULL;
} }
config->url_list = NULL;
} }
bool output_expected(const char *url, const char *uploadfile) bool output_expected(const char *url, const char *uploadfile)

2
src/tool_setopt.h
View File

@ -33,7 +33,7 @@
if(!tool_setopt_skip(opt)) { \ if(!tool_setopt_skip(opt)) { \
result = (v); \ result = (v); \
if(result) \ if(result) \
goto show_error; \ break; \
} \ } \
} WHILE_FALSE } WHILE_FALSE

3
src/tool_urlglob.c
View File

@ -488,6 +488,9 @@ void glob_cleanup(URLGlob* glob)
size_t i; size_t i;
int elem; int elem;
if(!glob)
return;
for(i = 0; i < glob->size; i++) { for(i = 0; i < glob->size; i++) {
if((glob->pattern[i].type == UPTSet) && if((glob->pattern[i].type == UPTSet) &&
(glob->pattern[i].content.Set.elements)) { (glob->pattern[i].content.Set.elements)) {

4
tests/data/test62
View File

@ -29,7 +29,7 @@ http
HTTP, send cookies when using custom Host: HTTP, send cookies when using custom Host:
</name> </name>
<command> <command>
http://%HOSTIP:%HTTPPORT/we/want/62 http://%HOSTIP:%HTTPPORT/we/want?hoge=fuga -b log/jar62.txt -H "Host: www.host.foo.com" http://%HOSTIP:%HTTPPORT/we/want/62 http://%HOSTIP:%HTTPPORT/we/want/62?hoge=fuga -b log/jar62.txt -H "Host: www.host.foo.com"
</command> </command>
<file name="log/jar62.txt"> <file name="log/jar62.txt">
# Netscape HTTP Cookie File # Netscape HTTP Cookie File
@ -55,7 +55,7 @@ Host: www.host.foo.com
Accept: */* Accept: */*
Cookie: test2=yes; test=yes Cookie: test2=yes; test=yes
GET /we/want?hoge=fuga HTTP/1.1 GET /we/want/62?hoge=fuga HTTP/1.1
Host: www.host.foo.com Host: www.host.foo.com
Accept: */* Accept: */*
Cookie: test2=yes; test=yes Cookie: test2=yes; test=yes