mirror of
https://github.com/moparisthebest/curl
synced 2024-11-11 20:15:03 -05:00
curl: create easy handles on-demand and not ahead of time
This should again enable crazy-large download ranges of the style [1-10000000] that otherwise easily ran out of memory starting in 7.66.0 when this new handle allocating scheme was introduced. Reported-by: Peter Sumatra Fixes #4393 Closes #4438
This commit is contained in:
Daniel Stenberg
parent
c124e6b3c0
commit
e59371a493
@ -22,11 +22,9 @@
|
||||
*
|
||||
***************************************************************************/
|
||||
#include "tool_setup.h"
|
||||
|
||||
#include "tool_sdecls.h"
|
||||
|
||||
#include "tool_metalink.h"
|
||||
|
||||
#include "tool_urlglob.h"
|
||||
#include "tool_formparse.h"
|
||||
|
||||
typedef enum {
|
||||
@ -37,6 +35,20 @@ typedef enum {
|
||||
|
||||
struct GlobalConfig;
|
||||
|
||||
struct State {
|
||||
struct getout *urlnode;
|
||||
URLGlob *inglob;
|
||||
URLGlob *urls;
|
||||
char *outfiles;
|
||||
char *httpgetfields;
|
||||
char *uploadfile;
|
||||
unsigned long infilenum; /* number of files to upload */
|
||||
unsigned long up; /* upload file counter within a single upload glob */
|
||||
unsigned long urlnum; /* how many iterations this single URL has with ranges
|
||||
etc */
|
||||
unsigned long li;
|
||||
};
|
||||
|
||||
struct OperationConfig {
|
||||
bool remote_time;
|
||||
char *random_file;
|
||||
@ -262,6 +274,7 @@ struct OperationConfig {
|
||||
struct GlobalConfig *global;
|
||||
struct OperationConfig *prev;
|
||||
struct OperationConfig *next; /* Always last in the struct */
|
||||
struct State state; /* for create_transfer() */
|
||||
};
|
||||
|
||||
struct GlobalConfig {
|
||||
|
||||
@ -984,12 +984,14 @@ void delete_metalinkfile(metalinkfile *mlfile)
|
||||
|
||||
void clean_metalink(struct OperationConfig *config)
|
||||
{
|
||||
if(config) {
|
||||
while(config->metalinkfile_list) {
|
||||
metalinkfile *mlfile = config->metalinkfile_list;
|
||||
config->metalinkfile_list = config->metalinkfile_list->next;
|
||||
delete_metalinkfile(mlfile);
|
||||
}
|
||||
config->metalinkfile_last = 0;
|
||||
}
|
||||
}
|
||||
|
||||
void metalink_cleanup(void)
|
||||
|
||||
@ -103,10 +103,14 @@ CURLcode curl_easy_perform_ev(CURL *easy);
|
||||
"this situation and\nhow to fix it, please visit the web page mentioned " \
|
||||
"above.\n"
|
||||
|
||||
static CURLcode create_transfers(struct GlobalConfig *global,
|
||||
static CURLcode single_transfer(struct GlobalConfig *global,
|
||||
struct OperationConfig *config,
|
||||
CURLSH *share,
|
||||
bool capath_from_env);
|
||||
bool capath_from_env,
|
||||
bool *added);
|
||||
static CURLcode get_transfer(struct GlobalConfig *global,
|
||||
CURLSH *share,
|
||||
bool *added);
|
||||
|
||||
static bool is_fatal_error(CURLcode code)
|
||||
{
|
||||
@ -316,7 +320,6 @@ static CURLcode pre_transfer(struct GlobalConfig *global,
|
||||
my_setopt(per->curl, CURLOPT_INFILESIZE_LARGE, uploadfilesize);
|
||||
per->input.fd = per->infd;
|
||||
}
|
||||
show_error:
|
||||
return result;
|
||||
}
|
||||
|
||||
@ -324,7 +327,6 @@ static CURLcode pre_transfer(struct GlobalConfig *global,
|
||||
* Call this after a transfer has completed.
|
||||
*/
|
||||
static CURLcode post_transfer(struct GlobalConfig *global,
|
||||
CURLSH *share,
|
||||
struct per_transfer *per,
|
||||
CURLcode result,
|
||||
bool *retryp)
|
||||
@ -333,6 +335,9 @@ static CURLcode post_transfer(struct GlobalConfig *global,
|
||||
CURL *curl = per->curl;
|
||||
struct OperationConfig *config = per->config;
|
||||
|
||||
if(!curl || !config)
|
||||
return result;
|
||||
|
||||
*retryp = FALSE;
|
||||
|
||||
if(per->infdopen)
|
||||
@ -401,7 +406,6 @@ static CURLcode post_transfer(struct GlobalConfig *global,
|
||||
else if(rv == -1)
|
||||
fprintf(config->global->errors, "Metalink: parsing (%s) FAILED\n",
|
||||
per->this_url);
|
||||
result = create_transfers(global, config, share, FALSE);
|
||||
}
|
||||
else if(per->metalink && result == CURLE_OK && !per->metalink_next_res) {
|
||||
int rv;
|
||||
@ -410,8 +414,6 @@ static CURLcode post_transfer(struct GlobalConfig *global,
|
||||
if(!rv)
|
||||
per->metalink_next_res = 1;
|
||||
}
|
||||
#else
|
||||
(void)share;
|
||||
#endif /* USE_METALINK */
|
||||
|
||||
#ifdef USE_METALINK
|
||||
@ -651,60 +653,81 @@ static CURLcode post_transfer(struct GlobalConfig *global,
|
||||
return CURLE_OK;
|
||||
}
|
||||
|
||||
/* go through the list of URLs and configs and add transfers */
|
||||
static void single_transfer_cleanup(struct OperationConfig *config)
|
||||
{
|
||||
if(config) {
|
||||
struct State *state = &config->state;
|
||||
if(state->urls) {
|
||||
/* Free list of remaining URLs */
|
||||
glob_cleanup(state->urls);
|
||||
state->urls = NULL;
|
||||
}
|
||||
Curl_safefree(state->outfiles);
|
||||
Curl_safefree(state->httpgetfields);
|
||||
Curl_safefree(state->uploadfile);
|
||||
if(state->inglob) {
|
||||
/* Free list of globbed upload files */
|
||||
glob_cleanup(state->inglob);
|
||||
state->inglob = NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static CURLcode create_transfers(struct GlobalConfig *global,
|
||||
/* create the next (singular) transfer */
|
||||
|
||||
static CURLcode single_transfer(struct GlobalConfig *global,
|
||||
struct OperationConfig *config,
|
||||
CURLSH *share,
|
||||
bool capath_from_env)
|
||||
bool capath_from_env,
|
||||
bool *added)
|
||||
{
|
||||
CURLcode result = CURLE_OK;
|
||||
struct getout *urlnode;
|
||||
metalinkfile *mlfile_last = NULL;
|
||||
bool orig_noprogress = global->noprogress;
|
||||
bool orig_isatty = global->isatty;
|
||||
char *httpgetfields = NULL;
|
||||
struct State *state = &config->state;
|
||||
char *httpgetfields = state->httpgetfields;
|
||||
*added = FALSE; /* not yet */
|
||||
|
||||
if(config->postfields) {
|
||||
if(config->use_httpget) {
|
||||
if(!httpgetfields) {
|
||||
/* Use the postfields data for a http get */
|
||||
httpgetfields = strdup(config->postfields);
|
||||
httpgetfields = state->httpgetfields = strdup(config->postfields);
|
||||
Curl_safefree(config->postfields);
|
||||
if(!httpgetfields) {
|
||||
helpf(global->errors, "out of memory\n");
|
||||
result = CURLE_OUT_OF_MEMORY;
|
||||
goto quit_curl;
|
||||
}
|
||||
if(SetHTTPrequest(config,
|
||||
else if(SetHTTPrequest(config,
|
||||
(config->no_body?HTTPREQ_HEAD:HTTPREQ_GET),
|
||||
&config->httpreq)) {
|
||||
result = CURLE_FAILED_INIT;
|
||||
goto quit_curl;
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
if(SetHTTPrequest(config, HTTPREQ_SIMPLEPOST, &config->httpreq)) {
|
||||
if(SetHTTPrequest(config, HTTPREQ_SIMPLEPOST, &config->httpreq))
|
||||
result = CURLE_FAILED_INIT;
|
||||
goto quit_curl;
|
||||
}
|
||||
if(result)
|
||||
return result;
|
||||
}
|
||||
if(!state->urlnode) {
|
||||
/* first time caller, setup things */
|
||||
state->urlnode = config->url_list;
|
||||
state->infilenum = 1;
|
||||
}
|
||||
|
||||
for(urlnode = config->url_list; urlnode; urlnode = urlnode->next) {
|
||||
unsigned long li;
|
||||
unsigned long up; /* upload file counter within a single upload glob */
|
||||
while(config->state.urlnode) {
|
||||
char *infiles; /* might be a glob pattern */
|
||||
char *outfiles;
|
||||
unsigned long infilenum;
|
||||
URLGlob *inglob;
|
||||
URLGlob *inglob = state->inglob;
|
||||
bool metalink = FALSE; /* metalink download? */
|
||||
metalinkfile *mlfile;
|
||||
metalink_resource *mlres;
|
||||
|
||||
outfiles = NULL;
|
||||
infilenum = 1;
|
||||
inglob = NULL;
|
||||
|
||||
urlnode = config->state.urlnode;
|
||||
if(urlnode->flags & GETOUT_METALINK) {
|
||||
metalink = 1;
|
||||
if(mlfile_last == NULL) {
|
||||
@ -727,13 +750,15 @@ static CURLcode create_transfers(struct GlobalConfig *global,
|
||||
Curl_safefree(urlnode->outfile);
|
||||
Curl_safefree(urlnode->infile);
|
||||
urlnode->flags = 0;
|
||||
config->state.urlnode = urlnode->next;
|
||||
state->up = 0;
|
||||
continue; /* next URL please */
|
||||
}
|
||||
|
||||
/* save outfile pattern before expansion */
|
||||
if(urlnode->outfile) {
|
||||
outfiles = strdup(urlnode->outfile);
|
||||
if(!outfiles) {
|
||||
if(urlnode->outfile && !state->outfiles) {
|
||||
state->outfiles = strdup(urlnode->outfile);
|
||||
if(!state->outfiles) {
|
||||
helpf(global->errors, "out of memory\n");
|
||||
result = CURLE_OUT_OF_MEMORY;
|
||||
break;
|
||||
@ -742,48 +767,41 @@ static CURLcode create_transfers(struct GlobalConfig *global,
|
||||
|
||||
infiles = urlnode->infile;
|
||||
|
||||
if(!config->globoff && infiles) {
|
||||
if(!config->globoff && infiles && !inglob) {
|
||||
/* Unless explicitly shut off */
|
||||
result = glob_url(&inglob, infiles, &infilenum,
|
||||
result = glob_url(&inglob, infiles, &state->infilenum,
|
||||
global->showerror?global->errors:NULL);
|
||||
if(result) {
|
||||
Curl_safefree(outfiles);
|
||||
if(result)
|
||||
break;
|
||||
}
|
||||
config->state.inglob = inglob;
|
||||
}
|
||||
|
||||
/* Here's the loop for uploading multiple files within the same
|
||||
single globbed string. If no upload, we enter the loop once anyway. */
|
||||
for(up = 0 ; up < infilenum; up++) {
|
||||
|
||||
char *uploadfile; /* a single file, never a glob */
|
||||
{
|
||||
int separator;
|
||||
URLGlob *urls;
|
||||
unsigned long urlnum;
|
||||
|
||||
uploadfile = NULL;
|
||||
urls = NULL;
|
||||
urlnum = 0;
|
||||
|
||||
if(!up && !infiles)
|
||||
if(!state->up && !infiles)
|
||||
Curl_nop_stmt;
|
||||
else {
|
||||
if(!state->uploadfile) {
|
||||
if(inglob) {
|
||||
result = glob_next_url(&uploadfile, inglob);
|
||||
result = glob_next_url(&state->uploadfile, inglob);
|
||||
if(result == CURLE_OUT_OF_MEMORY)
|
||||
helpf(global->errors, "out of memory\n");
|
||||
}
|
||||
else if(!up) {
|
||||
uploadfile = strdup(infiles);
|
||||
if(!uploadfile) {
|
||||
else if(!state->up) {
|
||||
state->uploadfile = strdup(infiles);
|
||||
if(!state->uploadfile) {
|
||||
helpf(global->errors, "out of memory\n");
|
||||
result = CURLE_OUT_OF_MEMORY;
|
||||
}
|
||||
}
|
||||
if(!uploadfile)
|
||||
}
|
||||
if(result)
|
||||
break;
|
||||
}
|
||||
|
||||
if(!state->urlnum) {
|
||||
if(metalink) {
|
||||
/* For Metalink download, we don't use glob. Instead we use
|
||||
the number of resources as urlnum. */
|
||||
@ -792,38 +810,52 @@ static CURLcode create_transfers(struct GlobalConfig *global,
|
||||
else if(!config->globoff) {
|
||||
/* Unless explicitly shut off, we expand '{...}' and '[...]'
|
||||
expressions and return total number of URLs in pattern set */
|
||||
result = glob_url(&urls, urlnode->url, &urlnum,
|
||||
result = glob_url(&state->urls, urlnode->url, &state->urlnum,
|
||||
global->showerror?global->errors:NULL);
|
||||
if(result) {
|
||||
Curl_safefree(uploadfile);
|
||||
if(result)
|
||||
break;
|
||||
}
|
||||
urlnum = state->urlnum;
|
||||
}
|
||||
else
|
||||
urlnum = 1; /* without globbing, this is a single URL */
|
||||
}
|
||||
else
|
||||
urlnum = state->urlnum;
|
||||
|
||||
/* if multiple files extracted to stdout, insert separators! */
|
||||
separator = ((!outfiles || !strcmp(outfiles, "-")) && urlnum > 1);
|
||||
separator = ((!state->outfiles ||
|
||||
!strcmp(state->outfiles, "-")) && urlnum > 1);
|
||||
|
||||
/* Here's looping around each globbed URL */
|
||||
for(li = 0 ; li < urlnum; li++) {
|
||||
|
||||
if(state->li >= urlnum) {
|
||||
state->li = 0;
|
||||
state->up++;
|
||||
}
|
||||
if(state->up < state->infilenum) {
|
||||
struct per_transfer *per;
|
||||
struct OutStruct *outs;
|
||||
struct InStruct *input;
|
||||
struct OutStruct *heads;
|
||||
struct HdrCbData *hdrcbdata = NULL;
|
||||
CURL *curl = curl_easy_init();
|
||||
|
||||
result = add_transfer(&per);
|
||||
if(result || !curl) {
|
||||
free(uploadfile);
|
||||
curl_easy_cleanup(curl);
|
||||
result = CURLE_OUT_OF_MEMORY;
|
||||
goto show_error;
|
||||
break;
|
||||
}
|
||||
if(state->uploadfile) {
|
||||
per->uploadfile = strdup(state->uploadfile);
|
||||
if(!per->uploadfile) {
|
||||
curl_easy_cleanup(curl);
|
||||
result = CURLE_OUT_OF_MEMORY;
|
||||
break;
|
||||
}
|
||||
}
|
||||
*added = TRUE;
|
||||
per->config = config;
|
||||
per->curl = curl;
|
||||
per->uploadfile = uploadfile;
|
||||
|
||||
/* default headers output stream is stdout */
|
||||
heads = &per->heads;
|
||||
@ -838,7 +870,7 @@ static CURLcode create_transfers(struct GlobalConfig *global,
|
||||
if(!newfile) {
|
||||
warnf(config->global, "Failed to open %s\n", config->headerfile);
|
||||
result = CURLE_WRITE_ERROR;
|
||||
goto quit_curl;
|
||||
break;
|
||||
}
|
||||
else {
|
||||
heads->filename = config->headerfile;
|
||||
@ -873,26 +905,26 @@ static CURLcode create_transfers(struct GlobalConfig *global,
|
||||
per->outfile = strdup(mlfile->filename);
|
||||
if(!per->outfile) {
|
||||
result = CURLE_OUT_OF_MEMORY;
|
||||
goto show_error;
|
||||
break;
|
||||
}
|
||||
per->this_url = strdup(mlres->url);
|
||||
if(!per->this_url) {
|
||||
result = CURLE_OUT_OF_MEMORY;
|
||||
goto show_error;
|
||||
break;
|
||||
}
|
||||
per->mlfile = mlfile;
|
||||
}
|
||||
else {
|
||||
if(urls) {
|
||||
result = glob_next_url(&per->this_url, urls);
|
||||
if(state->urls) {
|
||||
result = glob_next_url(&per->this_url, state->urls);
|
||||
if(result)
|
||||
goto show_error;
|
||||
break;
|
||||
}
|
||||
else if(!li) {
|
||||
else if(!state->li) {
|
||||
per->this_url = strdup(urlnode->url);
|
||||
if(!per->this_url) {
|
||||
result = CURLE_OUT_OF_MEMORY;
|
||||
goto show_error;
|
||||
break;
|
||||
}
|
||||
}
|
||||
else
|
||||
@ -900,11 +932,11 @@ static CURLcode create_transfers(struct GlobalConfig *global,
|
||||
if(!per->this_url)
|
||||
break;
|
||||
|
||||
if(outfiles) {
|
||||
per->outfile = strdup(outfiles);
|
||||
if(state->outfiles) {
|
||||
per->outfile = strdup(state->outfiles);
|
||||
if(!per->outfile) {
|
||||
result = CURLE_OUT_OF_MEMORY;
|
||||
goto show_error;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -922,22 +954,22 @@ static CURLcode create_transfers(struct GlobalConfig *global,
|
||||
/* extract the file name from the URL */
|
||||
result = get_url_file_name(&per->outfile, per->this_url);
|
||||
if(result)
|
||||
goto show_error;
|
||||
break;
|
||||
if(!*per->outfile && !config->content_disposition) {
|
||||
helpf(global->errors, "Remote file name has no length!\n");
|
||||
result = CURLE_WRITE_ERROR;
|
||||
goto quit_urls;
|
||||
break;
|
||||
}
|
||||
}
|
||||
else if(urls) {
|
||||
else if(state->urls) {
|
||||
/* fill '#1' ... '#9' terms from URL pattern */
|
||||
char *storefile = per->outfile;
|
||||
result = glob_match_url(&per->outfile, storefile, urls);
|
||||
result = glob_match_url(&per->outfile, storefile, state->urls);
|
||||
Curl_safefree(storefile);
|
||||
if(result) {
|
||||
/* bad globbing */
|
||||
warnf(config->global, "bad output glob!\n");
|
||||
goto quit_urls;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@ -947,11 +979,8 @@ static CURLcode create_transfers(struct GlobalConfig *global,
|
||||
if(config->create_dirs || metalink) {
|
||||
result = create_dir_hierarchy(per->outfile, global->errors);
|
||||
/* create_dir_hierarchy shows error upon CURLE_WRITE_ERROR */
|
||||
if(result == CURLE_WRITE_ERROR)
|
||||
goto quit_urls;
|
||||
if(result) {
|
||||
goto show_error;
|
||||
}
|
||||
if(result)
|
||||
break;
|
||||
}
|
||||
|
||||
if((urlnode->flags & GETOUT_USEREMOTE)
|
||||
@ -986,7 +1015,7 @@ static CURLcode create_transfers(struct GlobalConfig *global,
|
||||
if(!file) {
|
||||
helpf(global->errors, "Can't open '%s'!\n", per->outfile);
|
||||
result = CURLE_WRITE_ERROR;
|
||||
goto quit_urls;
|
||||
break;
|
||||
}
|
||||
outs->fopened = TRUE;
|
||||
outs->stream = file;
|
||||
@ -1006,7 +1035,7 @@ static CURLcode create_transfers(struct GlobalConfig *global,
|
||||
char *nurl = add_file_name_to_url(per->this_url, per->uploadfile);
|
||||
if(!nurl) {
|
||||
result = CURLE_OUT_OF_MEMORY;
|
||||
goto show_error;
|
||||
break;
|
||||
}
|
||||
per->this_url = nurl;
|
||||
}
|
||||
@ -1065,7 +1094,7 @@ static CURLcode create_transfers(struct GlobalConfig *global,
|
||||
if(urlnum > 1 && !global->mute) {
|
||||
per->separator_err =
|
||||
aprintf("\n[%lu/%lu]: %s --> %s",
|
||||
li + 1, urlnum, per->this_url,
|
||||
state->li + 1, urlnum, per->this_url,
|
||||
per->outfile ? per->outfile : "<stdout>");
|
||||
if(separator)
|
||||
per->separator = aprintf("%s%s", CURLseparator, per->this_url);
|
||||
@ -1103,7 +1132,7 @@ static CURLcode create_transfers(struct GlobalConfig *global,
|
||||
|
||||
if(!urlbuffer) {
|
||||
result = CURLE_OUT_OF_MEMORY;
|
||||
goto show_error;
|
||||
break;
|
||||
}
|
||||
|
||||
Curl_safefree(per->this_url); /* free previous URL */
|
||||
@ -1124,11 +1153,10 @@ static CURLcode create_transfers(struct GlobalConfig *global,
|
||||
config->terminal_binary_ok =
|
||||
(per->outfile && !strcmp(per->outfile, "-"));
|
||||
|
||||
/* avoid having this setopt added to the --libcurl source
|
||||
output */
|
||||
/* Avoid having this setopt added to the --libcurl source output. */
|
||||
result = curl_easy_setopt(curl, CURLOPT_SHARE, share);
|
||||
if(result)
|
||||
goto show_error;
|
||||
break;
|
||||
|
||||
if(!config->tcp_nodelay)
|
||||
my_setopt(curl, CURLOPT_TCP_NODELAY, 0L);
|
||||
@ -1256,12 +1284,14 @@ static CURLcode create_transfers(struct GlobalConfig *global,
|
||||
case HTTPREQ_MIMEPOST:
|
||||
result = tool2curlmime(curl, config->mimeroot, &config->mimepost);
|
||||
if(result)
|
||||
goto show_error;
|
||||
break;
|
||||
my_setopt_mimepost(curl, CURLOPT_MIMEPOST, config->mimepost);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
if(result)
|
||||
break;
|
||||
|
||||
/* new in libcurl 7.10.6 (default is Basic) */
|
||||
if(config->authtype)
|
||||
@ -1371,7 +1401,7 @@ static CURLcode create_transfers(struct GlobalConfig *global,
|
||||
"SSL_CERT_DIR environment variable":"--capath");
|
||||
}
|
||||
else if(result)
|
||||
goto show_error;
|
||||
break;
|
||||
}
|
||||
/* For the time being if --proxy-capath is not set then we use the
|
||||
--capath value for it, if any. See #1257 */
|
||||
@ -1388,7 +1418,7 @@ static CURLcode create_transfers(struct GlobalConfig *global,
|
||||
}
|
||||
}
|
||||
else if(result)
|
||||
goto show_error;
|
||||
break;
|
||||
}
|
||||
|
||||
if(config->crlfile)
|
||||
@ -1503,7 +1533,7 @@ static CURLcode create_transfers(struct GlobalConfig *global,
|
||||
Curl_safefree(home);
|
||||
}
|
||||
if(result)
|
||||
goto show_error;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@ -1607,7 +1637,7 @@ static CURLcode create_transfers(struct GlobalConfig *global,
|
||||
if(config->engine) {
|
||||
result = res_setopt_str(curl, CURLOPT_SSLENGINE, config->engine);
|
||||
if(result)
|
||||
goto show_error;
|
||||
break;
|
||||
}
|
||||
|
||||
/* new in curl 7.10.7, extended in 7.19.4. Modified to use
|
||||
@ -1855,7 +1885,7 @@ static CURLcode create_transfers(struct GlobalConfig *global,
|
||||
outs->metalink_parser = metalink_parser_context_new();
|
||||
if(outs->metalink_parser == NULL) {
|
||||
result = CURLE_OUT_OF_MEMORY;
|
||||
goto show_error;
|
||||
break;
|
||||
}
|
||||
fprintf(config->global->errors,
|
||||
"Metalink: parsing (%s) metalink/XML...\n", per->this_url);
|
||||
@ -1874,48 +1904,37 @@ static CURLcode create_transfers(struct GlobalConfig *global,
|
||||
per->retry_sleep = per->retry_sleep_default; /* ms */
|
||||
per->retrystart = tvnow();
|
||||
|
||||
} /* loop to the next URL */
|
||||
|
||||
show_error:
|
||||
quit_urls:
|
||||
|
||||
if(urls) {
|
||||
/* Free list of remaining URLs */
|
||||
glob_cleanup(urls);
|
||||
urls = NULL;
|
||||
state->li++;
|
||||
}
|
||||
|
||||
if(result)
|
||||
/* exit loop upon error */
|
||||
break;
|
||||
} /* loop to the next globbed upload file */
|
||||
|
||||
/* Free loop-local allocated memory */
|
||||
|
||||
Curl_safefree(outfiles);
|
||||
|
||||
if(inglob) {
|
||||
/* Free list of globbed upload files */
|
||||
glob_cleanup(inglob);
|
||||
inglob = NULL;
|
||||
}
|
||||
|
||||
else {
|
||||
/* Free this URL node data without destroying the
|
||||
the node itself nor modifying next pointer. */
|
||||
Curl_safefree(urlnode->url);
|
||||
Curl_safefree(urlnode->outfile);
|
||||
Curl_safefree(urlnode->infile);
|
||||
urlnode->flags = 0;
|
||||
glob_cleanup(state->urls);
|
||||
state->urls = NULL;
|
||||
state->urlnum = 0;
|
||||
|
||||
if(result)
|
||||
/* exit loop upon error */
|
||||
Curl_safefree(state->outfiles);
|
||||
Curl_safefree(state->uploadfile);
|
||||
if(state->inglob) {
|
||||
/* Free list of globbed upload files */
|
||||
glob_cleanup(state->inglob);
|
||||
state->inglob = NULL;
|
||||
}
|
||||
config->state.urlnode = urlnode->next;
|
||||
state->up = 0;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
break;
|
||||
} /* for-loop through all URLs */
|
||||
quit_curl:
|
||||
|
||||
/* Free function-local referenced allocated memory */
|
||||
Curl_safefree(httpgetfields);
|
||||
}
|
||||
|
||||
if(!*added || result) {
|
||||
*added = FALSE;
|
||||
single_transfer_cleanup(config);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@ -1926,18 +1945,23 @@ static long all_added; /* number of easy handles currently added */
|
||||
* to add even after this call returns. sets 'addedp' to TRUE if one or more
|
||||
* transfers were added.
|
||||
*/
|
||||
static int add_parallel_transfers(struct GlobalConfig *global,
|
||||
static CURLcode add_parallel_transfers(struct GlobalConfig *global,
|
||||
CURLM *multi,
|
||||
CURLSH *share,
|
||||
bool *morep,
|
||||
bool *addedp)
|
||||
{
|
||||
struct per_transfer *per;
|
||||
CURLcode result;
|
||||
CURLcode result = CURLE_OK;
|
||||
CURLMcode mcode;
|
||||
*addedp = FALSE;
|
||||
*morep = FALSE;
|
||||
result = get_transfer(global, share, addedp);
|
||||
if(result || !*addedp)
|
||||
return result;
|
||||
for(per = transfers; per && (all_added < global->parallel_max);
|
||||
per = per->next) {
|
||||
bool getadded = FALSE;
|
||||
if(per->added)
|
||||
/* already added */
|
||||
continue;
|
||||
@ -1953,6 +1977,10 @@ static int add_parallel_transfers(struct GlobalConfig *global,
|
||||
mcode = curl_multi_add_handle(multi, per->curl);
|
||||
if(mcode)
|
||||
return CURLE_OUT_OF_MEMORY;
|
||||
/* get the next transfer created */
|
||||
result = get_transfer(global, share, &getadded);
|
||||
if(result)
|
||||
return result;
|
||||
per->added = TRUE;
|
||||
all_added++;
|
||||
*addedp = TRUE;
|
||||
@ -1976,7 +2004,7 @@ static CURLcode parallel_transfers(struct GlobalConfig *global,
|
||||
if(!multi)
|
||||
return CURLE_OUT_OF_MEMORY;
|
||||
|
||||
result = add_parallel_transfers(global, multi,
|
||||
result = add_parallel_transfers(global, multi, share,
|
||||
&more_transfers, &added_transfers);
|
||||
if(result)
|
||||
return result;
|
||||
@ -2002,7 +2030,7 @@ static CURLcode parallel_transfers(struct GlobalConfig *global,
|
||||
curl_easy_getinfo(easy, CURLINFO_PRIVATE, (void *)&ended);
|
||||
curl_multi_remove_handle(multi, easy);
|
||||
|
||||
result = post_transfer(global, share, ended, result, &retry);
|
||||
result = post_transfer(global, ended, result, &retry);
|
||||
if(retry)
|
||||
continue;
|
||||
progress_finalize(ended); /* before it goes away */
|
||||
@ -2013,7 +2041,8 @@ static CURLcode parallel_transfers(struct GlobalConfig *global,
|
||||
} while(msg);
|
||||
if(removed) {
|
||||
/* one or more transfers completed, add more! */
|
||||
(void)add_parallel_transfers(global, multi, &more_transfers,
|
||||
(void)add_parallel_transfers(global, multi, share,
|
||||
&more_transfers,
|
||||
&added_transfers);
|
||||
if(added_transfers)
|
||||
/* we added new ones, make sure the loop doesn't exit yet */
|
||||
@ -2043,8 +2072,14 @@ static CURLcode serial_transfers(struct GlobalConfig *global,
|
||||
CURLcode returncode = CURLE_OK;
|
||||
CURLcode result = CURLE_OK;
|
||||
struct per_transfer *per;
|
||||
bool added = FALSE;
|
||||
|
||||
result = get_transfer(global, share, &added);
|
||||
if(result || !added)
|
||||
return result;
|
||||
for(per = transfers; per;) {
|
||||
bool retry;
|
||||
bool bailout = FALSE;
|
||||
result = pre_transfer(global, per);
|
||||
if(result)
|
||||
break;
|
||||
@ -2066,32 +2101,47 @@ static CURLcode serial_transfers(struct GlobalConfig *global,
|
||||
/* store the result of the actual transfer */
|
||||
returncode = result;
|
||||
|
||||
result = post_transfer(global, share, per, result, &retry);
|
||||
result = post_transfer(global, per, result, &retry);
|
||||
if(retry)
|
||||
continue;
|
||||
|
||||
/* Bail out upon critical errors or --fail-early */
|
||||
if(result || is_fatal_error(returncode) ||
|
||||
(returncode && global->fail_early))
|
||||
bailout = TRUE;
|
||||
else {
|
||||
/* setup the next one just before we delete this */
|
||||
result = get_transfer(global, share, &added);
|
||||
if(result)
|
||||
bailout = TRUE;
|
||||
}
|
||||
|
||||
/* Release metalink related resources here */
|
||||
delete_metalinkfile(per->mlfile);
|
||||
|
||||
per = del_transfer(per);
|
||||
|
||||
/* Bail out upon critical errors or --fail-early */
|
||||
if(result || is_fatal_error(returncode) ||
|
||||
(returncode && global->fail_early))
|
||||
if(bailout)
|
||||
break;
|
||||
}
|
||||
if(returncode)
|
||||
/* returncode errors have priority */
|
||||
result = returncode;
|
||||
|
||||
if(result)
|
||||
single_transfer_cleanup(global->current);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
static CURLcode operate_do(struct GlobalConfig *global,
|
||||
struct OperationConfig *config,
|
||||
CURLSH *share)
|
||||
CURLSH *share,
|
||||
bool *added)
|
||||
{
|
||||
CURLcode result = CURLE_OK;
|
||||
bool capath_from_env;
|
||||
*added = FALSE;
|
||||
|
||||
/* Check we have a url */
|
||||
if(!config->url_list || !config->url_list->url) {
|
||||
@ -2180,12 +2230,33 @@ static CURLcode operate_do(struct GlobalConfig *global,
|
||||
}
|
||||
|
||||
if(!result)
|
||||
/* loop through the list of given URLs */
|
||||
result = create_transfers(global, config, share, capath_from_env);
|
||||
result = single_transfer(global, config, share, capath_from_env, added);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/*
|
||||
* 'get_transfer' gets the details and sets up *one* new transfer if 'added'
|
||||
* returns TRUE.
|
||||
*/
|
||||
static CURLcode get_transfer(struct GlobalConfig *global,
|
||||
CURLSH *share,
|
||||
bool *added)
|
||||
{
|
||||
CURLcode result = CURLE_OK;
|
||||
*added = FALSE;
|
||||
while(global->current) {
|
||||
result = operate_do(global, global->current, share, added);
|
||||
if(!result && !*added) {
|
||||
/* when one set is drained, continue to next */
|
||||
global->current = global->current->next;
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
static CURLcode operate_transfers(struct GlobalConfig *global,
|
||||
CURLSH *share,
|
||||
CURLcode result)
|
||||
@ -2206,7 +2277,11 @@ static CURLcode operate_transfers(struct GlobalConfig *global,
|
||||
/* cleanup if there are any left */
|
||||
for(per = transfers; per;) {
|
||||
bool retry;
|
||||
(void)post_transfer(global, share, per, result, &retry);
|
||||
CURLcode result2 = post_transfer(global, per, result, &retry);
|
||||
if(!result)
|
||||
/* don't overwrite the original error */
|
||||
result = result2;
|
||||
|
||||
/* Free list of given URLs */
|
||||
clean_getout(per->config);
|
||||
|
||||
@ -2223,7 +2298,7 @@ static CURLcode operate_transfers(struct GlobalConfig *global,
|
||||
return result;
|
||||
}
|
||||
|
||||
CURLcode operate(struct GlobalConfig *config, int argc, argv_item_t argv[])
|
||||
CURLcode operate(struct GlobalConfig *global, int argc, argv_item_t argv[])
|
||||
{
|
||||
CURLcode result = CURLE_OK;
|
||||
|
||||
@ -2236,18 +2311,18 @@ CURLcode operate(struct GlobalConfig *config, int argc, argv_item_t argv[])
|
||||
if((argc == 1) ||
|
||||
(!curl_strequal(argv[1], "-q") &&
|
||||
!curl_strequal(argv[1], "--disable"))) {
|
||||
parseconfig(NULL, config); /* ignore possible failure */
|
||||
parseconfig(NULL, global); /* ignore possible failure */
|
||||
|
||||
/* If we had no arguments then make sure a url was specified in .curlrc */
|
||||
if((argc < 2) && (!config->first->url_list)) {
|
||||
helpf(config->errors, NULL);
|
||||
if((argc < 2) && (!global->first->url_list)) {
|
||||
helpf(global->errors, NULL);
|
||||
result = CURLE_FAILED_INIT;
|
||||
}
|
||||
}
|
||||
|
||||
if(!result) {
|
||||
/* Parse the command line arguments */
|
||||
ParameterError res = parse_args(config, argc, argv);
|
||||
ParameterError res = parse_args(global, argc, argv);
|
||||
if(res) {
|
||||
result = CURLE_OK;
|
||||
|
||||
@ -2270,7 +2345,7 @@ CURLcode operate(struct GlobalConfig *config, int argc, argv_item_t argv[])
|
||||
}
|
||||
else {
|
||||
#ifndef CURL_DISABLE_LIBCURL_OPTION
|
||||
if(config->libcurl) {
|
||||
if(global->libcurl) {
|
||||
/* Initialise the libcurl source output */
|
||||
result = easysrc_init();
|
||||
}
|
||||
@ -2279,11 +2354,11 @@ CURLcode operate(struct GlobalConfig *config, int argc, argv_item_t argv[])
|
||||
/* Perform the main operations */
|
||||
if(!result) {
|
||||
size_t count = 0;
|
||||
struct OperationConfig *operation = config->first;
|
||||
struct OperationConfig *operation = global->first;
|
||||
CURLSH *share = curl_share_init();
|
||||
if(!share) {
|
||||
#ifndef CURL_DISABLE_LIBCURL_OPTION
|
||||
if(config->libcurl) {
|
||||
if(global->libcurl) {
|
||||
/* Cleanup the libcurl source output */
|
||||
easysrc_cleanup();
|
||||
}
|
||||
@ -2305,30 +2380,24 @@ CURLcode operate(struct GlobalConfig *config, int argc, argv_item_t argv[])
|
||||
} while(!result && operation);
|
||||
|
||||
/* Set the current operation pointer */
|
||||
config->current = config->first;
|
||||
|
||||
/* Setup all transfers */
|
||||
while(!result && config->current) {
|
||||
result = operate_do(config, config->current, share);
|
||||
config->current = config->current->next;
|
||||
}
|
||||
global->current = global->first;
|
||||
|
||||
/* now run! */
|
||||
result = operate_transfers(config, share, result);
|
||||
result = operate_transfers(global, share, result);
|
||||
|
||||
curl_share_cleanup(share);
|
||||
#ifndef CURL_DISABLE_LIBCURL_OPTION
|
||||
if(config->libcurl) {
|
||||
if(global->libcurl) {
|
||||
/* Cleanup the libcurl source output */
|
||||
easysrc_cleanup();
|
||||
|
||||
/* Dump the libcurl code if previously enabled */
|
||||
dumpeasysrc(config);
|
||||
dumpeasysrc(global);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
else
|
||||
helpf(config->errors, "out of memory\n");
|
||||
helpf(global->errors, "out of memory\n");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -37,6 +37,7 @@
|
||||
|
||||
void clean_getout(struct OperationConfig *config)
|
||||
{
|
||||
if(config) {
|
||||
struct getout *next;
|
||||
struct getout *node = config->url_list;
|
||||
|
||||
@ -49,6 +50,7 @@ void clean_getout(struct OperationConfig *config)
|
||||
node = next;
|
||||
}
|
||||
config->url_list = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
bool output_expected(const char *url, const char *uploadfile)
|
||||
|
||||
@ -33,7 +33,7 @@
|
||||
if(!tool_setopt_skip(opt)) { \
|
||||
result = (v); \
|
||||
if(result) \
|
||||
goto show_error; \
|
||||
break; \
|
||||
} \
|
||||
} WHILE_FALSE
|
||||
|
||||
|
||||
@ -488,6 +488,9 @@ void glob_cleanup(URLGlob* glob)
|
||||
size_t i;
|
||||
int elem;
|
||||
|
||||
if(!glob)
|
||||
return;
|
||||
|
||||
for(i = 0; i < glob->size; i++) {
|
||||
if((glob->pattern[i].type == UPTSet) &&
|
||||
(glob->pattern[i].content.Set.elements)) {
|
||||
|
||||
@ -29,7 +29,7 @@ http
|
||||
HTTP, send cookies when using custom Host:
|
||||
</name>
|
||||
<command>
|
||||
http://%HOSTIP:%HTTPPORT/we/want/62 http://%HOSTIP:%HTTPPORT/we/want?hoge=fuga -b log/jar62.txt -H "Host: www.host.foo.com"
|
||||
http://%HOSTIP:%HTTPPORT/we/want/62 http://%HOSTIP:%HTTPPORT/we/want/62?hoge=fuga -b log/jar62.txt -H "Host: www.host.foo.com"
|
||||
</command>
|
||||
<file name="log/jar62.txt">
|
||||
# Netscape HTTP Cookie File
|
||||
@ -55,7 +55,7 @@ Host: www.host.foo.com
|
||||
Accept: */*
|
||||
Cookie: test2=yes; test=yes
|
||||
|
||||
GET /we/want?hoge=fuga HTTP/1.1
|
||||
GET /we/want/62?hoge=fuga HTTP/1.1
|
||||
Host: www.host.foo.com
|
||||
Accept: */*
|
||||
Cookie: test2=yes; test=yes
|
||||
|
||||
Loading…
Reference in New Issue
Block a user