mirror of https://github.com/moparisthebest/curl
Multiple URL support added
This commit is contained in:
parent
53c27c7722
commit
14ca732a8f
245
src/main.c
245
src/main.c
|
@ -39,7 +39,6 @@
|
|||
#include "writeout.h"
|
||||
|
||||
#define CURLseparator "--_curl_--"
|
||||
#define MIMEseparator "_curl_"
|
||||
|
||||
/* This define make use of the "Curlseparator" as opposed to the
|
||||
MIMEseparator. We might add support for the latter one in the
|
||||
|
@ -222,6 +221,20 @@ static void helpf(char *fmt, ...)
|
|||
fprintf(stderr, "curl: try 'curl --help' for more information\n");
|
||||
}
|
||||
|
||||
/*
|
||||
* A chain of these nodes contain URL to get and where to put the URL's
|
||||
* contents.
|
||||
*/
|
||||
struct getout {
|
||||
struct getout *next;
|
||||
char *url;
|
||||
char *outfile;
|
||||
int flags;
|
||||
};
|
||||
#define GETOUT_OUTFILE (1<<0) /* set when outfile is deemed done */
|
||||
#define GETOUT_URL (1<<1) /* set when URL is deemed done */
|
||||
#define GETOUT_USEREMOTE (1<<2) /* use remote file name locally */
|
||||
|
||||
static void help(void)
|
||||
{
|
||||
printf(CURL_ID "%s\n"
|
||||
|
@ -303,9 +316,7 @@ struct Configurable {
|
|||
char *referer;
|
||||
long timeout;
|
||||
long maxredirs;
|
||||
char *outfile;
|
||||
char *headerfile;
|
||||
char remotefile;
|
||||
char *ftpport;
|
||||
char *iface;
|
||||
unsigned short porttouse;
|
||||
|
@ -320,7 +331,13 @@ struct Configurable {
|
|||
bool configread;
|
||||
bool proxytunnel;
|
||||
long conf;
|
||||
char *url;
|
||||
|
||||
struct getout *url_list; /* point to the first node */
|
||||
struct getout *url_last; /* point to the last/current node */
|
||||
|
||||
struct getout *url_get; /* point to the node to fill in URL */
|
||||
struct getout *url_out; /* point to the node to fill in outfile */
|
||||
|
||||
char *cert;
|
||||
char *cacert;
|
||||
char *cert_passwd;
|
||||
|
@ -426,6 +443,29 @@ static char *file2memory(FILE *file, long *size)
|
|||
return NULL; /* no string */
|
||||
}
|
||||
|
||||
struct getout *new_getout(struct Configurable *config)
|
||||
{
|
||||
struct getout *node =malloc(sizeof(struct getout));
|
||||
struct getout *last= config->url_last;
|
||||
if(node) {
|
||||
/* clear the struct */
|
||||
memset(node, 0, sizeof(struct getout));
|
||||
|
||||
/* append this new node last in the list */
|
||||
if(last)
|
||||
last->next = node;
|
||||
else
|
||||
config->url_list = node; /* first node */
|
||||
|
||||
/* move the last pointer */
|
||||
config->url_last = node;
|
||||
}
|
||||
return node;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
typedef enum {
|
||||
PARAM_OK,
|
||||
PARAM_OPTION_AMBIGUOUS,
|
||||
|
@ -610,7 +650,30 @@ static ParameterError getparameter(char *flag, /* f or -long-flag */
|
|||
break;
|
||||
case '5':
|
||||
/* the URL! */
|
||||
GetStr(&config->url, nextarg);
|
||||
{
|
||||
struct getout *url;
|
||||
if(config->url_get || (config->url_get=config->url_list)) {
|
||||
/* there's a node here, if it already is filled-in continue to find
|
||||
an "empty" node */
|
||||
while(config->url_get && (config->url_get->flags&GETOUT_URL))
|
||||
config->url_get = config->url_get->next;
|
||||
}
|
||||
|
||||
/* now there might or might not be an available node to fill in! */
|
||||
|
||||
if(config->url_get)
|
||||
/* existing node */
|
||||
url = config->url_get;
|
||||
else
|
||||
/* there was no free node, create one! */
|
||||
url=new_getout(config);
|
||||
|
||||
if(url) {
|
||||
/* fill in the URL */
|
||||
GetStr(&url->url, nextarg);
|
||||
url->flags |= GETOUT_URL;
|
||||
}
|
||||
}
|
||||
break;
|
||||
case '#': /* added 19990617 larsa */
|
||||
config->progressmode ^= CURL_PROGRESS_BAR;
|
||||
|
@ -794,12 +857,37 @@ static ParameterError getparameter(char *flag, /* f or -long-flag */
|
|||
config->nobuffer ^= 1;
|
||||
break;
|
||||
case 'o':
|
||||
/* output file */
|
||||
GetStr(&config->outfile, nextarg); /* write to this file */
|
||||
break;
|
||||
case 'O':
|
||||
/* output file */
|
||||
config->remotefile ^= TRUE;
|
||||
{
|
||||
struct getout *url;
|
||||
if(config->url_out || (config->url_out=config->url_list)) {
|
||||
/* there's a node here, if it already is filled-in continue to find
|
||||
an "empty" node */
|
||||
while(config->url_out && (config->url_out->flags&GETOUT_OUTFILE))
|
||||
config->url_out = config->url_out->next;
|
||||
}
|
||||
|
||||
/* now there might or might not be an available node to fill in! */
|
||||
|
||||
if(config->url_out)
|
||||
/* existing node */
|
||||
url = config->url_out;
|
||||
else
|
||||
/* there was no free node, create one! */
|
||||
url=new_getout(config);
|
||||
|
||||
if(url) {
|
||||
/* fill in the outfile */
|
||||
if('o' == letter)
|
||||
GetStr(&url->outfile, nextarg);
|
||||
else {
|
||||
url->outfile=NULL; /* leave it */
|
||||
url->flags |= GETOUT_USEREMOTE;
|
||||
}
|
||||
url->flags |= GETOUT_OUTFILE;
|
||||
}
|
||||
}
|
||||
break;
|
||||
case 'P':
|
||||
/* This makes the FTP sessions use PORT instead of PASV */
|
||||
|
@ -1253,8 +1341,6 @@ void progressbarinit(struct ProgressData *bar)
|
|||
|
||||
void free_config_fields(struct Configurable *config)
|
||||
{
|
||||
if(config->url)
|
||||
free(config->url);
|
||||
if(config->userpwd)
|
||||
free(config->userpwd);
|
||||
if(config->postfields)
|
||||
|
@ -1271,8 +1357,6 @@ void free_config_fields(struct Configurable *config)
|
|||
free(config->krb4level);
|
||||
if(config->headerfile)
|
||||
free(config->headerfile);
|
||||
if(config->outfile)
|
||||
free(config->outfile);
|
||||
if(config->ftpport)
|
||||
free(config->ftpport);
|
||||
if(config->infile)
|
||||
|
@ -1300,6 +1384,8 @@ operate(struct Configurable *config, int argc, char *argv[])
|
|||
char errorbuffer[CURL_ERROR_SIZE];
|
||||
char useragent[128]; /* buah, we don't want a larger default user agent */
|
||||
struct ProgressData progressbar;
|
||||
struct getout *urlnode;
|
||||
struct getout *nextnode;
|
||||
|
||||
struct OutStruct outs;
|
||||
struct OutStruct heads;
|
||||
|
@ -1323,9 +1409,6 @@ operate(struct Configurable *config, int argc, char *argv[])
|
|||
int res;
|
||||
int i;
|
||||
|
||||
outs.stream = stdout;
|
||||
outs.config = config;
|
||||
|
||||
#ifdef MALLOCDEBUG
|
||||
/* this sends all memory debug messages to a logfile named memdump */
|
||||
curl_memdebug("memdump");
|
||||
|
@ -1356,7 +1439,7 @@ operate(struct Configurable *config, int argc, char *argv[])
|
|||
return res;
|
||||
}
|
||||
|
||||
if ((argc < 2) && !config->url) {
|
||||
if ((argc < 2) && !config->url_list) {
|
||||
helpf(NULL);
|
||||
return CURLE_FAILED_INIT;
|
||||
}
|
||||
|
@ -1405,20 +1488,15 @@ operate(struct Configurable *config, int argc, char *argv[])
|
|||
}
|
||||
}
|
||||
else {
|
||||
if(url) {
|
||||
helpf("only one URL is supported!\n");
|
||||
return CURLE_FAILED_INIT;
|
||||
}
|
||||
url = argv[i];
|
||||
bool used;
|
||||
/* just add the URL please */
|
||||
res = getparameter("--url", argv[i], &used, config);
|
||||
if(res)
|
||||
return res;
|
||||
}
|
||||
}
|
||||
|
||||
/* if no URL was specified and there was one in the config file, get that
|
||||
one */
|
||||
if(!url && config->url)
|
||||
url = config->url;
|
||||
|
||||
if(!url) {
|
||||
if(!config->url_list) {
|
||||
helpf("no URL specified!\n");
|
||||
return CURLE_FAILED_INIT;
|
||||
}
|
||||
|
@ -1430,9 +1508,18 @@ operate(struct Configurable *config, int argc, char *argv[])
|
|||
}
|
||||
else
|
||||
allocuseragent = TRUE;
|
||||
#if 0
|
||||
fprintf(stderr, "URL: %s PROXY: %s\n", url, config->proxy?config->proxy:"none");
|
||||
#endif
|
||||
|
||||
urlnode = config->url_list;
|
||||
|
||||
/* loop through the list of given URLs */
|
||||
while(urlnode) {
|
||||
|
||||
/* get the full URL */
|
||||
url=urlnode->url;
|
||||
|
||||
/* default output stream is stdout */
|
||||
outs.stream = stdout;
|
||||
outs.config = config;
|
||||
|
||||
/* expand '{...}' and '[...]' expressions and return total number of URLs
|
||||
in pattern set */
|
||||
|
@ -1440,34 +1527,27 @@ operate(struct Configurable *config, int argc, char *argv[])
|
|||
if(res != CURLE_OK)
|
||||
return res;
|
||||
|
||||
/* save outfile pattern befor expansion */
|
||||
outfiles = config->outfile?strdup(config->outfile):NULL;
|
||||
/* save outfile pattern before expansion */
|
||||
outfiles = urlnode->outfile?strdup(urlnode->outfile):NULL;
|
||||
|
||||
if (!outfiles && !config->remotefile && urlnum > 1) {
|
||||
#ifdef CURL_SEPARATORS
|
||||
if (outfiles && strequal(outfiles, "-") && urlnum > 1) {
|
||||
/* multiple files extracted to stdout, insert separators! */
|
||||
separator = 1;
|
||||
#endif
|
||||
#ifdef MIME_SEPARATORS
|
||||
/* multiple files extracted to stdout, insert MIME separators! */
|
||||
separator = 1;
|
||||
printf("MIME-Version: 1.0\n");
|
||||
printf("Content-Type: multipart/mixed; boundary=%s\n\n", MIMEseparator);
|
||||
#endif
|
||||
}
|
||||
for (i = 0; (url = next_url(urls)); ++i) {
|
||||
if (config->outfile) {
|
||||
free(config->outfile);
|
||||
config->outfile = outfiles?strdup(outfiles):NULL;
|
||||
}
|
||||
char *outfile;
|
||||
outfile = outfiles?strdup(outfiles):NULL;
|
||||
|
||||
|
||||
if((urlnode->flags&GETOUT_USEREMOTE) ||
|
||||
(outfile && !strequal("-", outfile)) ) {
|
||||
|
||||
if (config->outfile || config->remotefile) {
|
||||
/*
|
||||
* We have specified a file name to store the result in, or we have
|
||||
* decided we want to use the remote file name.
|
||||
*/
|
||||
|
||||
if(!config->outfile && config->remotefile) {
|
||||
if(!outfile) {
|
||||
/* Find and get the remote file name */
|
||||
char * pc =strstr(url, "://");
|
||||
if(pc)
|
||||
|
@ -1475,25 +1555,26 @@ operate(struct Configurable *config, int argc, char *argv[])
|
|||
else
|
||||
pc=url;
|
||||
pc = strrchr(pc, '/');
|
||||
config->outfile = (char *) NULL == pc ? NULL : strdup(pc+1) ;
|
||||
if(!config->outfile || !strlen(config->outfile)) {
|
||||
outfile = (char *) NULL == pc ? NULL : strdup(pc+1) ;
|
||||
if(!outfile) {
|
||||
helpf("Remote file name has no length!\n");
|
||||
return CURLE_WRITE_ERROR;
|
||||
}
|
||||
}
|
||||
else {
|
||||
/* fill '#1' ... '#9' terms from URL pattern */
|
||||
char *outfile = config->outfile;
|
||||
config->outfile = match_url(config->outfile, urls);
|
||||
free(outfile);
|
||||
char *storefile = outfile;
|
||||
outfile = match_url(storefile, urls);
|
||||
free(storefile);
|
||||
}
|
||||
|
||||
if((0 == config->resume_from) && config->use_resume) {
|
||||
/* we're told to continue where we are now, then we get the size of the
|
||||
file as it is now and open it for append instead */
|
||||
/* we're told to continue where we are now, then we get the size of
|
||||
the file as it is now and open it for append instead */
|
||||
|
||||
struct stat fileinfo;
|
||||
|
||||
if(0 == stat(config->outfile, &fileinfo)) {
|
||||
if(0 == stat(outfile, &fileinfo)) {
|
||||
/* set offset to current file size: */
|
||||
config->resume_from = fileinfo.st_size;
|
||||
}
|
||||
|
@ -1502,14 +1583,14 @@ operate(struct Configurable *config, int argc, char *argv[])
|
|||
|
||||
if(config->resume_from) {
|
||||
/* open file for output: */
|
||||
outs.stream=(FILE *) fopen(config->outfile, config->resume_from?"ab":"wb");
|
||||
outs.stream=(FILE *) fopen(outfile, config->resume_from?"ab":"wb");
|
||||
if (!outs.stream) {
|
||||
helpf("Can't open '%s'!\n", config->outfile);
|
||||
helpf("Can't open '%s'!\n", outfile);
|
||||
return CURLE_WRITE_ERROR;
|
||||
}
|
||||
}
|
||||
else {
|
||||
outs.filename = config->outfile;
|
||||
outs.filename = outfile;
|
||||
outs.stream = NULL; /* open when needed */
|
||||
}
|
||||
}
|
||||
|
@ -1527,8 +1608,9 @@ operate(struct Configurable *config, int argc, char *argv[])
|
|||
ptr=url;
|
||||
ptr = strrchr(ptr, '/');
|
||||
if(!ptr || !strlen(++ptr)) {
|
||||
/* The URL has no file name part, add the local file name. In order to
|
||||
be able to do so, we have to create a new URL in another buffer.*/
|
||||
/* The URL has no file name part, add the local file name. In order
|
||||
to be able to do so, we have to create a new URL in another
|
||||
buffer.*/
|
||||
|
||||
urlbuffer=(char *)malloc(strlen(url) + strlen(config->infile) + 3);
|
||||
if(!urlbuffer) {
|
||||
|
@ -1579,23 +1661,16 @@ operate(struct Configurable *config, int argc, char *argv[])
|
|||
|
||||
if (urlnum > 1) {
|
||||
fprintf(stderr, "\n[%d/%d]: %s --> %s\n",
|
||||
i+1, urlnum, url, config->outfile ? config->outfile : "<stdout>");
|
||||
if (separator) {
|
||||
#ifdef CURL_SEPARATORS
|
||||
i+1, urlnum, url, outfile ? outfile : "<stdout>");
|
||||
if (separator)
|
||||
printf("%s%s\n", CURLseparator, url);
|
||||
#endif
|
||||
#ifdef MIME_SEPARATORS
|
||||
printf("--%s\n", MIMEseparator);
|
||||
printf("Content-ID: %s\n\n", url);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
if(!config->errors)
|
||||
config->errors = stderr;
|
||||
|
||||
#ifdef WIN32
|
||||
if(!config->outfile && !(config->conf & CONF_GETTEXT)) {
|
||||
if(!outfile && !(config->conf & CONF_GETTEXT)) {
|
||||
/* We get the output to stdout and we have not got the ASCII/text flag,
|
||||
then set stdout to be binary */
|
||||
setmode( 1, O_BINARY );
|
||||
|
@ -1605,7 +1680,6 @@ operate(struct Configurable *config, int argc, char *argv[])
|
|||
|
||||
main_init();
|
||||
|
||||
/* The new, v7-style easy-interface! */
|
||||
curl = curl_easy_init();
|
||||
if(curl) {
|
||||
curl_easy_setopt(curl, CURLOPT_FILE, (FILE *)&outs); /* where to store */
|
||||
|
@ -1676,7 +1750,6 @@ operate(struct Configurable *config, int argc, char *argv[])
|
|||
else
|
||||
curl_easy_setopt(curl, CURLOPT_MAXREDIRS, DEFAULT_MAXREDIRS);
|
||||
|
||||
|
||||
curl_easy_setopt(curl, CURLOPT_CRLF, config->crlf);
|
||||
curl_easy_setopt(curl, CURLOPT_QUOTE, config->quote);
|
||||
curl_easy_setopt(curl, CURLOPT_POSTQUOTE, config->postquote);
|
||||
|
@ -1730,7 +1803,7 @@ operate(struct Configurable *config, int argc, char *argv[])
|
|||
|
||||
if(urlbuffer)
|
||||
free(urlbuffer);
|
||||
if (config->outfile && outs.stream)
|
||||
if (outfile && !strequal(outfile, "-") && outs.stream)
|
||||
fclose(outs.stream);
|
||||
if (config->infile)
|
||||
fclose(infd);
|
||||
|
@ -1740,21 +1813,31 @@ operate(struct Configurable *config, int argc, char *argv[])
|
|||
if(url)
|
||||
free(url);
|
||||
|
||||
if(outfile)
|
||||
free(outfile);
|
||||
}
|
||||
if(outfiles)
|
||||
free(outfiles);
|
||||
|
||||
#ifdef MIME_SEPARATORS
|
||||
if (separator)
|
||||
printf("--%s--\n", MIMEseparator);
|
||||
#endif
|
||||
/* cleanup memory used for URL globbing patterns */
|
||||
glob_cleanup(urls);
|
||||
|
||||
/* empty this urlnode struct */
|
||||
if(urlnode->url)
|
||||
free(urlnode->url);
|
||||
if(urlnode->outfile)
|
||||
free(urlnode->outfile);
|
||||
|
||||
/* move on to the next URL */
|
||||
nextnode=urlnode->next;
|
||||
free(urlnode); /* free the node */
|
||||
urlnode = nextnode;
|
||||
|
||||
} /* while-loop through all URLs */
|
||||
|
||||
if(allocuseragent)
|
||||
free(config->useragent);
|
||||
|
||||
/* cleanup memory used for URL globbing patterns */
|
||||
glob_cleanup(urls);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
|
|
|
@ -213,6 +213,7 @@ int glob_url(URLGlob** glob, char* url, int *urlnum)
|
|||
glob_expand->size = 0;
|
||||
glob_expand->urllen = strlen(url);
|
||||
glob_expand->glob_buffer = glob_buffer;
|
||||
glob_expand->beenhere=0;
|
||||
*urlnum = glob_word(glob_expand, url, 1);
|
||||
*glob = glob_expand;
|
||||
return CURLE_OK;
|
||||
|
@ -240,15 +241,14 @@ void glob_cleanup(URLGlob* glob)
|
|||
|
||||
char *next_url(URLGlob *glob)
|
||||
{
|
||||
static int beenhere = 0;
|
||||
char *buf = glob->glob_buffer;
|
||||
URLPattern *pat;
|
||||
char *lit;
|
||||
signed int i;
|
||||
int carry;
|
||||
|
||||
if (!beenhere)
|
||||
beenhere = 1;
|
||||
if (!glob->beenhere)
|
||||
glob->beenhere = 1;
|
||||
else {
|
||||
carry = 1;
|
||||
|
||||
|
|
|
@ -50,6 +50,7 @@ typedef struct {
|
|||
int size;
|
||||
int urllen;
|
||||
char *glob_buffer;
|
||||
char beenhere;
|
||||
} URLGlob;
|
||||
|
||||
int glob_url(URLGlob**, char*, int *);
|
||||
|
|
Loading…
Reference in New Issue