mirror of
https://github.com/moparisthebest/curl
synced 2024-10-31 15:45:12 -04:00
CURLOPT_BUFFERSIZE: support enlarging receive buffer
Replace use of fixed macro BUFSIZE to define the size of the receive buffer. Reappropriate CURLOPT_BUFFERSIZE to include enlarging receive buffer size. Upon setting, resize buffer if larger than the current default size up to a MAX_BUFSIZE (512KB). This can benefit protocols like SFTP. Closes #1222
This commit is contained in:
parent
81cb255cb3
commit
6b7616690e
@ -190,7 +190,7 @@ Timeout for DNS cache. See \fICURLOPT_DNS_CACHE_TIMEOUT(3)\fP
|
|||||||
.IP CURLOPT_DNS_USE_GLOBAL_CACHE
|
.IP CURLOPT_DNS_USE_GLOBAL_CACHE
|
||||||
OBSOLETE Enable global DNS cache. See \fICURLOPT_DNS_USE_GLOBAL_CACHE(3)\fP
|
OBSOLETE Enable global DNS cache. See \fICURLOPT_DNS_USE_GLOBAL_CACHE(3)\fP
|
||||||
.IP CURLOPT_BUFFERSIZE
|
.IP CURLOPT_BUFFERSIZE
|
||||||
Ask for smaller buffer size. See \fICURLOPT_BUFFERSIZE(3)\fP
|
Ask for alternate buffer size. See \fICURLOPT_BUFFERSIZE(3)\fP
|
||||||
.IP CURLOPT_PORT
|
.IP CURLOPT_PORT
|
||||||
Port number to connect to. See \fICURLOPT_PORT(3)\fP
|
Port number to connect to. See \fICURLOPT_PORT(3)\fP
|
||||||
.IP CURLOPT_TCP_FASTOPEN
|
.IP CURLOPT_TCP_FASTOPEN
|
||||||
|
@ -5,7 +5,7 @@
|
|||||||
.\" * | (__| |_| | _ <| |___
|
.\" * | (__| |_| | _ <| |___
|
||||||
.\" * \___|\___/|_| \_\_____|
|
.\" * \___|\___/|_| \_\_____|
|
||||||
.\" *
|
.\" *
|
||||||
.\" * Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel@haxx.se>, et al.
|
.\" * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al.
|
||||||
.\" *
|
.\" *
|
||||||
.\" * This software is licensed as described in the file COPYING, which
|
.\" * This software is licensed as described in the file COPYING, which
|
||||||
.\" * you should have received as part of this distribution. The terms
|
.\" * you should have received as part of this distribution. The terms
|
||||||
@ -30,20 +30,22 @@ CURLcode curl_easy_setopt(CURL *handle, CURLOPT_BUFFERSIZE, long size);
|
|||||||
.SH DESCRIPTION
|
.SH DESCRIPTION
|
||||||
Pass a long specifying your preferred \fIsize\fP (in bytes) for the receive
|
Pass a long specifying your preferred \fIsize\fP (in bytes) for the receive
|
||||||
buffer in libcurl. The main point of this would be that the write callback
|
buffer in libcurl. The main point of this would be that the write callback
|
||||||
gets called more often and with smaller chunks. This is just treated as a
|
gets called more often and with smaller chunks. Secondly, for some protocols,
|
||||||
request, not an order. You cannot be guaranteed to actually get the given
|
there's a benefit of having a larger buffer for performance.
|
||||||
size.
|
|
||||||
|
|
||||||
This size is by default set as big as possible (\fICURL_MAX_WRITE_SIZE\fP), so
|
This is just treated as a request, not an order. You cannot be guaranteed to
|
||||||
it only makes sense to use this option if you want it smaller.
|
actually get the given size.
|
||||||
|
|
||||||
|
This buffer size is by default \fICURL_MAX_WRITE_SIZE\fP (16kB). The maximum
|
||||||
|
buffer size allowed to set is \fICURL_MAX_READ_SIZE\fP (512kB).
|
||||||
.SH DEFAULT
|
.SH DEFAULT
|
||||||
CURL_MAX_WRITE_SIZE
|
CURL_MAX_WRITE_SIZE (16kB)
|
||||||
.SH PROTOCOLS
|
.SH PROTOCOLS
|
||||||
All
|
All
|
||||||
.SH EXAMPLE
|
.SH EXAMPLE
|
||||||
TODO
|
TODO
|
||||||
.SH AVAILABILITY
|
.SH AVAILABILITY
|
||||||
Added in 7.10
|
Added in 7.10. Growing the buffer was added in 7.53.0.
|
||||||
.SH RETURN VALUE
|
.SH RETURN VALUE
|
||||||
Returns CURLE_OK if the option is supported, and CURLE_UNKNOWN_OPTION if not.
|
Returns CURLE_OK if the option is supported, and CURLE_UNKNOWN_OPTION if not.
|
||||||
.SH "SEE ALSO"
|
.SH "SEE ALSO"
|
||||||
|
@ -748,6 +748,7 @@ CURL_LOCK_TYPE_DNS 7.10 - 7.10.2
|
|||||||
CURL_LOCK_TYPE_NONE 7.10 - 7.10.2
|
CURL_LOCK_TYPE_NONE 7.10 - 7.10.2
|
||||||
CURL_LOCK_TYPE_SSL_SESSION 7.10 - 7.10.2
|
CURL_LOCK_TYPE_SSL_SESSION 7.10 - 7.10.2
|
||||||
CURL_MAX_HTTP_HEADER 7.19.7
|
CURL_MAX_HTTP_HEADER 7.19.7
|
||||||
|
CURL_MAX_READ_SIZE 7.53.0
|
||||||
CURL_MAX_WRITE_SIZE 7.9.7
|
CURL_MAX_WRITE_SIZE 7.9.7
|
||||||
CURL_NETRC_IGNORED 7.9.8
|
CURL_NETRC_IGNORED 7.9.8
|
||||||
CURL_NETRC_OPTIONAL 7.9.8
|
CURL_NETRC_OPTIONAL 7.9.8
|
||||||
|
@ -193,6 +193,11 @@ typedef int (*curl_xferinfo_callback)(void *clientp,
|
|||||||
curl_off_t ultotal,
|
curl_off_t ultotal,
|
||||||
curl_off_t ulnow);
|
curl_off_t ulnow);
|
||||||
|
|
||||||
|
#ifndef CURL_MAX_READ_SIZE
|
||||||
|
/* The maximum receive buffer size configurable via CURLOPT_BUFFERSIZE. */
|
||||||
|
#define CURL_MAX_READ_SIZE 524288
|
||||||
|
#endif
|
||||||
|
|
||||||
#ifndef CURL_MAX_WRITE_SIZE
|
#ifndef CURL_MAX_WRITE_SIZE
|
||||||
/* Tests have proven that 20K is a very bad buffer size for uploads on
|
/* Tests have proven that 20K is a very bad buffer size for uploads on
|
||||||
Windows, while 16K for some odd reason performed a lot better.
|
Windows, while 16K for some odd reason performed a lot better.
|
||||||
|
@ -870,6 +870,11 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data)
|
|||||||
* get setup on-demand in the code, as that would probably decrease
|
* get setup on-demand in the code, as that would probably decrease
|
||||||
* the likeliness of us forgetting to init a buffer here in the future.
|
* the likeliness of us forgetting to init a buffer here in the future.
|
||||||
*/
|
*/
|
||||||
|
outcurl->set.buffer_size = data->set.buffer_size;
|
||||||
|
outcurl->state.buffer = malloc(CURL_BUFSIZE(outcurl->set.buffer_size) + 1);
|
||||||
|
if(!outcurl->state.buffer)
|
||||||
|
goto fail;
|
||||||
|
|
||||||
outcurl->state.headerbuff = malloc(HEADERSIZE);
|
outcurl->state.headerbuff = malloc(HEADERSIZE);
|
||||||
if(!outcurl->state.headerbuff)
|
if(!outcurl->state.headerbuff)
|
||||||
goto fail;
|
goto fail;
|
||||||
@ -940,6 +945,7 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data)
|
|||||||
if(outcurl) {
|
if(outcurl) {
|
||||||
curl_slist_free_all(outcurl->change.cookielist);
|
curl_slist_free_all(outcurl->change.cookielist);
|
||||||
outcurl->change.cookielist = NULL;
|
outcurl->change.cookielist = NULL;
|
||||||
|
Curl_safefree(outcurl->state.buffer);
|
||||||
Curl_safefree(outcurl->state.headerbuff);
|
Curl_safefree(outcurl->state.headerbuff);
|
||||||
Curl_safefree(outcurl->change.url);
|
Curl_safefree(outcurl->change.url);
|
||||||
Curl_safefree(outcurl->change.referer);
|
Curl_safefree(outcurl->change.referer);
|
||||||
|
@ -476,7 +476,7 @@ static CURLcode file_do(struct connectdata *conn, bool *done)
|
|||||||
time_t filetime;
|
time_t filetime;
|
||||||
struct tm buffer;
|
struct tm buffer;
|
||||||
const struct tm *tm = &buffer;
|
const struct tm *tm = &buffer;
|
||||||
snprintf(buf, sizeof(data->state.buffer),
|
snprintf(buf, CURL_BUFSIZE(data->set.buffer_size),
|
||||||
"Content-Length: %" CURL_FORMAT_CURL_OFF_T "\r\n", expected_size);
|
"Content-Length: %" CURL_FORMAT_CURL_OFF_T "\r\n", expected_size);
|
||||||
result = Curl_client_write(conn, CLIENTWRITE_BOTH, buf, 0);
|
result = Curl_client_write(conn, CLIENTWRITE_BOTH, buf, 0);
|
||||||
if(result)
|
if(result)
|
||||||
|
@ -2107,7 +2107,7 @@ static CURLcode ftp_state_mdtm_resp(struct connectdata *conn,
|
|||||||
/* we have a time, reformat it */
|
/* we have a time, reformat it */
|
||||||
time_t secs=time(NULL);
|
time_t secs=time(NULL);
|
||||||
/* using the good old yacc/bison yuck */
|
/* using the good old yacc/bison yuck */
|
||||||
snprintf(buf, sizeof(conn->data->state.buffer),
|
snprintf(buf, CURL_BUFSIZE(conn->data->set.buffer_size),
|
||||||
"%04d%02d%02d %02d:%02d:%02d GMT",
|
"%04d%02d%02d %02d:%02d:%02d GMT",
|
||||||
year, month, day, hour, minute, second);
|
year, month, day, hour, minute, second);
|
||||||
/* now, convert this into a time() value: */
|
/* now, convert this into a time() value: */
|
||||||
@ -2318,7 +2318,7 @@ static CURLcode ftp_state_size_resp(struct connectdata *conn,
|
|||||||
if(instate == FTP_SIZE) {
|
if(instate == FTP_SIZE) {
|
||||||
#ifdef CURL_FTP_HTTPSTYLE_HEAD
|
#ifdef CURL_FTP_HTTPSTYLE_HEAD
|
||||||
if(-1 != filesize) {
|
if(-1 != filesize) {
|
||||||
snprintf(buf, sizeof(data->state.buffer),
|
snprintf(buf, CURL_BUFSIZE(data->set.buffer_size),
|
||||||
"Content-Length: %" CURL_FORMAT_CURL_OFF_T "\r\n", filesize);
|
"Content-Length: %" CURL_FORMAT_CURL_OFF_T "\r\n", filesize);
|
||||||
result = Curl_client_write(conn, CLIENTWRITE_BOTH, buf, 0);
|
result = Curl_client_write(conn, CLIENTWRITE_BOTH, buf, 0);
|
||||||
if(result)
|
if(result)
|
||||||
@ -2823,6 +2823,7 @@ static CURLcode ftp_statemach_act(struct connectdata *conn)
|
|||||||
case FTP_PWD:
|
case FTP_PWD:
|
||||||
if(ftpcode == 257) {
|
if(ftpcode == 257) {
|
||||||
char *ptr=&data->state.buffer[4]; /* start on the first letter */
|
char *ptr=&data->state.buffer[4]; /* start on the first letter */
|
||||||
|
const size_t buf_size = CURL_BUFSIZE(data->set.buffer_size);
|
||||||
char *dir;
|
char *dir;
|
||||||
char *store;
|
char *store;
|
||||||
|
|
||||||
@ -2840,7 +2841,7 @@ static CURLcode ftp_statemach_act(struct connectdata *conn)
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
/* scan for the first double-quote for non-standard responses */
|
/* scan for the first double-quote for non-standard responses */
|
||||||
while(ptr < &data->state.buffer[sizeof(data->state.buffer)]
|
while(ptr < &data->state.buffer[buf_size]
|
||||||
&& *ptr != '\n' && *ptr != '\0' && *ptr != '"')
|
&& *ptr != '\n' && *ptr != '\0' && *ptr != '"')
|
||||||
ptr++;
|
ptr++;
|
||||||
|
|
||||||
|
@ -297,7 +297,8 @@ static CURLcode http_output_basic(struct connectdata *conn, bool proxy)
|
|||||||
pwd = conn->passwd;
|
pwd = conn->passwd;
|
||||||
}
|
}
|
||||||
|
|
||||||
snprintf(data->state.buffer, sizeof(data->state.buffer), "%s:%s", user, pwd);
|
snprintf(data->state.buffer, CURL_BUFSIZE(data->set.buffer_size),
|
||||||
|
"%s:%s", user, pwd);
|
||||||
|
|
||||||
result = Curl_base64_encode(data,
|
result = Curl_base64_encode(data,
|
||||||
data->state.buffer, strlen(data->state.buffer),
|
data->state.buffer, strlen(data->state.buffer),
|
||||||
|
@ -1416,6 +1416,7 @@ static CURLcode telnet_do(struct connectdata *conn, bool *done)
|
|||||||
|
|
||||||
/* Keep on listening and act on events */
|
/* Keep on listening and act on events */
|
||||||
while(keepon) {
|
while(keepon) {
|
||||||
|
const size_t buf_size = CURL_BUFSIZE(data->set.buffer_size);
|
||||||
waitret = WaitForMultipleObjects(obj_count, objs, FALSE, wait_timeout);
|
waitret = WaitForMultipleObjects(obj_count, objs, FALSE, wait_timeout);
|
||||||
switch(waitret) {
|
switch(waitret) {
|
||||||
case WAIT_TIMEOUT:
|
case WAIT_TIMEOUT:
|
||||||
@ -1451,7 +1452,7 @@ static CURLcode telnet_do(struct connectdata *conn, bool *done)
|
|||||||
if(!readfile_read)
|
if(!readfile_read)
|
||||||
break;
|
break;
|
||||||
|
|
||||||
if(!ReadFile(stdin_handle, buf, sizeof(data->state.buffer),
|
if(!ReadFile(stdin_handle, buf, buf_size,
|
||||||
&readfile_read, NULL)) {
|
&readfile_read, NULL)) {
|
||||||
keepon = FALSE;
|
keepon = FALSE;
|
||||||
result = CURLE_READ_ERROR;
|
result = CURLE_READ_ERROR;
|
||||||
@ -1470,7 +1471,7 @@ static CURLcode telnet_do(struct connectdata *conn, bool *done)
|
|||||||
|
|
||||||
case WAIT_OBJECT_0 + 1:
|
case WAIT_OBJECT_0 + 1:
|
||||||
{
|
{
|
||||||
if(!ReadFile(stdin_handle, buf, sizeof(data->state.buffer),
|
if(!ReadFile(stdin_handle, buf, buf_size,
|
||||||
&readfile_read, NULL)) {
|
&readfile_read, NULL)) {
|
||||||
keepon = FALSE;
|
keepon = FALSE;
|
||||||
result = CURLE_READ_ERROR;
|
result = CURLE_READ_ERROR;
|
||||||
|
25
lib/url.c
25
lib/url.c
@ -452,6 +452,7 @@ CURLcode Curl_close(struct Curl_easy *data)
|
|||||||
}
|
}
|
||||||
data->change.url = NULL;
|
data->change.url = NULL;
|
||||||
|
|
||||||
|
Curl_safefree(data->state.buffer);
|
||||||
Curl_safefree(data->state.headerbuff);
|
Curl_safefree(data->state.headerbuff);
|
||||||
|
|
||||||
Curl_flush_cookies(data, 1);
|
Curl_flush_cookies(data, 1);
|
||||||
@ -641,6 +642,12 @@ CURLcode Curl_open(struct Curl_easy **curl)
|
|||||||
|
|
||||||
/* We do some initial setup here, all those fields that can't be just 0 */
|
/* We do some initial setup here, all those fields that can't be just 0 */
|
||||||
|
|
||||||
|
data->state.buffer = malloc(BUFSIZE + 1);
|
||||||
|
if(!data->state.buffer) {
|
||||||
|
DEBUGF(fprintf(stderr, "Error: malloc of buffer failed\n"));
|
||||||
|
result = CURLE_OUT_OF_MEMORY;
|
||||||
|
}
|
||||||
|
|
||||||
data->state.headerbuff = malloc(HEADERSIZE);
|
data->state.headerbuff = malloc(HEADERSIZE);
|
||||||
if(!data->state.headerbuff) {
|
if(!data->state.headerbuff) {
|
||||||
DEBUGF(fprintf(stderr, "Error: malloc of headerbuff failed\n"));
|
DEBUGF(fprintf(stderr, "Error: malloc of headerbuff failed\n"));
|
||||||
@ -671,6 +678,7 @@ CURLcode Curl_open(struct Curl_easy **curl)
|
|||||||
|
|
||||||
if(result) {
|
if(result) {
|
||||||
Curl_resolver_cleanup(data->state.resolver);
|
Curl_resolver_cleanup(data->state.resolver);
|
||||||
|
free(data->state.buffer);
|
||||||
free(data->state.headerbuff);
|
free(data->state.headerbuff);
|
||||||
Curl_freeset(data);
|
Curl_freeset(data);
|
||||||
free(data);
|
free(data);
|
||||||
@ -2268,9 +2276,20 @@ CURLcode Curl_setopt(struct Curl_easy *data, CURLoption option,
|
|||||||
*/
|
*/
|
||||||
data->set.buffer_size = va_arg(param, long);
|
data->set.buffer_size = va_arg(param, long);
|
||||||
|
|
||||||
if((data->set.buffer_size> (BUFSIZE -1)) ||
|
if(data->set.buffer_size > MAX_BUFSIZE)
|
||||||
(data->set.buffer_size < 1))
|
data->set.buffer_size = MAX_BUFSIZE; /* huge internal default */
|
||||||
data->set.buffer_size = 0; /* huge internal default */
|
else if(data->set.buffer_size < 1)
|
||||||
|
data->set.buffer_size = BUFSIZE;
|
||||||
|
|
||||||
|
/* Resize only if larger than default buffer size. */
|
||||||
|
if(data->set.buffer_size > BUFSIZE) {
|
||||||
|
data->state.buffer = realloc(data->state.buffer,
|
||||||
|
data->set.buffer_size + 1);
|
||||||
|
if(!data->state.buffer) {
|
||||||
|
DEBUGF(fprintf(stderr, "Error: realloc of buffer failed\n"));
|
||||||
|
result = CURLE_OUT_OF_MEMORY;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
break;
|
break;
|
||||||
|
|
||||||
|
@ -201,6 +201,9 @@
|
|||||||
/* Download buffer size, keep it fairly big for speed reasons */
|
/* Download buffer size, keep it fairly big for speed reasons */
|
||||||
#undef BUFSIZE
|
#undef BUFSIZE
|
||||||
#define BUFSIZE CURL_MAX_WRITE_SIZE
|
#define BUFSIZE CURL_MAX_WRITE_SIZE
|
||||||
|
#undef MAX_BUFSIZE
|
||||||
|
#define MAX_BUFSIZE CURL_MAX_READ_SIZE
|
||||||
|
#define CURL_BUFSIZE(x) ((x)?(x):(BUFSIZE))
|
||||||
|
|
||||||
/* Initial size of the buffer to store headers in, it'll be enlarged in case
|
/* Initial size of the buffer to store headers in, it'll be enlarged in case
|
||||||
of need. */
|
of need. */
|
||||||
@ -1304,7 +1307,7 @@ struct UrlState {
|
|||||||
char *headerbuff; /* allocated buffer to store headers in */
|
char *headerbuff; /* allocated buffer to store headers in */
|
||||||
size_t headersize; /* size of the allocation */
|
size_t headersize; /* size of the allocation */
|
||||||
|
|
||||||
char buffer[BUFSIZE+1]; /* download buffer */
|
char *buffer; /* download buffer */
|
||||||
char uploadbuffer[BUFSIZE+1]; /* upload buffer */
|
char uploadbuffer[BUFSIZE+1]; /* upload buffer */
|
||||||
curl_off_t current_speed; /* the ProgressShow() funcion sets this,
|
curl_off_t current_speed; /* the ProgressShow() funcion sets this,
|
||||||
bytes / second */
|
bytes / second */
|
||||||
|
Loading…
Reference in New Issue
Block a user