1
0
mirror of https://github.com/moparisthebest/curl synced 2024-10-31 15:45:12 -04:00

CURLOPT_BUFFERSIZE: support enlarging receive buffer

Replace use of fixed macro BUFSIZE to define the size of the receive
buffer.  Reappropriate CURLOPT_BUFFERSIZE to include enlarging receive
buffer size.  Upon setting, resize buffer if larger than the current
default size up to a MAX_BUFSIZE (512KB). This can benefit protocols
like SFTP.

Closes #1222
This commit is contained in:
Richy Kim 2016-12-20 05:48:15 -05:00 committed by Daniel Stenberg
parent 81cb255cb3
commit 6b7616690e
11 changed files with 59 additions and 20 deletions

View File

@ -190,7 +190,7 @@ Timeout for DNS cache. See \fICURLOPT_DNS_CACHE_TIMEOUT(3)\fP
.IP CURLOPT_DNS_USE_GLOBAL_CACHE .IP CURLOPT_DNS_USE_GLOBAL_CACHE
OBSOLETE Enable global DNS cache. See \fICURLOPT_DNS_USE_GLOBAL_CACHE(3)\fP OBSOLETE Enable global DNS cache. See \fICURLOPT_DNS_USE_GLOBAL_CACHE(3)\fP
.IP CURLOPT_BUFFERSIZE .IP CURLOPT_BUFFERSIZE
Ask for smaller buffer size. See \fICURLOPT_BUFFERSIZE(3)\fP Ask for alternate buffer size. See \fICURLOPT_BUFFERSIZE(3)\fP
.IP CURLOPT_PORT .IP CURLOPT_PORT
Port number to connect to. See \fICURLOPT_PORT(3)\fP Port number to connect to. See \fICURLOPT_PORT(3)\fP
.IP CURLOPT_TCP_FASTOPEN .IP CURLOPT_TCP_FASTOPEN

View File

@ -5,7 +5,7 @@
.\" * | (__| |_| | _ <| |___ .\" * | (__| |_| | _ <| |___
.\" * \___|\___/|_| \_\_____| .\" * \___|\___/|_| \_\_____|
.\" * .\" *
.\" * Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel@haxx.se>, et al. .\" * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al.
.\" * .\" *
.\" * This software is licensed as described in the file COPYING, which .\" * This software is licensed as described in the file COPYING, which
.\" * you should have received as part of this distribution. The terms .\" * you should have received as part of this distribution. The terms
@ -30,20 +30,22 @@ CURLcode curl_easy_setopt(CURL *handle, CURLOPT_BUFFERSIZE, long size);
.SH DESCRIPTION .SH DESCRIPTION
Pass a long specifying your preferred \fIsize\fP (in bytes) for the receive Pass a long specifying your preferred \fIsize\fP (in bytes) for the receive
buffer in libcurl. The main point of this would be that the write callback buffer in libcurl. The main point of this would be that the write callback
gets called more often and with smaller chunks. This is just treated as a gets called more often and with smaller chunks. Secondly, for some protocols,
request, not an order. You cannot be guaranteed to actually get the given there's a benefit of having a larger buffer for performance.
size.
This size is by default set as big as possible (\fICURL_MAX_WRITE_SIZE\fP), so This is just treated as a request, not an order. You cannot be guaranteed to
it only makes sense to use this option if you want it smaller. actually get the given size.
This buffer size is by default \fICURL_MAX_WRITE_SIZE\fP (16kB). The maximum
buffer size allowed to set is \fICURL_MAX_READ_SIZE\fP (512kB).
.SH DEFAULT .SH DEFAULT
CURL_MAX_WRITE_SIZE CURL_MAX_WRITE_SIZE (16kB)
.SH PROTOCOLS .SH PROTOCOLS
All All
.SH EXAMPLE .SH EXAMPLE
TODO TODO
.SH AVAILABILITY .SH AVAILABILITY
Added in 7.10 Added in 7.10. Growing the buffer was added in 7.53.0.
.SH RETURN VALUE .SH RETURN VALUE
Returns CURLE_OK if the option is supported, and CURLE_UNKNOWN_OPTION if not. Returns CURLE_OK if the option is supported, and CURLE_UNKNOWN_OPTION if not.
.SH "SEE ALSO" .SH "SEE ALSO"

View File

@ -748,6 +748,7 @@ CURL_LOCK_TYPE_DNS 7.10 - 7.10.2
CURL_LOCK_TYPE_NONE 7.10 - 7.10.2 CURL_LOCK_TYPE_NONE 7.10 - 7.10.2
CURL_LOCK_TYPE_SSL_SESSION 7.10 - 7.10.2 CURL_LOCK_TYPE_SSL_SESSION 7.10 - 7.10.2
CURL_MAX_HTTP_HEADER 7.19.7 CURL_MAX_HTTP_HEADER 7.19.7
CURL_MAX_READ_SIZE 7.53.0
CURL_MAX_WRITE_SIZE 7.9.7 CURL_MAX_WRITE_SIZE 7.9.7
CURL_NETRC_IGNORED 7.9.8 CURL_NETRC_IGNORED 7.9.8
CURL_NETRC_OPTIONAL 7.9.8 CURL_NETRC_OPTIONAL 7.9.8

View File

@ -193,6 +193,11 @@ typedef int (*curl_xferinfo_callback)(void *clientp,
curl_off_t ultotal, curl_off_t ultotal,
curl_off_t ulnow); curl_off_t ulnow);
#ifndef CURL_MAX_READ_SIZE
/* The maximum receive buffer size configurable via CURLOPT_BUFFERSIZE. */
#define CURL_MAX_READ_SIZE 524288
#endif
#ifndef CURL_MAX_WRITE_SIZE #ifndef CURL_MAX_WRITE_SIZE
/* Tests have proven that 20K is a very bad buffer size for uploads on /* Tests have proven that 20K is a very bad buffer size for uploads on
Windows, while 16K for some odd reason performed a lot better. Windows, while 16K for some odd reason performed a lot better.

View File

@ -870,6 +870,11 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data)
* get setup on-demand in the code, as that would probably decrease * get setup on-demand in the code, as that would probably decrease
* the likeliness of us forgetting to init a buffer here in the future. * the likeliness of us forgetting to init a buffer here in the future.
*/ */
outcurl->set.buffer_size = data->set.buffer_size;
outcurl->state.buffer = malloc(CURL_BUFSIZE(outcurl->set.buffer_size) + 1);
if(!outcurl->state.buffer)
goto fail;
outcurl->state.headerbuff = malloc(HEADERSIZE); outcurl->state.headerbuff = malloc(HEADERSIZE);
if(!outcurl->state.headerbuff) if(!outcurl->state.headerbuff)
goto fail; goto fail;
@ -940,6 +945,7 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data)
if(outcurl) { if(outcurl) {
curl_slist_free_all(outcurl->change.cookielist); curl_slist_free_all(outcurl->change.cookielist);
outcurl->change.cookielist = NULL; outcurl->change.cookielist = NULL;
Curl_safefree(outcurl->state.buffer);
Curl_safefree(outcurl->state.headerbuff); Curl_safefree(outcurl->state.headerbuff);
Curl_safefree(outcurl->change.url); Curl_safefree(outcurl->change.url);
Curl_safefree(outcurl->change.referer); Curl_safefree(outcurl->change.referer);

View File

@ -476,7 +476,7 @@ static CURLcode file_do(struct connectdata *conn, bool *done)
time_t filetime; time_t filetime;
struct tm buffer; struct tm buffer;
const struct tm *tm = &buffer; const struct tm *tm = &buffer;
snprintf(buf, sizeof(data->state.buffer), snprintf(buf, CURL_BUFSIZE(data->set.buffer_size),
"Content-Length: %" CURL_FORMAT_CURL_OFF_T "\r\n", expected_size); "Content-Length: %" CURL_FORMAT_CURL_OFF_T "\r\n", expected_size);
result = Curl_client_write(conn, CLIENTWRITE_BOTH, buf, 0); result = Curl_client_write(conn, CLIENTWRITE_BOTH, buf, 0);
if(result) if(result)

View File

@ -2107,7 +2107,7 @@ static CURLcode ftp_state_mdtm_resp(struct connectdata *conn,
/* we have a time, reformat it */ /* we have a time, reformat it */
time_t secs=time(NULL); time_t secs=time(NULL);
/* using the good old yacc/bison yuck */ /* using the good old yacc/bison yuck */
snprintf(buf, sizeof(conn->data->state.buffer), snprintf(buf, CURL_BUFSIZE(conn->data->set.buffer_size),
"%04d%02d%02d %02d:%02d:%02d GMT", "%04d%02d%02d %02d:%02d:%02d GMT",
year, month, day, hour, minute, second); year, month, day, hour, minute, second);
/* now, convert this into a time() value: */ /* now, convert this into a time() value: */
@ -2318,7 +2318,7 @@ static CURLcode ftp_state_size_resp(struct connectdata *conn,
if(instate == FTP_SIZE) { if(instate == FTP_SIZE) {
#ifdef CURL_FTP_HTTPSTYLE_HEAD #ifdef CURL_FTP_HTTPSTYLE_HEAD
if(-1 != filesize) { if(-1 != filesize) {
snprintf(buf, sizeof(data->state.buffer), snprintf(buf, CURL_BUFSIZE(data->set.buffer_size),
"Content-Length: %" CURL_FORMAT_CURL_OFF_T "\r\n", filesize); "Content-Length: %" CURL_FORMAT_CURL_OFF_T "\r\n", filesize);
result = Curl_client_write(conn, CLIENTWRITE_BOTH, buf, 0); result = Curl_client_write(conn, CLIENTWRITE_BOTH, buf, 0);
if(result) if(result)
@ -2823,6 +2823,7 @@ static CURLcode ftp_statemach_act(struct connectdata *conn)
case FTP_PWD: case FTP_PWD:
if(ftpcode == 257) { if(ftpcode == 257) {
char *ptr=&data->state.buffer[4]; /* start on the first letter */ char *ptr=&data->state.buffer[4]; /* start on the first letter */
const size_t buf_size = CURL_BUFSIZE(data->set.buffer_size);
char *dir; char *dir;
char *store; char *store;
@ -2840,7 +2841,7 @@ static CURLcode ftp_statemach_act(struct connectdata *conn)
*/ */
/* scan for the first double-quote for non-standard responses */ /* scan for the first double-quote for non-standard responses */
while(ptr < &data->state.buffer[sizeof(data->state.buffer)] while(ptr < &data->state.buffer[buf_size]
&& *ptr != '\n' && *ptr != '\0' && *ptr != '"') && *ptr != '\n' && *ptr != '\0' && *ptr != '"')
ptr++; ptr++;

View File

@ -297,7 +297,8 @@ static CURLcode http_output_basic(struct connectdata *conn, bool proxy)
pwd = conn->passwd; pwd = conn->passwd;
} }
snprintf(data->state.buffer, sizeof(data->state.buffer), "%s:%s", user, pwd); snprintf(data->state.buffer, CURL_BUFSIZE(data->set.buffer_size),
"%s:%s", user, pwd);
result = Curl_base64_encode(data, result = Curl_base64_encode(data,
data->state.buffer, strlen(data->state.buffer), data->state.buffer, strlen(data->state.buffer),

View File

@ -1416,6 +1416,7 @@ static CURLcode telnet_do(struct connectdata *conn, bool *done)
/* Keep on listening and act on events */ /* Keep on listening and act on events */
while(keepon) { while(keepon) {
const size_t buf_size = CURL_BUFSIZE(data->set.buffer_size);
waitret = WaitForMultipleObjects(obj_count, objs, FALSE, wait_timeout); waitret = WaitForMultipleObjects(obj_count, objs, FALSE, wait_timeout);
switch(waitret) { switch(waitret) {
case WAIT_TIMEOUT: case WAIT_TIMEOUT:
@ -1451,7 +1452,7 @@ static CURLcode telnet_do(struct connectdata *conn, bool *done)
if(!readfile_read) if(!readfile_read)
break; break;
if(!ReadFile(stdin_handle, buf, sizeof(data->state.buffer), if(!ReadFile(stdin_handle, buf, buf_size,
&readfile_read, NULL)) { &readfile_read, NULL)) {
keepon = FALSE; keepon = FALSE;
result = CURLE_READ_ERROR; result = CURLE_READ_ERROR;
@ -1470,7 +1471,7 @@ static CURLcode telnet_do(struct connectdata *conn, bool *done)
case WAIT_OBJECT_0 + 1: case WAIT_OBJECT_0 + 1:
{ {
if(!ReadFile(stdin_handle, buf, sizeof(data->state.buffer), if(!ReadFile(stdin_handle, buf, buf_size,
&readfile_read, NULL)) { &readfile_read, NULL)) {
keepon = FALSE; keepon = FALSE;
result = CURLE_READ_ERROR; result = CURLE_READ_ERROR;

View File

@ -452,6 +452,7 @@ CURLcode Curl_close(struct Curl_easy *data)
} }
data->change.url = NULL; data->change.url = NULL;
Curl_safefree(data->state.buffer);
Curl_safefree(data->state.headerbuff); Curl_safefree(data->state.headerbuff);
Curl_flush_cookies(data, 1); Curl_flush_cookies(data, 1);
@ -641,6 +642,12 @@ CURLcode Curl_open(struct Curl_easy **curl)
/* We do some initial setup here, all those fields that can't be just 0 */ /* We do some initial setup here, all those fields that can't be just 0 */
data->state.buffer = malloc(BUFSIZE + 1);
if(!data->state.buffer) {
DEBUGF(fprintf(stderr, "Error: malloc of buffer failed\n"));
result = CURLE_OUT_OF_MEMORY;
}
data->state.headerbuff = malloc(HEADERSIZE); data->state.headerbuff = malloc(HEADERSIZE);
if(!data->state.headerbuff) { if(!data->state.headerbuff) {
DEBUGF(fprintf(stderr, "Error: malloc of headerbuff failed\n")); DEBUGF(fprintf(stderr, "Error: malloc of headerbuff failed\n"));
@ -671,6 +678,7 @@ CURLcode Curl_open(struct Curl_easy **curl)
if(result) { if(result) {
Curl_resolver_cleanup(data->state.resolver); Curl_resolver_cleanup(data->state.resolver);
free(data->state.buffer);
free(data->state.headerbuff); free(data->state.headerbuff);
Curl_freeset(data); Curl_freeset(data);
free(data); free(data);
@ -2268,9 +2276,20 @@ CURLcode Curl_setopt(struct Curl_easy *data, CURLoption option,
*/ */
data->set.buffer_size = va_arg(param, long); data->set.buffer_size = va_arg(param, long);
if((data->set.buffer_size> (BUFSIZE -1)) || if(data->set.buffer_size > MAX_BUFSIZE)
(data->set.buffer_size < 1)) data->set.buffer_size = MAX_BUFSIZE; /* huge internal default */
data->set.buffer_size = 0; /* huge internal default */ else if(data->set.buffer_size < 1)
data->set.buffer_size = BUFSIZE;
/* Resize only if larger than default buffer size. */
if(data->set.buffer_size > BUFSIZE) {
data->state.buffer = realloc(data->state.buffer,
data->set.buffer_size + 1);
if(!data->state.buffer) {
DEBUGF(fprintf(stderr, "Error: realloc of buffer failed\n"));
result = CURLE_OUT_OF_MEMORY;
}
}
break; break;

View File

@ -201,6 +201,9 @@
/* Download buffer size, keep it fairly big for speed reasons */ /* Download buffer size, keep it fairly big for speed reasons */
#undef BUFSIZE #undef BUFSIZE
#define BUFSIZE CURL_MAX_WRITE_SIZE #define BUFSIZE CURL_MAX_WRITE_SIZE
#undef MAX_BUFSIZE
#define MAX_BUFSIZE CURL_MAX_READ_SIZE
#define CURL_BUFSIZE(x) ((x)?(x):(BUFSIZE))
/* Initial size of the buffer to store headers in, it'll be enlarged in case /* Initial size of the buffer to store headers in, it'll be enlarged in case
of need. */ of need. */
@ -1304,7 +1307,7 @@ struct UrlState {
char *headerbuff; /* allocated buffer to store headers in */ char *headerbuff; /* allocated buffer to store headers in */
size_t headersize; /* size of the allocation */ size_t headersize; /* size of the allocation */
char buffer[BUFSIZE+1]; /* download buffer */ char *buffer; /* download buffer */
char uploadbuffer[BUFSIZE+1]; /* upload buffer */ char uploadbuffer[BUFSIZE+1]; /* upload buffer */
curl_off_t current_speed; /* the ProgressShow() funcion sets this, curl_off_t current_speed; /* the ProgressShow() funcion sets this,
bytes / second */ bytes / second */