moparisthebest/curl
1
0
Fork 0
mirror of https://github.com/moparisthebest/curl synced 2025-02-28 17:31:46 -05:00
Code Issues Releases Wiki Activity

CURLOPT_UPLOAD_BUFFERSIZE: set upload buffer size

Browse Source 
This is step 3 of #2888.

Fixes #2888
Closes #2896
This commit is contained in:
Daniel Stenberg 2018-08-18 16:17:05 +02:00
parent 87fefe5ebf
commit 2825f46d95
No known key found for this signature in database
GPG Key ID: 5CC908FDB71E12C2
13 changed files with 115 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/libcurl/curl_easy_setopt.3
View File

@ -421,6 +421,8 @@ Size of file to send. \fICURLOPT_INFILESIZE(3)\fP
Size of file to send. \fICURLOPT_INFILESIZE_LARGE(3)\fP
.IP CURLOPT_UPLOAD
Upload data. See \fICURLOPT_UPLOAD(3)\fP
.IP CURLOPT_UPLOAD_BUFFERSIZE
Set upload buffer size. See \fICURLOPT_UPLOAD_BUFFERSIZE(3)\fP
.IP CURLOPT_MIMEPOST
Post/send MIME data. See \fICURLOPT_MIMEPOST(3)\fP
.IP CURLOPT_MAXFILESIZE

69
docs/libcurl/opts/CURLOPT_UPLOAD_BUFFERSIZE.3 Normal file
View File

@ -0,0 +1,69 @@
.\" **************************************************************************
.\" * _ _ ____ _
.\" * Project ___| | | | _ \| |
.\" * / __| | | | |_) | |
.\" * | (__| |_| | _ <| |___
.\" * \___|\___/|_| \_\_____|
.\" *
.\" * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al.
.\" *
.\" * This software is licensed as described in the file COPYING, which
.\" * you should have received as part of this distribution. The terms
.\" * are also available at https://curl.haxx.se/docs/copyright.html.
.\" *
.\" * You may opt to use, copy, modify, merge, publish, distribute and/or sell
.\" * copies of the Software, and permit persons to whom the Software is
.\" * furnished to do so, under the terms of the COPYING file.
.\" *
.\" * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
.\" * KIND, either express or implied.
.\" *
.\" **************************************************************************
.\"
.TH CURLOPT_UPLOAD_BUFFERSIZE 3 "18 Aug 2018" "libcurl 7.62.0" "curl_easy_setopt options"
.SH NAME
CURLOPT_UPLOAD_BUFFERSIZE \- set preferred upload buffer size
.SH SYNOPSIS
#include <curl/curl.h>
CURLcode curl_easy_setopt(CURL *handle, CURLOPT_UPLOAD_BUFFERSIZE, long size);
.SH DESCRIPTION
Pass a long specifying your preferred \fIsize\fP (in bytes) for the upload
buffer in libcurl. It makes libcurl uses a larger buffer that gets passed to
the next layer in the stack to get sent off. In some setups and for some
protocols, there's a huge performance benefit of having a larger upload
buffer.
This is just treated as a request, not an order. You cannot be guaranteed to
actually get the given size.
The upload buffer size is by default 64 kilobytes. The maximum buffer size
allowed to be set is 2 megabytes. The minimum buffer size allowed to be set is
16 kilobytes.
Since curl 7.61.1 the upload buffer is allocated on-demand - so if the handle
isn't used for upload, this buffer will not be allocated at all.
.SH DEFAULT
64 kB
.SH PROTOCOLS
All
.SH EXAMPLE
.nf
CURL *curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, "sftp://example.com/foo.bin");
/* ask libcurl to allocate a larger upload buffer */
curl_easy_setopt(curl, CURLOPT_UPLOAD_BUFFERSIZE, 120000L);
ret = curl_easy_perform(curl);
curl_easy_cleanup(curl);
}
.fi
.SH AVAILABILITY
Added in 7.62.0.
.SH RETURN VALUE
Returns CURLE_OK if the option is supported, and CURLE_UNKNOWN_OPTION if not.
.SH "SEE ALSO"
.BR CURLOPT_BUFFERSIZE "(3), " CURLOPT_READFUNCTION "(3), "

1
docs/libcurl/opts/Makefile.inc
View File

@ -327,6 +327,7 @@ man_MANS = \
CURLOPT_UNIX_SOCKET_PATH.3 \
CURLOPT_UNRESTRICTED_AUTH.3 \
CURLOPT_UPLOAD.3 \
CURLOPT_UPLOAD_BUFFERSIZE.3 \
CURLOPT_URL.3 \
CURLOPT_USERAGENT.3 \
CURLOPT_USERNAME.3 \

1
docs/libcurl/symbols-in-versions
View File

@ -616,6 +616,7 @@ CURLOPT_TRANSFER_ENCODING 7.21.6
CURLOPT_UNIX_SOCKET_PATH 7.40.0
CURLOPT_UNRESTRICTED_AUTH 7.10.4
CURLOPT_UPLOAD 7.1
CURLOPT_UPLOAD_BUFFERSIZE 7.62.0
CURLOPT_URL 7.1
CURLOPT_USERAGENT 7.1
CURLOPT_USERNAME 7.19.1

3
include/curl/curl.h
View File

@ -1862,6 +1862,9 @@ typedef enum {
/* DNS-over-HTTPS URL */
CINIT(DOH_URL, STRINGPOINT, 279),
/* Preferred buffer size to use for uploads */
CINIT(UPLOAD_BUFFERSIZE, LONG, 280),
CURLOPT_LASTENTRY /* the last unused */
} CURLoption;

2
lib/file.c
View File

@ -307,7 +307,7 @@ static CURLcode file_upload(struct connectdata *conn)
size_t nread;
size_t nwrite;
size_t readcount;
result = Curl_fillreadbuffer(conn, (int)data->set.buffer_size, &readcount);
result = Curl_fillreadbuffer(conn, data->set.buffer_size, &readcount);
if(result)
break;

16
lib/setopt.c
View File

@ -1940,6 +1940,22 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option,
break;
case CURLOPT_UPLOAD_BUFFERSIZE:
/*
* The application kindly asks for a differently sized upload buffer.
* Cap it to sensible.
*/
arg = va_arg(param, long);
if(arg > UPLOADBUFFER_MAX)
arg = UPLOADBUFFER_MAX;
else if(arg < UPLOADBUFFER_MIN)
arg = UPLOADBUFFER_MIN;
data->set.upload_buffer_size = arg;
Curl_safefree(data->state.ulbuf); /* force a realloc next opportunity */
break;
case CURLOPT_NOSIGNAL:
/*
* The application asks not to set any signal() or alarm() handlers,

3
lib/smb.c
View File

@ -610,7 +610,8 @@ static CURLcode smb_send_and_recv(struct connectdata *conn, void **msg)
/* Check if there is data in the transfer buffer */
if(!smbc->send_size && smbc->upload_size) {
size_t nread = smbc->upload_size > UPLOAD_BUFSIZE ? UPLOAD_BUFSIZE :
size_t nread = smbc->upload_size > conn->data->set.upload_buffer_size ?
conn->data->set.upload_buffer_size :
smbc->upload_size;
conn->data->req.upload_fromhere = conn->data->state.ulbuf;
result = Curl_fillreadbuffer(conn, nread, &nread);

4
lib/smtp.c
View File

@ -1563,14 +1563,14 @@ CURLcode Curl_smtp_escape_eob(struct connectdata *conn, const ssize_t nread)
if(!scratch || data->set.crlf) {
oldscratch = scratch;
scratch = newscratch = malloc(2 * UPLOAD_BUFSIZE);
scratch = newscratch = malloc(2 * data->set.upload_buffer_size);
if(!newscratch) {
failf(data, "Failed to alloc scratch buffer!");
return CURLE_OUT_OF_MEMORY;
}
}
DEBUGASSERT(UPLOAD_BUFSIZE >= nread);
DEBUGASSERT(data->set.upload_buffer_size >= (size_t)nread);
/* Have we already sent part of the EOB? */
eob_sent = smtp->eob;

5
lib/transfer.c
View File

@ -959,7 +959,8 @@ static CURLcode readwrite_upload(struct Curl_easy *data,
sending_http_headers = FALSE;
}
result = Curl_fillreadbuffer(conn, UPLOAD_BUFSIZE, &fillcount);
result = Curl_fillreadbuffer(conn, data->set.upload_buffer_size,
&fillcount);
if(result)
return result;
@ -991,7 +992,7 @@ static CURLcode readwrite_upload(struct Curl_easy *data,
(data->set.crlf))) {
/* Do we need to allocate a scratch buffer? */
if(!data->state.scratch) {
data->state.scratch = malloc(2 * UPLOAD_BUFSIZE);
data->state.scratch = malloc(2 * data->set.upload_buffer_size);
if(!data->state.scratch) {
failf(data, "Failed to alloc scratch buffer!");

2
lib/url.c
View File

@ -526,7 +526,7 @@ CURLcode Curl_init_userdefined(struct Curl_easy *data)
set->expect_100_timeout = 1000L; /* Wait for a second by default. */
set->sep_headers = TRUE; /* separated header lists by default */
set->buffer_size = READBUFFER_SIZE;
set->upload_buffer_size = UPLOAD_BUFSIZE;
set->upload_buffer_size = UPLOADBUFFER_DEFAULT;
set->happy_eyeballs_timeout = CURL_HET_DEFAULT;
set->fnmatch = ZERO_NULL;
set->maxconnects = DEFAULT_CONNCACHE_SIZE; /* for easy handles */

12
lib/url.h
View File

@ -27,6 +27,18 @@
#define READBUFFER_MAX CURL_MAX_READ_SIZE
#define READBUFFER_MIN 1024
/* The default upload buffer size, should not be smaller than
CURL_MAX_WRITE_SIZE, as it needs to hold a full buffer as could be sent in
a write callback.
The size was 16KB for many years but was bumped to 64KB because it makes
libcurl able to do significantly faster uploads in some circumstances. Even
larger buffers can help further, but this is deemed a fair memory/speed
compromise. */
#define UPLOADBUFFER_DEFAULT 65536
#define UPLOADBUFFER_MAX (2*1024*1024)
#define UPLOADBUFFER_MIN CURL_MAX_WRITE_SIZE
/*
* Prototypes for library-wide functions provided by url.c
*/

10
lib/urldata.h
View File

@ -142,14 +142,6 @@ typedef ssize_t (Curl_recv)(struct connectdata *conn, /* connection data */
#include <libssh2_sftp.h>
#endif /* HAVE_LIBSSH2_H */
/* The upload buffer size, should not be smaller than CURL_MAX_WRITE_SIZE, as
it needs to hold a full buffer as could be sent in a write callback.
The size was 16KB for many years but was bumped to 64KB because it makes
libcurl able to do significantly faster uploads in some circumstances. Even
larger buffers can help further, but this is deemed a fair memory/speed
compromise. */
#define UPLOAD_BUFSIZE 65536
/* The "master buffer" is for HTTP pipelining */
#define MASTERBUF_SIZE 16384
@ -1585,7 +1577,7 @@ struct UserDefined {
curl_proxytype proxytype; /* what kind of proxy that is in use */
long dns_cache_timeout; /* DNS cache timeout */
long buffer_size; /* size of receive buffer to use */
long upload_buffer_size; /* size of upload buffer to use,
size_t upload_buffer_size; /* size of upload buffer to use,
keep it >= CURL_MAX_WRITE_SIZE */
void *private_data; /* application-private data */