moparisthebest/curl
1
0
Fork 0
mirror of https://github.com/moparisthebest/curl synced 2025-02-28 09:21:50 -05:00
Code Issues Releases Wiki Activity

multi: improved pending transfers handling => improved performance

Browse Source 
When a transfer is requested to get done and it is put in the pending
queue when limited by number of connections, total or per-host, libcurl
would previously very aggressively retry *ALL* pending transfers to get
them transferring. That was very time consuming.

By reducing the aggressiveness in how pending are being retried, we
waste MUCH less time on putting transfers back into pending again.

Some test cases got a factor 30(!) speed improvement with this change.

Reported-by: Cyril B
Fixes #2369
Closes #2383
This commit is contained in:
Daniel Stenberg 2018-03-13 16:13:56 +01:00
parent 2404aa080e
commit 7f9ce0851a
No known key found for this signature in database
GPG Key ID: 5CC908FDB71E12C2
2 changed files with 21 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
lib/multi.c
View File

@ -77,6 +77,7 @@ static CURLMcode add_next_timeout(struct curltime now,
struct Curl_easy *d);
static CURLMcode multi_timeout(struct Curl_multi *multi,
long *timeout_ms);
static void process_pending_handles(struct Curl_multi *multi);
#ifdef DEBUGBUILD
static const char * const statename[]={
@ -538,6 +539,8 @@ static CURLcode multi_done(struct connectdata **connp,
result = CURLE_ABORTED_BY_CALLBACK;
}
process_pending_handles(data->multi); /* connection / multiplex */
if(conn->send_pipe.size || conn->recv_pipe.size) {
/* Stop if pipeline is not empty . */
data->easy_conn = NULL;
@ -653,10 +656,6 @@ CURLMcode curl_multi_remove_handle(struct Curl_multi *multi,
/* this handle is "alive" so we need to count down the total number of
alive connections when this is removed */
multi->num_alive--;
/* When this handle gets removed, other handles may be able to get the
connection */
Curl_multi_process_pending_handles(multi);
}
if(data->easy_conn &&
@ -1339,7 +1338,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
if(multi_ischanged(multi, TRUE)) {
DEBUGF(infof(data, "multi changed, check CONNECT_PEND queue!\n"));
Curl_multi_process_pending_handles(multi);
process_pending_handles(multi); /* pipelined/multiplexed */
}
if(data->easy_conn && data->mstate > CURLM_STATE_CONNECT &&
@ -1785,8 +1784,10 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
case CURLM_STATE_DO_DONE:
/* Move ourselves from the send to recv pipeline */
Curl_move_handle_from_send_to_recv_pipe(data, data->easy_conn);
/* Check if we can move pending requests to send pipe */
Curl_multi_process_pending_handles(multi);
if(data->easy_conn->bits.multiplex || data->easy_conn->send_pipe.size)
/* Check if we can move pending requests to send pipe */
process_pending_handles(multi); /* pipelined/multiplexed */
/* Only perform the transfer if there's a good socket to work with.
Having both BAD is a signal to skip immediately to DONE */
@ -1943,9 +1944,6 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
if(data->easy_conn->recv_pipe.head)
Curl_expire(data->easy_conn->recv_pipe.head->ptr, 0, EXPIRE_RUN_NOW);
/* Check if we can move pending requests to send pipe */
Curl_multi_process_pending_handles(multi);
/* When we follow redirects or is set to retry the connection, we must
to go back to the CONNECT state */
if(data->req.newurl || retry) {
@ -2002,8 +2000,10 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
/* Remove ourselves from the receive pipeline, if we are there. */
Curl_removeHandleFromPipeline(data, &data->easy_conn->recv_pipe);
/* Check if we can move pending requests to send pipe */
Curl_multi_process_pending_handles(multi);
if(data->easy_conn->bits.multiplex || data->easy_conn->send_pipe.size)
/* Check if we can move pending requests to connection */
process_pending_handles(multi); /* pipelined/multiplexing */
/* post-transfer command */
res = multi_done(&data->easy_conn, result, FALSE);
@ -2071,7 +2071,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
data->state.pipe_broke = FALSE;
/* Check if we can move pending requests to send pipe */
Curl_multi_process_pending_handles(multi);
process_pending_handles(multi); /* connection */
if(data->easy_conn) {
/* if this has a connection, unsubscribe from the pipelines */
@ -3079,25 +3079,21 @@ struct curl_llist *Curl_multi_pipelining_server_bl(struct Curl_multi *multi)
return &multi->pipelining_server_bl;
}
void Curl_multi_process_pending_handles(struct Curl_multi *multi)
static void process_pending_handles(struct Curl_multi *multi)
{
struct curl_llist_element *e = multi->pending.head;
while(e) {
if(e) {
struct Curl_easy *data = e->ptr;
struct curl_llist_element *next = e->next;
if(data->mstate == CURLM_STATE_CONNECT_PEND) {
multistate(data, CURLM_STATE_CONNECT);
DEBUGASSERT(data->mstate == CURLM_STATE_CONNECT_PEND);
/* Remove this node from the list */
Curl_llist_remove(&multi->pending, e, NULL);
multistate(data, CURLM_STATE_CONNECT);
/* Make sure that the handle will be processed soonish. */
Curl_expire(data, 0, EXPIRE_RUN_NOW);
}
/* Remove this node from the list */
Curl_llist_remove(&multi->pending, e, NULL);
e = next; /* operate on next handle */
/* Make sure that the handle will be processed soonish. */
Curl_expire(data, 0, EXPIRE_RUN_NOW);
}
}

2
lib/multiif.h
View File

@ -59,8 +59,6 @@ struct Curl_multi *Curl_multi_handle(int hashsize, int chashsize);
void Curl_multi_dump(struct Curl_multi *multi);
#endif
void Curl_multi_process_pending_handles(struct Curl_multi *multi);
/* Return the value of the CURLMOPT_MAX_HOST_CONNECTIONS option */
size_t Curl_multi_max_host_connections(struct Curl_multi *multi);