From 0f147887b0d592d5fa72215282e84103eb165ad7 Mon Sep 17 00:00:00 2001 From: Linus Nielsen Feltzing Date: Fri, 15 Feb 2013 11:50:45 +0100 Subject: [PATCH] Multiple pipelines and limiting the number of connections. Introducing a number of options to the multi interface that allows for multiple pipelines to the same host, in order to optimize the balance between the penalty for opening new connections and the potential pipelining latency. Two new options for limiting the number of connections: CURLMOPT_MAX_HOST_CONNECTIONS - Limits the number of running connections to the same host. When adding a handle that exceeds this limit, that handle will be put in a pending state until another handle is finished, so we can reuse the connection. CURLMOPT_MAX_TOTAL_CONNECTIONS - Limits the number of connections in total. When adding a handle that exceeds this limit, that handle will be put in a pending state until another handle is finished. The free connection will then be reused, if possible, or closed if the pending handle can't reuse it. Several new options for pipelining: CURLMOPT_MAX_PIPELINE_LENGTH - Limits the pipeling length. If a pipeline is "full" when a connection is to be reused, a new connection will be opened if the CURLMOPT_MAX_xxx_CONNECTIONS limits allow it. If not, the handle will be put in a pending state until a connection is ready (either free or a pipe got shorter). CURLMOPT_CONTENT_LENGTH_PENALTY_SIZE - A pipelined connection will not be reused if it is currently processing a transfer with a content length that is larger than this. CURLMOPT_CHUNK_LENGTH_PENALTY_SIZE - A pipelined connection will not be reused if it is currently processing a chunk larger than this. CURLMOPT_PIPELINING_SITE_BL - A blacklist of hosts that don't allow pipelining. CURLMOPT_PIPELINING_SERVER_BL - A blacklist of server types that don't allow pipelining. See the curl_multi_setopt() man page for details. --- docs/libcurl/curl_multi_setopt.3 | 106 ++++++++ docs/libcurl/libcurl-errors.3 | 3 + docs/libcurl/symbols-in-versions | 8 + include/curl/curl.h | 2 + include/curl/multi.h | 25 ++ lib/Makefile.inc | 4 +- lib/README.pipelining | 7 - lib/hash.h | 1 - lib/http.c | 20 +- lib/multi.c | 261 ++++++++---------- lib/multihandle.h | 63 +++-- lib/multiif.h | 30 ++- lib/pipeline.c | 366 +++++++++++++++++++++++++ lib/pipeline.h | 50 ++++ lib/sendf.c | 3 +- lib/strerror.c | 3 + lib/transfer.c | 7 +- lib/url.c | 268 +++++++++++++----- lib/urldata.h | 17 +- tests/README | 1 + tests/data/Makefile.am | 3 +- tests/data/test1900 | 57 ++++ tests/data/test1901 | 58 ++++ tests/data/test1902 | 57 ++++ tests/data/test1903 | 57 ++++ tests/data/test2033 | 144 ++++++++++ tests/data/test530 | 2 +- tests/data/test536 | 8 + tests/data/test584 | 2 +- tests/http_pipe.py | 447 +++++++++++++++++++++++++++++++ tests/libtest/.gitignore | 2 + tests/libtest/Makefile.inc | 10 +- tests/libtest/lib1900.c | 256 ++++++++++++++++++ tests/libtest/lib530.c | 15 +- tests/libtest/libntlmconnect.c | 6 + tests/runtests.pl | 116 +++++++- tests/serverhelp.pm | 4 +- 37 files changed, 2210 insertions(+), 279 deletions(-) create mode 100644 lib/pipeline.c create mode 100644 lib/pipeline.h create mode 100644 tests/data/test1900 create mode 100644 tests/data/test1901 create mode 100644 tests/data/test1902 create mode 100644 tests/data/test1903 create mode 100644 tests/data/test2033 create mode 100755 tests/http_pipe.py create mode 100644 tests/libtest/lib1900.c diff --git a/docs/libcurl/curl_multi_setopt.3 b/docs/libcurl/curl_multi_setopt.3 index 9e456a093..99984cf49 100644 --- a/docs/libcurl/curl_multi_setopt.3 +++ b/docs/libcurl/curl_multi_setopt.3 @@ -95,6 +95,112 @@ This option is for the multi handle's use only, when using the easy interface you should instead use the \fICURLOPT_MAXCONNECTS\fP option. (Added in 7.16.3) +.IP CURLMOPT_MAX_HOST_CONNECTIONS +Pass a long. The set number will be used as the maximum amount of +simultaneously open connections to a single host. For each new session to +a host, libcurl will open a new connection up to the limit set by +CURLMOPT_MAX_HOST_CONNECTIONS. When the limit is reached, the sessions will +be pending until there are available connections. If CURLMOPT_PIPELINING is +1, libcurl will try to pipeline if the host is capable of it. + +The default value is 0, which means that there is no limit. +However, for backwards compatibility, setting it to 0 when CURLMOPT_PIPELINING +is 1 will not be treated as unlimited. Instead it will open only 1 connection +and try to pipeline on it. + +(Added in 7.30.0) +.IP CURLMOPT_MAX_PIPELINE_LENGTH +Pass a long. The set number will be used as the maximum amount of requests +in a pipelined connection. When this limit is reached, libcurl will use another +connection to the same host (see CURLMOPT_MAX_HOST_CONNECTIONS), or queue the +requests until one of the pipelines to the host is ready to accept a request. +Thus, the total number of requests in-flight is CURLMOPT_MAX_HOST_CONNECTIONS * +CURLMOPT_MAX_PIPELINE_LENGTH. +The default value is 5. + +(Added in 7.30.0) +.IP CURLMOPT_CONTENT_LENGTH_PENALTY_SIZE +Pass a long. If a pipelined connection is currently processing a request +with a Content-Length larger than CURLMOPT_CONTENT_LENGTH_PENALTY_SIZE, that +connection will not be considered for additional requests, even if it is +shorter than CURLMOPT_MAX_PIPELINE_LENGTH. +The default value is 0, which means that the penalization is inactive. + +(Added in 7.30.0) +.IP CURLMOPT_CHUNK_LENGTH_PENALTY_SIZE +Pass a long. If a pipelined connection is currently processing a +chunked (Transfer-encoding: chunked) request with a current chunk length +larger than CURLMOPT_CHUNK_LENGTH_PENALTY_SIZE, that connection will not be +considered for additional requests, even if it is shorter than +CURLMOPT_MAX_PIPELINE_LENGTH. +The default value is 0, which means that the penalization is inactive. + +(Added in 7.30.0) +.IP CURLMOPT_PIPELINING_SITE_BL +Pass an array of char *, ending with NULL. This is a list of sites that are +blacklisted from pipelining, i.e sites that are known to not support HTTP +pipelining. The array is copied by libcurl. + +The default value is NULL, which means that there is no blacklist. + +Pass a NULL pointer to clear the blacklist. + +Example: + +.nf + site_blacklist[] = + { + "www.haxx.se", + "www.example.com:1234", + NULL + }; + + curl_multi_setopt(m, CURLMOPT_PIPELINE_SITE_BL, site_blacklist); +.fi + +(Added in 7.30.0) +.IP CURLMOPT_PIPELINING_SERVER_BL +Pass an array of char *, ending with NULL. This is a list of server types +prefixes (in the Server: HTTP header) that are blacklisted from pipelining, +i.e server types that are known to not support HTTP pipelining. The array is +copied by libcurl. + +Note that the comparison matches if the Server: header begins with the string +in the blacklist, i.e "Server: Ninja 1.2.3" and "Server: Ninja 1.4.0" can +both be blacklisted by having "Ninja" in the backlist. + +The default value is NULL, which means that there is no blacklist. + +Pass a NULL pointer to clear the blacklist. + +Example: + +.nf + server_blacklist[] = + { + "Microsoft-IIS/6.0", + "nginx/0.8.54", + NULL + }; + + curl_multi_setopt(m, CURLMOPT_PIPELINE_SERVER_BL, server_blacklist); +.fi + +(Added in 7.30.0) +.IP CURLMOPT_MAX_TOTAL_CONNECTIONS +Pass a long. The set number will be used as the maximum amount of +simultaneously open connections in total. For each new session, libcurl +will open a new connection up to the limit set by +CURLMOPT_MAX_TOTAL_CONNECTIONS. When the limit is reached, the sessions will +be pending until there are available connections. If CURLMOPT_PIPELINING is +1, libcurl will try to pipeline if the host is capable of it. + +The default value is 0, which means that there is no limit. +However, for backwards compatibility, setting it to 0 when CURLMOPT_PIPELINING +is 1 will not be treated as unlimited. Instead it will open only 1 connection +and try to pipeline on it. + +(Added in 7.30.0) .SH RETURNS The standard CURLMcode for multi interface error codes. Note that it returns a CURLM_UNKNOWN_OPTION if you try setting an option that this version of libcurl diff --git a/docs/libcurl/libcurl-errors.3 b/docs/libcurl/libcurl-errors.3 index beee3971f..7b6823735 100644 --- a/docs/libcurl/libcurl-errors.3 +++ b/docs/libcurl/libcurl-errors.3 @@ -240,6 +240,9 @@ Mismatch of RTSP Session Identifiers. Unable to parse FTP file list (during FTP wildcard downloading). .IP "CURLE_CHUNK_FAILED (88)" Chunk callback reported error. +.IP "CURLE_NO_CONNECTION_AVAILABLE (89)" +(For internal use only, will never be returned by libcurl) No connection +available, the session will be queued. (added in 7.30.0) .IP "CURLE_OBSOLETE*" These error codes will never be returned. They were used in an old libcurl version and are currently unused. diff --git a/docs/libcurl/symbols-in-versions b/docs/libcurl/symbols-in-versions index 37b5e277d..5ed3f8477 100644 --- a/docs/libcurl/symbols-in-versions +++ b/docs/libcurl/symbols-in-versions @@ -85,6 +85,7 @@ CURLE_LDAP_SEARCH_FAILED 7.1 CURLE_LIBRARY_NOT_FOUND 7.1 7.17.0 CURLE_LOGIN_DENIED 7.13.1 CURLE_MALFORMAT_USER 7.1 7.17.0 +CURLE_NO_CONNECTION_AVAILABLE 7.30.0 CURLE_NOT_BUILT_IN 7.21.5 CURLE_OK 7.1 CURLE_OPERATION_TIMEDOUT 7.10.2 @@ -267,8 +268,15 @@ CURLKHTYPE_DSS 7.19.6 CURLKHTYPE_RSA 7.19.6 CURLKHTYPE_RSA1 7.19.6 CURLKHTYPE_UNKNOWN 7.19.6 +CURLMOPT_CHUNK_LENGTH_PENALTY_SIZE 7.30.0 +CURLMOPT_CONTENT_LENGTH_PENALTY_SIZE 7.30.0 +CURLMOPT_MAX_HOST_CONNECTIONS 7.30.0 +CURLMOPT_MAX_PIPELINE_LENGTH 7.30.0 +CURLMOPT_MAX_TOTAL_CONNECTIONS 7.30.0 CURLMOPT_MAXCONNECTS 7.16.3 CURLMOPT_PIPELINING 7.16.0 +CURLMOPT_PIPELINING_SERVER_BL 7.30.0 +CURLMOPT_PIPELINING_SITE_BL 7.30.0 CURLMOPT_SOCKETDATA 7.15.4 CURLMOPT_SOCKETFUNCTION 7.15.4 CURLMOPT_TIMERDATA 7.16.0 diff --git a/include/curl/curl.h b/include/curl/curl.h index 0b2b7ea44..c7a052d1e 100644 --- a/include/curl/curl.h +++ b/include/curl/curl.h @@ -507,6 +507,8 @@ typedef enum { CURLE_RTSP_SESSION_ERROR, /* 86 - mismatch of RTSP Session Ids */ CURLE_FTP_BAD_FILE_LIST, /* 87 - unable to parse FTP file list */ CURLE_CHUNK_FAILED, /* 88 - chunk callback reported error */ + CURLE_NO_CONNECTION_AVAILABLE, /* 89 - No connection available, the + session will be queued */ CURL_LAST /* never use! */ } CURLcode; diff --git a/include/curl/multi.h b/include/curl/multi.h index 6dcd2bac4..a5eb3c643 100644 --- a/include/curl/multi.h +++ b/include/curl/multi.h @@ -338,6 +338,31 @@ typedef enum { /* maximum number of entries in the connection cache */ CINIT(MAXCONNECTS, LONG, 6), + /* maximum number of (pipelining) connections to one host */ + CINIT(MAX_HOST_CONNECTIONS, LONG, 7), + + /* maximum number of requests in a pipeline */ + CINIT(MAX_PIPELINE_LENGTH, LONG, 8), + + /* a connection with a content-length longer than this + will not be considered for pipelining */ + CINIT(CONTENT_LENGTH_PENALTY_SIZE, OFF_T, 9), + + /* a connection with a chunk length longer than this + will not be considered for pipelining */ + CINIT(CHUNK_LENGTH_PENALTY_SIZE, OFF_T, 10), + + /* a list of site names(+port) that are blacklisted from + pipelining */ + CINIT(PIPELINING_SITE_BL, OBJECTPOINT, 11), + + /* a list of server types that are blacklisted from + pipelining */ + CINIT(PIPELINING_SERVER_BL, OBJECTPOINT, 12), + + /* maximum number of open connections in total */ + CINIT(MAX_TOTAL_CONNECTIONS, LONG, 13), + CURLMOPT_LASTENTRY /* the last unused */ } CURLMoption; diff --git a/lib/Makefile.inc b/lib/Makefile.inc index db0597365..f76e1ec83 100644 --- a/lib/Makefile.inc +++ b/lib/Makefile.inc @@ -25,7 +25,7 @@ CSOURCES = file.c timeval.c base64.c hostip.c progress.c formdata.c \ http_proxy.c non-ascii.c asyn-ares.c asyn-thread.c curl_gssapi.c \ curl_ntlm.c curl_ntlm_wb.c curl_ntlm_core.c curl_ntlm_msgs.c \ curl_sasl.c curl_schannel.c curl_multibyte.c curl_darwinssl.c \ - hostcheck.c bundles.c conncache.c + hostcheck.c bundles.c conncache.c pipeline.c HHEADERS = arpa_telnet.h netrc.h file.h timeval.h qssl.h hostip.h \ progress.h formdata.h cookie.h http.h sendf.h ftp.h url.h dict.h \ @@ -44,4 +44,4 @@ HHEADERS = arpa_telnet.h netrc.h file.h timeval.h qssl.h hostip.h \ asyn.h curl_ntlm.h curl_gssapi.h curl_ntlm_wb.h curl_ntlm_core.h \ curl_ntlm_msgs.h curl_sasl.h curl_schannel.h curl_multibyte.h \ curl_darwinssl.h hostcheck.h bundles.h conncache.h curl_setup_once.h \ - multihandle.h setup-vms.h + multihandle.h setup-vms.h pipeline.h diff --git a/lib/README.pipelining b/lib/README.pipelining index c7b462248..e5bf6ec33 100644 --- a/lib/README.pipelining +++ b/lib/README.pipelining @@ -42,10 +42,3 @@ Details still resolve the second one properly to make sure that they actually _can_ be considered for pipelining. Also, asking for explicit pipelining on handle X may be tricky when handle X get a closed connection. - -- We need options to control max pipeline length, and probably how to behave - if we reach that limit. As was discussed on the list, it can probably be - made very complicated, so perhaps we can think of a way to pass all - variables involved to a callback and let the application decide how to act - in specific situations. Either way, these fancy options are only interesting - to work on when everything is working and we have working apps to test with. diff --git a/lib/hash.h b/lib/hash.h index 99a627405..aa935d4eb 100644 --- a/lib/hash.h +++ b/lib/hash.h @@ -104,4 +104,3 @@ void Curl_hash_print(struct curl_hash *h, #endif /* HEADER_CURL_HASH_H */ - diff --git a/lib/http.c b/lib/http.c index daaafe317..0ba11133f 100644 --- a/lib/http.c +++ b/lib/http.c @@ -73,6 +73,8 @@ #include "http_proxy.h" #include "warnless.h" #include "non-ascii.h" +#include "bundles.h" +#include "pipeline.h" #define _MPRINTF_REPLACE /* use our functions only */ #include @@ -3148,13 +3150,19 @@ CURLcode Curl_http_readwrite_headers(struct SessionHandle *data, } else if(conn->httpversion >= 11 && !conn->bits.close) { + struct connectbundle *cb_ptr; /* If HTTP version is >= 1.1 and connection is persistent server supports pipelining. */ DEBUGF(infof(data, "HTTP 1.1 or later with persistent connection, " "pipelining supported\n")); - conn->server_supports_pipelining = TRUE; + /* Activate pipelining if needed */ + cb_ptr = conn->bundle; + if(cb_ptr) { + if(!Curl_pipeline_site_blacklisted(data, conn)) + cb_ptr->server_supports_pipelining = TRUE; + } } switch(k->httpcode) { @@ -3231,6 +3239,16 @@ CURLcode Curl_http_readwrite_headers(struct SessionHandle *data, data->info.contenttype = contenttype; } } + else if(checkprefix("Server:", k->p)) { + char *server_name = copy_header_value(k->p); + + /* Turn off pipelining if the server version is blacklisted */ + if(conn->bundle && conn->bundle->server_supports_pipelining) { + if(Curl_pipeline_server_blacklisted(data, server_name)) + conn->bundle->server_supports_pipelining = FALSE; + } + Curl_safefree(server_name); + } else if((conn->httpversion == 10) && conn->bits.httpproxy && Curl_compareheader(k->p, diff --git a/lib/multi.c b/lib/multi.c index a369d0361..3e2583a21 100644 --- a/lib/multi.c +++ b/lib/multi.c @@ -40,6 +40,7 @@ #include "conncache.h" #include "bundles.h" #include "multihandle.h" +#include "pipeline.h" #define _MPRINTF_REPLACE /* use our functions only */ #include @@ -69,13 +70,6 @@ static void singlesocket(struct Curl_multi *multi, struct Curl_one_easy *easy); static int update_timer(struct Curl_multi *multi); -static CURLcode addHandleToSendOrPendPipeline(struct SessionHandle *handle, - struct connectdata *conn); -static int checkPendPipeline(struct connectdata *conn); -static void moveHandleFromSendToRecvPipeline(struct SessionHandle *handle, - struct connectdata *conn); -static void moveHandleFromRecvToDonePipeline(struct SessionHandle *handle, - struct connectdata *conn); static bool isHandleAtHead(struct SessionHandle *handle, struct curl_llist *pipeline); static CURLMcode add_next_timeout(struct timeval now, @@ -85,6 +79,7 @@ static CURLMcode add_next_timeout(struct timeval now, #ifdef DEBUGBUILD static const char * const statename[]={ "INIT", + "CONNECT_PEND", "CONNECT", "WAITRESOLVE", "WAITCONNECT", @@ -125,9 +120,9 @@ static void mstate(struct Curl_one_easy *easy, CURLMstate state easy->state = state; #ifdef DEBUGBUILD - if(easy->easy_conn) { - if(easy->state > CURLM_STATE_CONNECT && - easy->state < CURLM_STATE_COMPLETED) + if(easy->state >= CURLM_STATE_CONNECT_PEND && + easy->state < CURLM_STATE_COMPLETED) { + if(easy->easy_conn) connection_id = easy->easy_conn->connection_id; infof(easy->easy_handle, @@ -314,6 +309,7 @@ CURLM *curl_multi_init(void) multi->easy.next = &multi->easy; multi->easy.prev = &multi->easy; + multi->max_pipeline_length = 5; return (CURLM *) multi; error: @@ -580,7 +576,7 @@ CURLMcode curl_multi_remove_handle(CURLM *multi_handle, /* as this was using a shared connection cache we clear the pointer to that since we're not part of that multi handle anymore */ - easy->easy_handle->state.conn_cache = NULL; + easy->easy_handle->state.conn_cache = NULL; /* change state without using multistate(), only to make singlesocket() do what we want */ @@ -638,9 +634,9 @@ CURLMcode curl_multi_remove_handle(CURLM *multi_handle, return CURLM_BAD_EASY_HANDLE; /* twasn't found */ } -bool Curl_multi_canPipeline(const struct Curl_multi* multi) +bool Curl_multi_pipeline_enabled(const struct Curl_multi* multi) { - return multi->pipelining_enabled; + return multi && multi->pipelining_enabled; } void Curl_multi_handlePipeBreak(struct SessionHandle *data) @@ -1007,16 +1003,27 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, } break; + case CURLM_STATE_CONNECT_PEND: + /* We will stay here until there is a connection available. Then + we try again in the CURLM_STATE_CONNECT state. */ + break; + case CURLM_STATE_CONNECT: - /* Connect. We get a connection identifier filled in. */ + /* Connect. We want to get a connection identifier filled in. */ Curl_pgrsTime(data, TIMER_STARTSINGLE); easy->result = Curl_connect(data, &easy->easy_conn, &async, &protocol_connect); + if(CURLE_NO_CONNECTION_AVAILABLE == easy->result) { + /* There was no connection available. We will go to the pending + state and wait for an available connection. */ + multistate(easy, CURLM_STATE_CONNECT_PEND); + easy->result = CURLM_OK; + break; + } if(CURLE_OK == easy->result) { /* Add this handle to the send or pend pipeline */ - easy->result = addHandleToSendOrPendPipeline(data, - easy->easy_conn); + easy->result = Curl_add_handle_to_pipeline(data, easy->easy_conn); if(CURLE_OK != easy->result) disconnect_conn = TRUE; else { @@ -1357,9 +1364,9 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, case CURLM_STATE_DO_DONE: /* Move ourselves from the send to recv pipeline */ - moveHandleFromSendToRecvPipeline(data, easy->easy_conn); + Curl_move_handle_from_send_to_recv_pipe(data, easy->easy_conn); /* Check if we can move pending requests to send pipe */ - checkPendPipeline(easy->easy_conn); + Curl_multi_process_pending_handles(multi); multistate(easy, CURLM_STATE_WAITPERFORM); result = CURLM_CALL_MULTI_PERFORM; break; @@ -1491,15 +1498,14 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, Curl_posttransfer(data); /* we're no longer receiving */ - moveHandleFromRecvToDonePipeline(data, - easy->easy_conn); + Curl_removeHandleFromPipeline(data, easy->easy_conn->recv_pipe); /* expire the new receiving pipeline head */ if(easy->easy_conn->recv_pipe->head) Curl_expire(easy->easy_conn->recv_pipe->head->ptr, 1); /* Check if we can move pending requests to send pipe */ - checkPendPipeline(easy->easy_conn); + Curl_multi_process_pending_handles(multi); /* When we follow redirects or is set to retry the connection, we must to go back to the CONNECT state */ @@ -1554,14 +1560,11 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, case CURLM_STATE_DONE: if(easy->easy_conn) { - /* Remove ourselves from the receive and done pipelines. Handle - should be on one of these lists, depending upon how we got here. */ + /* Remove ourselves from the receive pipeline, if we are there. */ Curl_removeHandleFromPipeline(data, easy->easy_conn->recv_pipe); - Curl_removeHandleFromPipeline(data, - easy->easy_conn->done_pipe); /* Check if we can move pending requests to send pipe */ - checkPendPipeline(easy->easy_conn); + Curl_multi_process_pending_handles(multi); if(easy->easy_conn->bits.stream_was_rewound) { /* This request read past its response boundary so we quickly let @@ -1638,10 +1641,8 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, easy->easy_conn->send_pipe); Curl_removeHandleFromPipeline(data, easy->easy_conn->recv_pipe); - Curl_removeHandleFromPipeline(data, - easy->easy_conn->done_pipe); /* Check if we can move pending requests to send pipe */ - checkPendPipeline(easy->easy_conn); + Curl_multi_process_pending_handles(multi); if(disconnect_conn) { /* disconnect properly */ @@ -1789,8 +1790,8 @@ CURLMcode curl_multi_cleanup(CURLM *multi_handle) Curl_hostcache_clean(multi->closure_handle); Curl_close(multi->closure_handle); + multi->closure_handle = NULL; } - multi->closure_handle = NULL; Curl_hash_destroy(multi->sockhash); multi->sockhash = NULL; @@ -1825,6 +1826,10 @@ CURLMcode curl_multi_cleanup(CURLM *multi_handle) Curl_hash_destroy(multi->hostcache); multi->hostcache = NULL; + /* Free the blacklists by setting them to NULL */ + Curl_pipeline_set_site_blacklist(NULL, &multi->pipelining_site_bl); + Curl_pipeline_set_server_blacklist(NULL, &multi->pipelining_server_bl); + free(multi); return CURLM_OK; @@ -2242,6 +2247,29 @@ CURLMcode curl_multi_setopt(CURLM *multi_handle, case CURLMOPT_MAXCONNECTS: multi->maxconnects = va_arg(param, long); break; + case CURLMOPT_MAX_HOST_CONNECTIONS: + multi->max_host_connections = va_arg(param, long); + break; + case CURLMOPT_MAX_PIPELINE_LENGTH: + multi->max_pipeline_length = va_arg(param, long); + break; + case CURLMOPT_CONTENT_LENGTH_PENALTY_SIZE: + multi->content_length_penalty_size = va_arg(param, long); + break; + case CURLMOPT_CHUNK_LENGTH_PENALTY_SIZE: + multi->chunk_length_penalty_size = va_arg(param, long); + break; + case CURLMOPT_PIPELINING_SITE_BL: + res = Curl_pipeline_set_site_blacklist(va_arg(param, char **), + &multi->pipelining_site_bl); + break; + case CURLMOPT_PIPELINING_SERVER_BL: + res = Curl_pipeline_set_server_blacklist(va_arg(param, char **), + &multi->pipelining_server_bl); + break; + case CURLMOPT_MAX_TOTAL_CONNECTIONS: + multi->max_total_connections = va_arg(param, long); + break; default: res = CURLM_UNKNOWN_OPTION; break; @@ -2366,131 +2394,12 @@ static int update_timer(struct Curl_multi *multi) return multi->timer_cb((CURLM*)multi, timeout_ms, multi->timer_userp); } -static CURLcode addHandleToSendOrPendPipeline(struct SessionHandle *handle, +void Curl_multi_set_easy_connection(struct SessionHandle *handle, struct connectdata *conn) { - size_t pipeLen = conn->send_pipe->size + conn->recv_pipe->size; - struct curl_llist_element *sendhead = conn->send_pipe->head; - struct curl_llist *pipeline; - CURLcode rc; - - if(!Curl_isPipeliningEnabled(handle) || - pipeLen == 0) - pipeline = conn->send_pipe; - else { - if(conn->server_supports_pipelining && - pipeLen < MAX_PIPELINE_LENGTH) - pipeline = conn->send_pipe; - else - pipeline = conn->pend_pipe; - } - - rc = Curl_addHandleToPipeline(handle, pipeline); - - if(pipeline == conn->send_pipe && sendhead != conn->send_pipe->head) { - /* this is a new one as head, expire it */ - conn->writechannel_inuse = FALSE; /* not in use yet */ -#ifdef DEBUGBUILD - infof(conn->data, "%p is at send pipe head!\n", - conn->send_pipe->head->ptr); -#endif - Curl_expire(conn->send_pipe->head->ptr, 1); - } - - return rc; + handle->set.one_easy->easy_conn = conn; } -static int checkPendPipeline(struct connectdata *conn) -{ - int result = 0; - struct curl_llist_element *sendhead = conn->send_pipe->head; - - size_t pipeLen = conn->send_pipe->size + conn->recv_pipe->size; - if(conn->server_supports_pipelining || pipeLen == 0) { - struct curl_llist_element *curr = conn->pend_pipe->head; - const size_t maxPipeLen = - conn->server_supports_pipelining ? MAX_PIPELINE_LENGTH : 1; - - while(pipeLen < maxPipeLen && curr) { - Curl_llist_move(conn->pend_pipe, curr, - conn->send_pipe, conn->send_pipe->tail); - Curl_pgrsTime(curr->ptr, TIMER_PRETRANSFER); - ++result; /* count how many handles we moved */ - curr = conn->pend_pipe->head; - ++pipeLen; - } - } - - if(result) { - conn->now = Curl_tvnow(); - /* something moved, check for a new send pipeline leader */ - if(sendhead != conn->send_pipe->head) { - /* this is a new one as head, expire it */ - conn->writechannel_inuse = FALSE; /* not in use yet */ -#ifdef DEBUGBUILD - infof(conn->data, "%p is at send pipe head!\n", - conn->send_pipe->head->ptr); -#endif - Curl_expire(conn->send_pipe->head->ptr, 1); - } - } - - return result; -} - -/* Move this transfer from the sending list to the receiving list. - - Pay special attention to the new sending list "leader" as it needs to get - checked to update what sockets it acts on. - -*/ -static void moveHandleFromSendToRecvPipeline(struct SessionHandle *handle, - struct connectdata *conn) -{ - struct curl_llist_element *curr; - - curr = conn->send_pipe->head; - while(curr) { - if(curr->ptr == handle) { - Curl_llist_move(conn->send_pipe, curr, - conn->recv_pipe, conn->recv_pipe->tail); - - if(conn->send_pipe->head) { - /* Since there's a new easy handle at the start of the send pipeline, - set its timeout value to 1ms to make it trigger instantly */ - conn->writechannel_inuse = FALSE; /* not used now */ -#ifdef DEBUGBUILD - infof(conn->data, "%p is at send pipe head B!\n", - conn->send_pipe->head->ptr); -#endif - Curl_expire(conn->send_pipe->head->ptr, 1); - } - - /* The receiver's list is not really interesting here since either this - handle is now first in the list and we'll deal with it soon, or - another handle is already first and thus is already taken care of */ - - break; /* we're done! */ - } - curr = curr->next; - } -} - -static void moveHandleFromRecvToDonePipeline(struct SessionHandle *handle, - struct connectdata *conn) -{ - struct curl_llist_element *curr; - - curr = conn->recv_pipe->head; - while(curr) { - if(curr->ptr == handle) { - Curl_llist_move(conn->recv_pipe, curr, - conn->done_pipe, conn->done_pipe->tail); - break; - } - curr = curr->next; - } -} static bool isHandleAtHead(struct SessionHandle *handle, struct curl_llist *pipeline) { @@ -2670,6 +2579,56 @@ CURLMcode curl_multi_assign(CURLM *multi_handle, return CURLM_OK; } +size_t Curl_multi_max_host_connections(struct Curl_multi *multi) +{ + return multi ? multi->max_host_connections : 0; +} + +size_t Curl_multi_max_total_connections(struct Curl_multi *multi) +{ + return multi ? multi->max_total_connections : 0; +} + +size_t Curl_multi_max_pipeline_length(struct Curl_multi *multi) +{ + return multi ? multi->max_pipeline_length : 0; +} + +curl_off_t Curl_multi_content_length_penalty_size(struct Curl_multi *multi) +{ + return multi ? multi->content_length_penalty_size : 0; +} + +curl_off_t Curl_multi_chunk_length_penalty_size(struct Curl_multi *multi) +{ + return multi ? multi->chunk_length_penalty_size : 0; +} + +struct curl_llist *Curl_multi_pipelining_site_bl(struct Curl_multi *multi) +{ + return multi->pipelining_site_bl; +} + +struct curl_llist *Curl_multi_pipelining_server_bl(struct Curl_multi *multi) +{ + return multi->pipelining_server_bl; +} + +void Curl_multi_process_pending_handles(struct Curl_multi *multi) +{ + struct Curl_one_easy *easy; + + easy=multi->easy.next; + while(easy != &multi->easy) { + if(easy->state == CURLM_STATE_CONNECT_PEND) { + multistate(easy, CURLM_STATE_CONNECT); + /* Make sure that the handle will be processed soonish. */ + Curl_expire(easy->easy_handle, 1); + } + easy = easy->next; /* operate on next handle */ + } +} + #ifdef DEBUGBUILD void Curl_multi_dump(const struct Curl_multi *multi_handle) { diff --git a/lib/multihandle.h b/lib/multihandle.h index 941835941..3fcd9c0e6 100644 --- a/lib/multihandle.h +++ b/lib/multihandle.h @@ -31,25 +31,26 @@ struct Curl_message { well! */ typedef enum { - CURLM_STATE_INIT, /* 0 - start in this state */ - CURLM_STATE_CONNECT, /* 1 - resolve/connect has been sent off */ - CURLM_STATE_WAITRESOLVE, /* 2 - awaiting the resolve to finalize */ - CURLM_STATE_WAITCONNECT, /* 3 - awaiting the connect to finalize */ - CURLM_STATE_WAITPROXYCONNECT, /* 4 - awaiting proxy CONNECT to finalize */ - CURLM_STATE_PROTOCONNECT, /* 5 - completing the protocol-specific connect - phase */ - CURLM_STATE_WAITDO, /* 6 - wait for our turn to send the request */ - CURLM_STATE_DO, /* 7 - start send off the request (part 1) */ - CURLM_STATE_DOING, /* 8 - sending off the request (part 1) */ - CURLM_STATE_DO_MORE, /* 9 - send off the request (part 2) */ - CURLM_STATE_DO_DONE, /* 10 - done sending off request */ - CURLM_STATE_WAITPERFORM, /* 11 - wait for our turn to read the response */ - CURLM_STATE_PERFORM, /* 12 - transfer data */ - CURLM_STATE_TOOFAST, /* 13 - wait because limit-rate exceeded */ - CURLM_STATE_DONE, /* 14 - post data transfer operation */ - CURLM_STATE_COMPLETED, /* 15 - operation complete */ - CURLM_STATE_MSGSENT, /* 16 - the operation complete message is sent */ - CURLM_STATE_LAST /* 17 - not a true state, never use this */ + CURLM_STATE_INIT, /* 0 - start in this state */ + CURLM_STATE_CONNECT_PEND, /* 1 - no connections, waiting for one */ + CURLM_STATE_CONNECT, /* 2 - resolve/connect has been sent off */ + CURLM_STATE_WAITRESOLVE, /* 3 - awaiting the resolve to finalize */ + CURLM_STATE_WAITCONNECT, /* 4 - awaiting the connect to finalize */ + CURLM_STATE_WAITPROXYCONNECT, /* 5 - awaiting proxy CONNECT to finalize */ + CURLM_STATE_PROTOCONNECT, /* 6 - completing the protocol-specific connect + phase */ + CURLM_STATE_WAITDO, /* 7 - wait for our turn to send the request */ + CURLM_STATE_DO, /* 8 - start send off the request (part 1) */ + CURLM_STATE_DOING, /* 9 - sending off the request (part 1) */ + CURLM_STATE_DO_MORE, /* 10 - send off the request (part 2) */ + CURLM_STATE_DO_DONE, /* 11 - done sending off request */ + CURLM_STATE_WAITPERFORM, /* 12 - wait for our turn to read the response */ + CURLM_STATE_PERFORM, /* 13 - transfer data */ + CURLM_STATE_TOOFAST, /* 14 - wait because limit-rate exceeded */ + CURLM_STATE_DONE, /* 15 - post data transfer operation */ + CURLM_STATE_COMPLETED, /* 16 - operation complete */ + CURLM_STATE_MSGSENT, /* 17 - the operation complete message is sent */ + CURLM_STATE_LAST /* 18 - not a true state, never use this */ } CURLMstate; /* we support N sockets per easy handle. Set the corresponding bit to what @@ -123,6 +124,30 @@ struct Curl_multi { long maxconnects; /* if >0, a fixed limit of the maximum number of entries we're allowed to grow the connection cache to */ + long max_host_connections; /* if >0, a fixed limit of the maximum number + of connections per host */ + + long max_total_connections; /* if >0, a fixed limit of the maximum number + of connections in total */ + + long max_pipeline_length; /* if >0, maximum number of requests in a + pipeline */ + + long content_length_penalty_size; /* a connection with a + content-length bigger than + this is not considered + for pipelining */ + + long chunk_length_penalty_size; /* a connection with a chunk length + bigger than this is not + considered for pipelining */ + + struct curl_llist *pipelining_site_bl; /* List of sites that are blacklisted + from pipelining */ + + struct curl_llist *pipelining_server_bl; /* List of server types that are + blacklisted from pipelining */ + /* timer callback and user data pointer for the *socket() API */ curl_multi_timer_callback timer_cb; void *timer_userp; diff --git a/lib/multiif.h b/lib/multiif.h index c84b6184c..0dcdec76e 100644 --- a/lib/multiif.h +++ b/lib/multiif.h @@ -27,7 +27,7 @@ */ void Curl_expire(struct SessionHandle *data, long milli); -bool Curl_multi_canPipeline(const struct Curl_multi* multi); +bool Curl_multi_pipeline_enabled(const struct Curl_multi* multi); void Curl_multi_handlePipeBreak(struct SessionHandle *data); /* the write bits start at bit 16 for the *getsock() bitmap */ @@ -50,5 +50,31 @@ void Curl_multi_handlePipeBreak(struct SessionHandle *data); void Curl_multi_dump(const struct Curl_multi *multi_handle); #endif -#endif /* HEADER_CURL_MULTIIF_H */ +/* Update the current connection of a One_Easy handle */ +void Curl_multi_set_easy_connection(struct SessionHandle *handle, + struct connectdata *conn); +void Curl_multi_process_pending_handles(struct Curl_multi *multi); + +/* Return the value of the CURLMOPT_MAX_HOST_CONNECTIONS option */ +size_t Curl_multi_max_host_connections(struct Curl_multi *multi); + +/* Return the value of the CURLMOPT_MAX_PIPELINE_LENGTH option */ +size_t Curl_multi_max_pipeline_length(struct Curl_multi *multi); + +/* Return the value of the CURLMOPT_CONTENT_LENGTH_PENALTY_SIZE option */ +curl_off_t Curl_multi_content_length_penalty_size(struct Curl_multi *multi); + +/* Return the value of the CURLMOPT_CHUNK_LENGTH_PENALTY_SIZE option */ +curl_off_t Curl_multi_chunk_length_penalty_size(struct Curl_multi *multi); + +/* Return the value of the CURLMOPT_PIPELINING_SITE_BL option */ +struct curl_llist *Curl_multi_pipelining_site_bl(struct Curl_multi *multi); + +/* Return the value of the CURLMOPT_PIPELINING_SERVER_BL option */ +struct curl_llist *Curl_multi_pipelining_server_bl(struct Curl_multi *multi); + +/* Return the value of the CURLMOPT_MAX_TOTAL_CONNECTIONS option */ +size_t Curl_multi_max_total_connections(struct Curl_multi *multi); + +#endif /* HEADER_CURL_MULTIIF_H */ diff --git a/lib/pipeline.c b/lib/pipeline.c new file mode 100644 index 000000000..7abc35fd0 --- /dev/null +++ b/lib/pipeline.c @@ -0,0 +1,366 @@ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 2013, Linus Nielsen Feltzing, + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +#include "curl_setup.h" + +#include + +#include "urldata.h" +#include "url.h" +#include "progress.h" +#include "multiif.h" +#include "pipeline.h" +#include "sendf.h" +#include "rawstr.h" +#include "bundles.h" + +#include "curl_memory.h" +/* The last #include file should be: */ +#include "memdebug.h" + +struct site_blacklist_entry { + char *hostname; + unsigned short port; +}; + +static void site_blacklist_llist_dtor(void *user, void *element) +{ + struct site_blacklist_entry *entry = element; + (void)user; + + Curl_safefree(entry->hostname); + Curl_safefree(entry); +} + +static void server_blacklist_llist_dtor(void *user, void *element) +{ + char *server_name = element; + (void)user; + + Curl_safefree(server_name); +} + +bool Curl_pipeline_penalized(struct SessionHandle *data, + struct connectdata *conn) +{ + if(data) { + bool penalized = FALSE; + curl_off_t penalty_size = + Curl_multi_content_length_penalty_size(data->multi); + curl_off_t chunk_penalty_size = + Curl_multi_chunk_length_penalty_size(data->multi); + curl_off_t recv_size = -2; /* Make it easy to spot in the log */ + + /* Find the head of the recv pipe, if any */ + if(conn->recv_pipe && conn->recv_pipe->head) { + struct SessionHandle *recv_handle = conn->recv_pipe->head->ptr; + + recv_size = recv_handle->req.size; + + if(penalty_size > 0 && recv_size > penalty_size) + penalized = TRUE; + } + + if(chunk_penalty_size > 0 && + (curl_off_t)conn->chunk.datasize > chunk_penalty_size) + penalized = TRUE; + + infof(data, "Conn: %d (%p) Receive pipe weight: (%d/%d), penalized: %d\n", + conn->connection_id, conn, recv_size, + conn->chunk.datasize, penalized); + return penalized; + } + return FALSE; +} + +/* Find the best connection in a bundle to use for the next request */ +struct connectdata * +Curl_bundle_find_best(struct SessionHandle *data, + struct connectbundle *cb_ptr) +{ + struct curl_llist_element *curr; + struct connectdata *conn; + struct connectdata *best_conn = NULL; + size_t pipe_len; + size_t best_pipe_len = 99; + + (void)data; + + curr = cb_ptr->conn_list->head; + while(curr) { + conn = curr->ptr; + pipe_len = conn->send_pipe->size + conn->recv_pipe->size; + + if(!Curl_pipeline_penalized(conn->data, conn) && + pipe_len < best_pipe_len) { + best_conn = conn; + best_pipe_len = pipe_len; + } + curr = curr->next; + } + + /* If we haven't found a connection, i.e all pipelines are penalized + or full, just pick one. The request will then be queued in + Curl_add_handle_to_pipeline(). */ + if(!best_conn) { + best_conn = cb_ptr->conn_list->head->ptr; + } + return best_conn; +} + +CURLcode Curl_add_handle_to_pipeline(struct SessionHandle *handle, + struct connectdata *conn) +{ + struct curl_llist_element *sendhead = conn->send_pipe->head; + struct curl_llist *pipeline; + CURLcode rc; + + pipeline = conn->send_pipe; + + infof(conn->data, "Adding handle: conn: %p\n", conn); + infof(conn->data, "Adding handle: send: %d\n", conn->send_pipe->size); + infof(conn->data, "Adding handle: recv: %d\n", conn->recv_pipe->size); + rc = Curl_addHandleToPipeline(handle, pipeline); + + if(pipeline == conn->send_pipe && sendhead != conn->send_pipe->head) { + /* this is a new one as head, expire it */ + conn->writechannel_inuse = FALSE; /* not in use yet */ +#ifdef DEBUGBUILD + infof(conn->data, "%p is at send pipe head!\n", + conn->send_pipe->head->ptr); +#endif + Curl_expire(conn->send_pipe->head->ptr, 1); + } + + print_pipeline(conn); + + return rc; +} + +/* Move this transfer from the sending list to the receiving list. + + Pay special attention to the new sending list "leader" as it needs to get + checked to update what sockets it acts on. + +*/ +void Curl_move_handle_from_send_to_recv_pipe(struct SessionHandle *handle, + struct connectdata *conn) +{ + struct curl_llist_element *curr; + + curr = conn->send_pipe->head; + while(curr) { + if(curr->ptr == handle) { + Curl_llist_move(conn->send_pipe, curr, + conn->recv_pipe, conn->recv_pipe->tail); + + if(conn->send_pipe->head) { + /* Since there's a new easy handle at the start of the send pipeline, + set its timeout value to 1ms to make it trigger instantly */ + conn->writechannel_inuse = FALSE; /* not used now */ +#ifdef DEBUGBUILD + infof(conn->data, "%p is at send pipe head B!\n", + conn->send_pipe->head->ptr); +#endif + Curl_expire(conn->send_pipe->head->ptr, 1); + } + + /* The receiver's list is not really interesting here since either this + handle is now first in the list and we'll deal with it soon, or + another handle is already first and thus is already taken care of */ + + break; /* we're done! */ + } + curr = curr->next; + } +} + +bool Curl_pipeline_site_blacklisted(struct SessionHandle *handle, + struct connectdata *conn) +{ + if(handle->multi) { + struct curl_llist *blacklist = + Curl_multi_pipelining_site_bl(handle->multi); + + if(blacklist) { + struct curl_llist_element *curr; + + curr = blacklist->head; + while(curr) { + struct site_blacklist_entry *site; + + site = curr->ptr; + if(Curl_raw_equal(site->hostname, conn->host.name) && + site->port == conn->remote_port) { + infof(handle, "Site %s:%d is pipeline blacklisted\n", + conn->host.name, conn->remote_port); + return TRUE; + } + curr = curr->next; + } + } + } + return FALSE; +} + +CURLMcode Curl_pipeline_set_site_blacklist(char **sites, + struct curl_llist **list_ptr) +{ + struct curl_llist *old_list = *list_ptr; + struct curl_llist *new_list = NULL; + + if(sites) { + new_list = Curl_llist_alloc((curl_llist_dtor) site_blacklist_llist_dtor); + if(!new_list) + return CURLM_OUT_OF_MEMORY; + + /* Parse the URLs and populate the list */ + while(*sites) { + char *hostname; + char *port; + struct site_blacklist_entry *entry; + + entry = malloc(sizeof(struct site_blacklist_entry)); + + hostname = strdup(*sites); + if(!hostname) + return CURLM_OUT_OF_MEMORY; + + port = strchr(hostname, ':'); + if(port) { + *port = '\0'; + port++; + entry->port = (unsigned short)strtol(port, NULL, 10); + } + else { + /* Default port number for HTTP */ + entry->port = 80; + } + + entry->hostname = hostname; + + if(!Curl_llist_insert_next(new_list, new_list->tail, entry)) + return CURLM_OUT_OF_MEMORY; + + sites++; + } + } + + /* Free the old list */ + if(old_list) { + Curl_llist_destroy(old_list, NULL); + } + + /* This might be NULL if sites == NULL, i.e the blacklist is cleared */ + *list_ptr = new_list; + + return CURLM_OK; +} + +bool Curl_pipeline_server_blacklisted(struct SessionHandle *handle, + char *server_name) +{ + if(handle->multi) { + struct curl_llist *blacklist = + Curl_multi_pipelining_server_bl(handle->multi); + + if(blacklist) { + struct curl_llist_element *curr; + + curr = blacklist->head; + while(curr) { + char *bl_server_name; + + bl_server_name = curr->ptr; + if(Curl_raw_nequal(bl_server_name, server_name, + strlen(bl_server_name))) { + infof(handle, "Server %s is blacklisted\n", server_name); + return TRUE; + } + curr = curr->next; + } + } + + infof(handle, "Server %s is not blacklisted\n", server_name); + } + return FALSE; +} + +CURLMcode Curl_pipeline_set_server_blacklist(char **servers, + struct curl_llist **list_ptr) +{ + struct curl_llist *old_list = *list_ptr; + struct curl_llist *new_list = NULL; + + if(servers) { + new_list = Curl_llist_alloc((curl_llist_dtor) server_blacklist_llist_dtor); + if(!new_list) + return CURLM_OUT_OF_MEMORY; + + /* Parse the URLs and populate the list */ + while(*servers) { + char *server_name; + + server_name = strdup(*servers); + if(!server_name) + return CURLM_OUT_OF_MEMORY; + + if(!Curl_llist_insert_next(new_list, new_list->tail, server_name)) + return CURLM_OUT_OF_MEMORY; + + servers++; + } + } + + /* Free the old list */ + if(old_list) { + Curl_llist_destroy(old_list, NULL); + } + + /* This might be NULL if sites == NULL, i.e the blacklist is cleared */ + *list_ptr = new_list; + + return CURLM_OK; +} + + +void print_pipeline(struct connectdata *conn) +{ + struct curl_llist_element *curr; + struct connectbundle *cb_ptr; + struct SessionHandle *data = conn->data; + + cb_ptr = conn->bundle; + + if(cb_ptr) { + curr = cb_ptr->conn_list->head; + while(curr) { + conn = curr->ptr; + infof(data, "- Conn %d (%p) send_pipe: %d, recv_pipe: %d\n", + conn->connection_id, + conn, + conn->send_pipe->size, + conn->recv_pipe->size); + curr = curr->next; + } + } +} diff --git a/lib/pipeline.h b/lib/pipeline.h new file mode 100644 index 000000000..f3a734c9a --- /dev/null +++ b/lib/pipeline.h @@ -0,0 +1,50 @@ +#ifndef HEADER_CURL_PIPELINE_H +#define HEADER_CURL_PIPELINE_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 2013, Linus Nielsen Feltzing, + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +struct connectdata * +Curl_bundle_find_best(struct SessionHandle *data, + struct connectbundle *cb_ptr); + +CURLcode Curl_add_handle_to_pipeline(struct SessionHandle *handle, + struct connectdata *conn); +void Curl_move_handle_from_send_to_recv_pipe(struct SessionHandle *handle, + struct connectdata *conn); +bool Curl_pipeline_penalized(struct SessionHandle *data, + struct connectdata *conn); + +bool Curl_pipeline_site_blacklisted(struct SessionHandle *handle, + struct connectdata *conn); + +CURLMcode Curl_pipeline_set_site_blacklist(char **sites, + struct curl_llist **list_ptr); + +bool Curl_pipeline_server_blacklisted(struct SessionHandle *handle, + char *server_name); + +CURLMcode Curl_pipeline_set_server_blacklist(char **servers, + struct curl_llist **list_ptr); + +void print_pipeline(struct connectdata *conn); + +#endif /* HEADER_CURL_PIPELINE_H */ diff --git a/lib/sendf.c b/lib/sendf.c index c64d686b9..d5bf17282 100644 --- a/lib/sendf.c +++ b/lib/sendf.c @@ -529,8 +529,7 @@ CURLcode Curl_read(struct connectdata *conn, /* connection data */ ssize_t nread = 0; size_t bytesfromsocket = 0; char *buffertofill = NULL; - bool pipelining = (conn->data->multi && - Curl_multi_canPipeline(conn->data->multi)) ? TRUE : FALSE; + bool pipelining = Curl_multi_pipeline_enabled(conn->data->multi); /* Set 'num' to 0 or 1, depending on which socket that has been sent here. If it is the second socket, we set num to 1. Otherwise to 0. This lets diff --git a/lib/strerror.c b/lib/strerror.c index 58c6dc310..a385f5572 100644 --- a/lib/strerror.c +++ b/lib/strerror.c @@ -292,6 +292,9 @@ curl_easy_strerror(CURLcode error) case CURLE_CHUNK_FAILED: return "Chunk callback failed"; + case CURLE_NO_CONNECTION_AVAILABLE: + return "The max connection limit is reached"; + /* error codes not used by current libcurl */ case CURLE_OBSOLETE16: case CURLE_OBSOLETE20: diff --git a/lib/transfer.c b/lib/transfer.c index 330b37a2b..db0318d5a 100644 --- a/lib/transfer.c +++ b/lib/transfer.c @@ -473,7 +473,7 @@ static CURLcode readwrite_data(struct SessionHandle *data, /* We've stopped dealing with input, get out of the do-while loop */ if(nread > 0) { - if(conn->data->multi && Curl_multi_canPipeline(conn->data->multi)) { + if(Curl_multi_pipeline_enabled(conn->data->multi)) { infof(data, "Rewinding stream by : %zd" " bytes on url %s (zero-length body)\n", @@ -602,8 +602,7 @@ static CURLcode readwrite_data(struct SessionHandle *data, if(dataleft != 0) { infof(conn->data, "Leftovers after chunking: %zu bytes\n", dataleft); - if(conn->data->multi && - Curl_multi_canPipeline(conn->data->multi)) { + if(Curl_multi_pipeline_enabled(conn->data->multi)) { /* only attempt the rewind if we truly are pipelining */ infof(conn->data, "Rewinding %zu bytes\n",dataleft); read_rewind(conn, dataleft); @@ -626,7 +625,7 @@ static CURLcode readwrite_data(struct SessionHandle *data, excess = (size_t)(k->bytecount + nread - k->maxdownload); if(excess > 0 && !k->ignorebody) { - if(conn->data->multi && Curl_multi_canPipeline(conn->data->multi)) { + if(Curl_multi_pipeline_enabled(conn->data->multi)) { /* The 'excess' amount below can't be more than BUFSIZE which always will fit in a size_t */ infof(data, diff --git a/lib/url.c b/lib/url.c index cbe0f4659..a14c0626b 100644 --- a/lib/url.c +++ b/lib/url.c @@ -123,6 +123,7 @@ int curl_win32_idn_to_ascii(const char *in, char **out); #include "bundles.h" #include "conncache.h" #include "multihandle.h" +#include "pipeline.h" #define _MPRINTF_REPLACE /* use our functions only */ #include @@ -134,6 +135,9 @@ int curl_win32_idn_to_ascii(const char *in, char **out); /* Local static prototypes */ static struct connectdata * find_oldest_idle_connection(struct SessionHandle *data); +static struct connectdata * +find_oldest_idle_connection_in_bundle(struct SessionHandle *data, + struct connectbundle *bundle); static void conn_free(struct connectdata *conn); static void signalPipeClose(struct curl_llist *pipeline, bool pipe_broke); static CURLcode do_init(struct connectdata *conn); @@ -2470,13 +2474,9 @@ static void conn_free(struct connectdata *conn) Curl_llist_destroy(conn->send_pipe, NULL); Curl_llist_destroy(conn->recv_pipe, NULL); - Curl_llist_destroy(conn->pend_pipe, NULL); - Curl_llist_destroy(conn->done_pipe, NULL); conn->send_pipe = NULL; conn->recv_pipe = NULL; - conn->pend_pipe = NULL; - conn->done_pipe = NULL; Curl_safefree(conn->localdev); Curl_free_ssl_config(&conn->ssl_config); @@ -2566,11 +2566,9 @@ CURLcode Curl_disconnect(struct connectdata *conn, bool dead_connection) Curl_ssl_close(conn, FIRSTSOCKET); /* Indicate to all handles on the pipe that we're dead */ - if(Curl_isPipeliningEnabled(data)) { + if(Curl_multi_pipeline_enabled(data->multi)) { signalPipeClose(conn->send_pipe, TRUE); signalPipeClose(conn->recv_pipe, TRUE); - signalPipeClose(conn->pend_pipe, TRUE); - signalPipeClose(conn->done_pipe, FALSE); } conn_free(conn); @@ -2602,7 +2600,7 @@ static bool IsPipeliningPossible(const struct SessionHandle *handle, const struct connectdata *conn) { if((conn->handler->protocol & CURLPROTO_HTTP) && - handle->multi && Curl_multi_canPipeline(handle->multi) && + Curl_multi_pipeline_enabled(handle->multi) && (handle->set.httpreq == HTTPREQ_GET || handle->set.httpreq == HTTPREQ_HEAD) && handle->set.httpversion != CURL_HTTP_VERSION_1_0) @@ -2613,10 +2611,7 @@ static bool IsPipeliningPossible(const struct SessionHandle *handle, bool Curl_isPipeliningEnabled(const struct SessionHandle *handle) { - if(handle->multi && Curl_multi_canPipeline(handle->multi)) - return TRUE; - - return FALSE; + return Curl_multi_pipeline_enabled(handle->multi); } CURLcode Curl_addHandleToPipeline(struct SessionHandle *data, @@ -2624,6 +2619,7 @@ CURLcode Curl_addHandleToPipeline(struct SessionHandle *data, { if(!Curl_llist_insert_next(pipeline, pipeline->tail, data)) return CURLE_OUT_OF_MEMORY; + infof(data, "Curl_addHandleToPipeline: length: %d\n", pipeline->size); return CURLE_OK; } @@ -2683,8 +2679,6 @@ void Curl_getoff_all_pipelines(struct SessionHandle *data, conn->readchannel_inuse = FALSE; if(Curl_removeHandleFromPipeline(data, conn->send_pipe) && send_head) conn->writechannel_inuse = FALSE; - Curl_removeHandleFromPipeline(data, conn->pend_pipe); - Curl_removeHandleFromPipeline(data, conn->done_pipe); } static void signalPipeClose(struct curl_llist *pipeline, bool pipe_broke) @@ -2715,8 +2709,8 @@ static void signalPipeClose(struct curl_llist *pipeline, bool pipe_broke) } /* - * This function kills and removes an existing connection in the connection - * cache. The connection that has been unused for the longest time. + * This function finds the connection in the connection + * cache that has been unused for the longest time. * * Returns the pointer to the oldest idle connection, or NULL if none was * found. @@ -2766,6 +2760,47 @@ find_oldest_idle_connection(struct SessionHandle *data) return conn_candidate; } +/* + * This function finds the connection in the connection + * bundle that has been unused for the longest time. + * + * Returns the pointer to the oldest idle connection, or NULL if none was + * found. + */ +static struct connectdata * +find_oldest_idle_connection_in_bundle(struct SessionHandle *data, + struct connectbundle *bundle) +{ + struct curl_llist_element *curr; + long highscore=-1; + long score; + struct timeval now; + struct connectdata *conn_candidate = NULL; + struct connectdata *conn; + + (void)data; + + now = Curl_tvnow(); + + curr = bundle->conn_list->head; + while(curr) { + conn = curr->ptr; + + if(!conn->inuse) { + /* Set higher score for the age passed since the connection was used */ + score = Curl_tvdiff(now, conn->now); + + if(score > highscore) { + highscore = score; + conn_candidate = conn; + } + } + curr = curr->next; + } + + return conn_candidate; +} + /* * Given one filled in connection struct (named needle), this function should * detect if there already is one that has all the significant details @@ -2774,11 +2809,15 @@ find_oldest_idle_connection(struct SessionHandle *data) * If there is a match, this function returns TRUE - and has marked the * connection as 'in-use'. It must later be called with ConnectionDone() to * return back to 'idle' (unused) state. + * + * The force_reuse flag is set if the connection must be used, even if + * the pipelining strategy wants to open a new connection instead of reusing. */ static bool ConnectionExists(struct SessionHandle *data, struct connectdata *needle, - struct connectdata **usethis) + struct connectdata **usethis, + bool *force_reuse) { struct connectdata *check; struct connectdata *chosen = 0; @@ -2787,15 +2826,30 @@ ConnectionExists(struct SessionHandle *data, (data->state.authhost.want==CURLAUTH_NTLM_WB) ? TRUE : FALSE; struct connectbundle *bundle; + *force_reuse = FALSE; + + /* We can't pipe if the site is blacklisted */ + if(canPipeline && Curl_pipeline_site_blacklisted(data, needle)) { + canPipeline = FALSE; + } + /* Look up the bundle with all the connections to this particular host */ bundle = Curl_conncache_find_bundle(data->state.conn_cache, needle->host.name); if(bundle) { + size_t max_pipe_len = Curl_multi_max_pipeline_length(data->multi); + size_t best_pipe_len = max_pipe_len; struct curl_llist_element *curr; infof(data, "Found bundle for host %s: %p\n", needle->host.name, bundle); + /* We can't pipe if we don't know anything about the server */ + if(canPipeline && !bundle->server_supports_pipelining) { + infof(data, "Server doesn't support pipelining\n"); + canPipeline = FALSE; + } + curr = bundle->conn_list->head; while(curr) { bool match = FALSE; @@ -2845,12 +2899,6 @@ ConnectionExists(struct SessionHandle *data, if(!IsPipeliningPossible(rh, check)) continue; } -#ifdef DEBUGBUILD - if(pipeLen > MAX_PIPELINE_LENGTH) { - infof(data, "BAD! Connection #%ld has too big pipeline!\n", - check->connection_id); - } -#endif } else { if(pipeLen > 0) { @@ -2989,26 +3037,60 @@ ConnectionExists(struct SessionHandle *data, } if(match) { - chosen = check; + /* If we are looking for an NTLM connection, check if this is already + authenticating with the right credentials. If not, keep looking so + that we can reuse NTLM connections if possible. (Especially we + must not reuse the same connection if partway through + a handshake!) */ + if(wantNTLM) { + if(credentialsMatch && check->ntlm.state != NTLMSTATE_NONE) { + chosen = check; - /* If we are not looking for an NTLM connection, we can choose this one - immediately. */ - if(!wantNTLM) - break; + /* We must use this connection, no other */ + *force_reuse = TRUE; + break; + } + else + continue; + } - /* Otherwise, check if this is already authenticating with the right - credentials. If not, keep looking so that we can reuse NTLM - connections if possible. (Especially we must reuse the same - connection if partway through a handshake!) */ - if(credentialsMatch && chosen->ntlm.state != NTLMSTATE_NONE) + if(canPipeline) { + /* We can pipeline if we want to. Let's continue looking for + the optimal connection to use, i.e the shortest pipe that is not + blacklisted. */ + + if(pipeLen == 0) { + /* We have the optimal connection. Let's stop looking. */ + chosen = check; + break; + } + + /* We can't use the connection if the pipe is full */ + if(pipeLen >= max_pipe_len) + continue; + + /* We can't use the connection if the pipe is penalized */ + if(Curl_pipeline_penalized(data, check)) + continue; + + if(pipeLen < best_pipe_len) { + /* This connection has a shorter pipe so far. We'll pick this + and continue searching */ + chosen = check; + best_pipe_len = pipeLen; + continue; + } + } + else { + /* We have found a connection. Let's stop searching. */ + chosen = check; break; + } } } } if(chosen) { - chosen->inuse = TRUE; /* mark this as being in use so that no other - handle in a multi stack may nick it */ *usethis = chosen; return TRUE; /* yes, we found one to use! */ } @@ -3475,7 +3557,7 @@ static struct connectdata *allocate_conn(struct SessionHandle *data) conn->response_header = NULL; #endif - if(data->multi && Curl_multi_canPipeline(data->multi) && + if(Curl_multi_pipeline_enabled(data->multi) && !conn->master_buffer) { /* Allocate master_buffer to be used for pipelining */ conn->master_buffer = calloc(BUFSIZE, sizeof (char)); @@ -3486,10 +3568,7 @@ static struct connectdata *allocate_conn(struct SessionHandle *data) /* Initialize the pipeline lists */ conn->send_pipe = Curl_llist_alloc((curl_llist_dtor) llist_dtor); conn->recv_pipe = Curl_llist_alloc((curl_llist_dtor) llist_dtor); - conn->pend_pipe = Curl_llist_alloc((curl_llist_dtor) llist_dtor); - conn->done_pipe = Curl_llist_alloc((curl_llist_dtor) llist_dtor); - if(!conn->send_pipe || !conn->recv_pipe || !conn->pend_pipe || - !conn->done_pipe) + if(!conn->send_pipe || !conn->recv_pipe) goto error; #if defined(HAVE_KRB4) || defined(HAVE_GSSAPI) @@ -3515,13 +3594,9 @@ static struct connectdata *allocate_conn(struct SessionHandle *data) Curl_llist_destroy(conn->send_pipe, NULL); Curl_llist_destroy(conn->recv_pipe, NULL); - Curl_llist_destroy(conn->pend_pipe, NULL); - Curl_llist_destroy(conn->done_pipe, NULL); conn->send_pipe = NULL; conn->recv_pipe = NULL; - conn->pend_pipe = NULL; - conn->done_pipe = NULL; Curl_safefree(conn->master_buffer); Curl_safefree(conn->localdev); @@ -4623,13 +4698,9 @@ static void reuse_conn(struct connectdata *old_conn, Curl_llist_destroy(old_conn->send_pipe, NULL); Curl_llist_destroy(old_conn->recv_pipe, NULL); - Curl_llist_destroy(old_conn->pend_pipe, NULL); - Curl_llist_destroy(old_conn->done_pipe, NULL); old_conn->send_pipe = NULL; old_conn->recv_pipe = NULL; - old_conn->pend_pipe = NULL; - old_conn->done_pipe = NULL; Curl_safefree(old_conn->master_buffer); } @@ -4663,6 +4734,10 @@ static CURLcode create_conn(struct SessionHandle *data, bool reuse; char *proxy = NULL; bool prot_missing = FALSE; + bool no_connections_available = FALSE; + bool force_reuse; + size_t max_host_connections = Curl_multi_max_host_connections(data->multi); + size_t max_total_connections = Curl_multi_max_total_connections(data->multi); *async = FALSE; @@ -4963,7 +5038,25 @@ static CURLcode create_conn(struct SessionHandle *data, if(data->set.reuse_fresh && !data->state.this_is_a_follow) reuse = FALSE; else - reuse = ConnectionExists(data, conn, &conn_temp); + reuse = ConnectionExists(data, conn, &conn_temp, &force_reuse); + + /* If we found a reusable connection, we may still want to + open a new connection if we are pipelining. */ + if(reuse && !force_reuse && IsPipeliningPossible(data, conn_temp)) { + size_t pipelen = conn_temp->send_pipe->size + conn_temp->recv_pipe->size; + if(pipelen > 0) { + infof(data, "Found connection %d, with requests in the pipe (%d)\n", + conn_temp->connection_id, pipelen); + + if(conn_temp->bundle->num_connections < max_host_connections && + data->state.conn_cache->num_connections < max_total_connections) { + /* We want a new connection anyway */ + reuse = FALSE; + + infof(data, "We can reuse, but we want a new connection anyway\n"); + } + } + } if(reuse) { /* @@ -4972,6 +5065,8 @@ static CURLcode create_conn(struct SessionHandle *data, * just allocated before we can move along and use the previously * existing one. */ + conn_temp->inuse = TRUE; /* mark this as being in use so that no other + handle in a multi stack may nick it */ reuse_conn(conn, conn_temp); free(conn); /* we don't need this anymore */ conn = conn_temp; @@ -4985,14 +5080,66 @@ static CURLcode create_conn(struct SessionHandle *data, conn->proxy.name?conn->proxy.dispname:conn->host.dispname); } else { - /* - * This is a brand new connection, so let's store it in the connection - * cache of ours! - */ - conn->inuse = TRUE; - ConnectionStore(data, conn); + /* We have decided that we want a new connection. However, we may not + be able to do that if we have reached the limit of how many + connections we are allowed to open. */ + struct connectbundle *bundle; + + bundle = Curl_conncache_find_bundle(data->state.conn_cache, + conn->host.name); + if(max_host_connections > 0 && bundle && + (bundle->num_connections >= max_host_connections)) { + struct connectdata *conn_candidate; + + /* The bundle is full. Let's see if we can kill a connection. */ + conn_candidate = find_oldest_idle_connection_in_bundle(data, bundle); + + if(conn_candidate) { + /* Set the connection's owner correctly, then kill it */ + conn_candidate->data = data; + (void)Curl_disconnect(conn_candidate, /* dead_connection */ FALSE); + } + else + no_connections_available = TRUE; + } + + if(max_total_connections > 0 && + (data->state.conn_cache->num_connections >= max_total_connections)) { + struct connectdata *conn_candidate; + + /* The cache is full. Let's see if we can kill a connection. */ + conn_candidate = find_oldest_idle_connection(data); + + if(conn_candidate) { + /* Set the connection's owner correctly, then kill it */ + conn_candidate->data = data; + (void)Curl_disconnect(conn_candidate, /* dead_connection */ FALSE); + } + else + no_connections_available = TRUE; + } + + + if(no_connections_available) { + infof(data, "No connections available.\n"); + + conn_free(conn); + *in_connect = NULL; + + return CURLE_NO_CONNECTION_AVAILABLE; + } + else { + /* + * This is a brand new connection, so let's store it in the connection + * cache of ours! + */ + ConnectionStore(data, conn); + } } + /* Mark the connection as used */ + conn->inuse = TRUE; + /* Setup and init stuff before DO starts, in preparing for the transfer. */ do_init(conn); @@ -5167,6 +5314,11 @@ CURLcode Curl_connect(struct SessionHandle *data, } } + if(code == CURLE_NO_CONNECTION_AVAILABLE) { + *in_connect = NULL; + return code; + } + if(code && *in_connect) { /* We're not allowed to return failure with memory left allocated in the connectdata struct, free those here */ @@ -5258,12 +5410,8 @@ CURLcode Curl_done(struct connectdata **connp, state it is for re-using, so we're forced to close it. In a perfect world we can add code that keep track of if we really must close it here or not, but currently we have no such detail knowledge. - - connection_id == -1 here means that the connection has not been added - to the connection cache (OOM) and thus we must disconnect it here. */ - if(data->set.reuse_forbid || conn->bits.close || premature || - (-1 == conn->connection_id)) { + if(data->set.reuse_forbid || conn->bits.close || premature) { CURLcode res2 = Curl_disconnect(conn, premature); /* close connection */ /* If we had an error already, make sure we return that one. But diff --git a/lib/urldata.h b/lib/urldata.h index 1cf7c38b0..b63d8eed6 100644 --- a/lib/urldata.h +++ b/lib/urldata.h @@ -935,17 +935,10 @@ struct connectdata { handle */ bool writechannel_inuse; /* whether the write channel is in use by an easy handle */ - bool server_supports_pipelining; /* TRUE if server supports pipelining, - set after first response */ struct curl_llist *send_pipe; /* List of handles waiting to send on this pipeline */ struct curl_llist *recv_pipe; /* List of handles waiting to read their responses on this pipeline */ - struct curl_llist *pend_pipe; /* List of pending handles on - this pipeline */ - struct curl_llist *done_pipe; /* Handles that are finished, but - still reference this connectdata */ -#define MAX_PIPELINE_LENGTH 5 char* master_buffer; /* The master buffer allocated on-demand; used for pipelining. */ size_t read_pos; /* Current read position in the master buffer */ @@ -1022,8 +1015,7 @@ struct connectdata { TUNNEL_CONNECT, /* CONNECT has been sent off */ TUNNEL_COMPLETE /* CONNECT response received completely */ } tunnel_state[2]; /* two separate ones to allow FTP */ - - struct connectbundle *bundle; /* The bundle we are member of */ + struct connectbundle *bundle; /* The bundle we are member of */ }; /* The end of connectdata. */ @@ -1172,13 +1164,6 @@ struct UrlState { /* buffers to store authentication data in, as parsed from input options */ struct timeval keeps_speed; /* for the progress meter really */ - struct connectdata *pending_conn; /* This points to the connection we want - to open when we are waiting in the - CONNECT_PEND state in the multi - interface. This to avoid recreating it - when we enter the CONNECT state again. - */ - struct connectdata *lastconnect; /* The last connection, NULL if undefined */ char *headerbuff; /* allocated buffer to store headers in */ diff --git a/tests/README b/tests/README index fff618ef2..3ddc1c93e 100644 --- a/tests/README +++ b/tests/README @@ -39,6 +39,7 @@ The cURL Test Suite 1.1 Requires to run perl (and a unix-style shell) + python (and a unix-style shell) diff (when a test fails, a diff is shown) stunnel (for HTTPS and FTPS tests) OpenSSH or SunSSH (for SCP, SFTP and SOCKS4/5 tests) diff --git a/tests/data/Makefile.am b/tests/data/Makefile.am index f65fe0688..7c2e648f5 100644 --- a/tests/data/Makefile.am +++ b/tests/data/Makefile.am @@ -95,13 +95,14 @@ test1400 test1401 test1402 test1403 test1404 test1405 test1406 test1407 \ test1408 test1409 test1410 test1411 test1412 test1413 \ test1500 test1501 test1502 test1503 test1504 test1505 test1506 test1507 \ test1508 \ +test1900 test1901 test1902 test1903 \ test2000 test2001 test2002 test2003 test2004 test2005 test2006 test2007 \ test2008 test2009 test2010 test2011 test2012 test2013 test2014 test2015 \ test2016 test2017 test2018 test2019 test2020 test2021 test2022 \ test2023 test2024 test2025 \ test2026 test2027 test2028 \ test2029 test2030 test2031 \ -test2032 +test2032 test2033 EXTRA_DIST = $(TESTCASES) DISABLED diff --git a/tests/data/test1900 b/tests/data/test1900 new file mode 100644 index 000000000..857c6096b --- /dev/null +++ b/tests/data/test1900 @@ -0,0 +1,57 @@ + + + +HTTP +pipelining +multi + + + +# Server-side + + +Adding handle 0 +Handle 0 Completed with status 0 +Adding handle 1 +Adding handle 2 +Adding handle 3 +Adding handle 4 +Adding handle 5 +Adding handle 6 +Handle 4 Completed with status 0 +Handle 5 Completed with status 0 +Handle 6 Completed with status 0 +Handle 1 Completed with status 0 +Handle 2 Completed with status 0 +Handle 3 Completed with status 0 + + + +# Client-side + + +http-pipe + + +lib1900 + + +HTTP GET using pipelining + + +http://%HOSTIP:%HTTPPIPEPORT/ + + +0 1k.txt +1000 100k.txt +0 1k.txt +0 1k.txt +0 1k.txt +0 1k.txt +0 1k.txt + + +# Verify data after the test has been "shot" + + + diff --git a/tests/data/test1901 b/tests/data/test1901 new file mode 100644 index 000000000..bacf9cb09 --- /dev/null +++ b/tests/data/test1901 @@ -0,0 +1,58 @@ + + + +HTTP +pipelining +multi + + + +# Server-side + + +Adding handle 0 +Handle 0 Completed with status 0 +Adding handle 1 +Adding handle 2 +Adding handle 3 +Adding handle 4 +Adding handle 5 +Adding handle 6 +Handle 2 Completed with status 0 +Handle 3 Completed with status 0 +Handle 4 Completed with status 0 +Handle 1 Completed with status 0 +Handle 5 Completed with status 0 +Handle 6 Completed with status 0 + + + +# Client-side + + +http-pipe + + +lib1900 + + +HTTP GET using pipelining, blacklisted site + + +http://%HOSTIP:%HTTPPIPEPORT/ + + +blacklist_site 127.0.0.1:%HTTPPIPEPORT +0 1k.txt +1000 100k.txt +0 1k.txt +0 1k.txt +0 1k.txt +0 1k.txt +0 1k.txt + + +# Verify data after the test has been "shot" + + + diff --git a/tests/data/test1902 b/tests/data/test1902 new file mode 100644 index 000000000..22f262176 --- /dev/null +++ b/tests/data/test1902 @@ -0,0 +1,57 @@ + + + +HTTP +pipelining +multi + + + +# Server-side + + +Adding handle 0 +Handle 0 Completed with status 0 +Adding handle 1 +Adding handle 2 +Adding handle 3 +Adding handle 4 +Adding handle 5 +Adding handle 6 +Handle 1 Completed with status 0 +Handle 4 Completed with status 0 +Handle 5 Completed with status 0 +Handle 6 Completed with status 0 +Handle 2 Completed with status 0 +Handle 3 Completed with status 0 + + + +# Client-side + + +http-pipe + + +lib1900 + + +HTTP GET using pipelining, broken pipe + + +http://%HOSTIP:%HTTPPIPEPORT/ + + +0 1k.txt +1000 connection_close.txt +0 1k.txt +0 1k.txt +0 1k.txt +0 1k.txt +0 1k.txt + + +# Verify data after the test has been "shot" + + + diff --git a/tests/data/test1903 b/tests/data/test1903 new file mode 100644 index 000000000..01efa67f8 --- /dev/null +++ b/tests/data/test1903 @@ -0,0 +1,57 @@ + + + +HTTP +pipelining +multi + + + +# Server-side + + +Adding handle 0 +Handle 0 Completed with status 0 +Adding handle 1 +Adding handle 2 +Adding handle 3 +Adding handle 4 +Adding handle 5 +Adding handle 6 +Handle 2 Completed with status 0 +Handle 3 Completed with status 0 +Handle 4 Completed with status 0 +Handle 5 Completed with status 0 +Handle 6 Completed with status 0 +Handle 1 Completed with status 0 + + + +# Client-side + + +http-pipe + + +lib1900 + + +HTTP GET using pipelining, penalized on content-length + + +http://%HOSTIP:%HTTPPIPEPORT/ + + +0 1k.txt +1000 100k.txt +550 alphabet.txt +10 alphabet.txt +10 alphabet.txt +10 alphabet.txt +10 alphabet.txt + + +# Verify data after the test has been "shot" + + + diff --git a/tests/data/test2033 b/tests/data/test2033 new file mode 100644 index 000000000..ad926ebce --- /dev/null +++ b/tests/data/test2033 @@ -0,0 +1,144 @@ + + + +HTTP +HTTP GET +HTTP Basic auth +HTTP NTLM auth +pipelining + + +# Server-side + + + + +HTTP/1.1 401 Need Basic or NTLM auth +Server: Microsoft-IIS/5.0 +Content-Type: text/html; charset=iso-8859-1 +Content-Length: 29 +WWW-Authenticate: NTLM +WWW-Authenticate: Basic realm="testrealm" + +This is a bad password page! + + + + +HTTP/1.1 401 Need Basic or NTLM auth (2) +Server: Microsoft-IIS/5.0 +Content-Type: text/html; charset=iso-8859-1 +Content-Length: 27 +WWW-Authenticate: NTLM +WWW-Authenticate: Basic realm="testrealm" + +This is not the real page! + + + +HTTP/1.1 401 NTLM intermediate (2) +Server: Microsoft-IIS/5.0 +Content-Type: text/html; charset=iso-8859-1 +Content-Length: 33 +WWW-Authenticate: NTLM TlRMTVNTUAACAAAACAAIADAAAAAGggEAq6U1NAWaJCIAAAAAAAAAAAAAAAA4AAAATlRMTUF1dGg= + +This is still not the real page! + + + +HTTP/1.1 200 Things are fine in server land +Server: Microsoft-IIS/5.0 +Content-Type: text/html; charset=iso-8859-1 +Content-Length: 32 + +Finally, this is the real page! + + + +HTTP/1.1 401 Need Basic or NTLM auth +Server: Microsoft-IIS/5.0 +Content-Type: text/html; charset=iso-8859-1 +Content-Length: 29 +WWW-Authenticate: NTLM +WWW-Authenticate: Basic realm="testrealm" + +This is a bad password page! +HTTP/1.1 401 Need Basic or NTLM auth +Server: Microsoft-IIS/5.0 +Content-Type: text/html; charset=iso-8859-1 +Content-Length: 29 +WWW-Authenticate: NTLM +WWW-Authenticate: Basic realm="testrealm" + +This is a bad password page! +HTTP/1.1 401 NTLM intermediate (2) +Server: Microsoft-IIS/5.0 +Content-Type: text/html; charset=iso-8859-1 +Content-Length: 33 +WWW-Authenticate: NTLM TlRMTVNTUAACAAAACAAIADAAAAAGggEAq6U1NAWaJCIAAAAAAAAAAAAAAAA4AAAATlRMTUF1dGg= + +HTTP/1.1 200 Things are fine in server land +Server: Microsoft-IIS/5.0 +Content-Type: text/html; charset=iso-8859-1 +Content-Length: 32 + +Finally, this is the real page! + + + + +# Client-side + + +http + + +lib2033 + + + +NTLM connection mapping, pipelining enabled + + +# we force our own host name, in order to make the test machine independent +CURL_GETHOSTNAME=curlhost +# we try to use the LD_PRELOAD hack, if not a debug build +LD_PRELOAD=%PWD/libtest/.libs/libhostname.so + + +http://%HOSTIP:%HTTPPORT/2032 + + +chkhostname curlhost + + + +# Verify data after the test has been "shot" + + +^User-Agent:.* + + +GET /20320100 HTTP/1.1 +Authorization: Basic dGVzdHVzZXI6dGVzdHBhc3M= +Host: 127.0.0.1:8990 +Accept: */* + +GET /20320100 HTTP/1.1 +Authorization: Basic dGVzdHVzZXI6dGVzdHBhc3M= +Host: 127.0.0.1:8990 +Accept: */* + +GET /20320200 HTTP/1.1 +Authorization: NTLM TlRMTVNTUAABAAAABoIIAAAAAAAAAAAAAAAAAAAAAAA= +Host: 127.0.0.1:8990 +Accept: */* + +GET /20320200 HTTP/1.1 +Authorization: NTLM TlRMTVNTUAADAAAAGAAYAEAAAAAYABgAWAAAAAAAAABwAAAACAAIAHAAAAAIAAgAeAAAAAAAAAAAAAAABoIBAI+/Fp9IERAQ74OsdNPbBpg7o8CVwLSO4DtFyIcZHUMKVktWIu92s2892OVpd2JzqnRlc3R1c2VyY3VybGhvc3Q= +Host: 127.0.0.1:8990 +Accept: */* + + + + diff --git a/tests/data/test530 b/tests/data/test530 index 359d04cf1..09e742179 100644 --- a/tests/data/test530 +++ b/tests/data/test530 @@ -2,7 +2,7 @@ HTTP -Pipelining +pipelining multi diff --git a/tests/data/test536 b/tests/data/test536 index 334c07f06..ef8263d98 100644 --- a/tests/data/test536 +++ b/tests/data/test536 @@ -1,4 +1,12 @@ + + +HTTP +pipelining +multi + + + HTTP/1.1 404 Badness diff --git a/tests/data/test584 b/tests/data/test584 index 81d6a083d..8d1ca92f9 100644 --- a/tests/data/test584 +++ b/tests/data/test584 @@ -2,7 +2,7 @@ HTTP -Pipelining +pipelining multi diff --git a/tests/http_pipe.py b/tests/http_pipe.py new file mode 100755 index 000000000..67185be8c --- /dev/null +++ b/tests/http_pipe.py @@ -0,0 +1,447 @@ +#!/usr/bin/python + +# Copyright 2012 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Modified by Linus Nielsen Feltzing for inclusion in the libcurl test +# framework +# +import SocketServer +import argparse +import re +import select +import socket +import time +import pprint +import os + +INFO_MESSAGE = ''' +This is a test server to test the libcurl pipelining functionality. +It is a modified version if Google's HTTP pipelining test server. More +information can be found here: + +http://dev.chromium.org/developers/design-documents/network-stack/http-pipelining + +Source code can be found here: + +http://code.google.com/p/http-pipelining-test/ +''' +MAX_REQUEST_SIZE = 1024 # bytes +MIN_POLL_TIME = 0.01 # seconds. Minimum time to poll, in order to prevent + # excessive looping because Python refuses to poll for + # small timeouts. +SEND_BUFFER_TIME = 0.5 # seconds +TIMEOUT = 30 # seconds + + +class Error(Exception): + pass + + +class RequestTooLargeError(Error): + pass + + +class ServeIndexError(Error): + pass + + +class UnexpectedMethodError(Error): + pass + + +class RequestParser(object): + """Parses an input buffer looking for HTTP GET requests.""" + + global logfile + + LOOKING_FOR_GET = 1 + READING_HEADERS = 2 + + HEADER_RE = re.compile('([^:]+):(.*)\n') + REQUEST_RE = re.compile('([^ ]+) ([^ ]+) HTTP/(\d+)\.(\d+)\n') + + def __init__(self): + """Initializer.""" + self._buffer = "" + self._pending_headers = {} + self._pending_request = "" + self._state = self.LOOKING_FOR_GET + self._were_all_requests_http_1_1 = True + self._valid_requests = [] + + def ParseAdditionalData(self, data): + """Finds HTTP requests in |data|. + + Args: + data: (String) Newly received input data from the socket. + + Returns: + (List of Tuples) + (String) The request path. + (Map of String to String) The header name and value. + + Raises: + RequestTooLargeError: If the request exceeds MAX_REQUEST_SIZE. + UnexpectedMethodError: On a non-GET method. + Error: On a programming error. + """ + logfile = open('log/server.input', 'a') + logfile.write(data) + logfile.close() + self._buffer += data.replace('\r', '') + should_continue_parsing = True + while should_continue_parsing: + if self._state == self.LOOKING_FOR_GET: + should_continue_parsing = self._DoLookForGet() + elif self._state == self.READING_HEADERS: + should_continue_parsing = self._DoReadHeader() + else: + raise Error('Unexpected state: ' + self._state) + if len(self._buffer) > MAX_REQUEST_SIZE: + raise RequestTooLargeError( + 'Request is at least %d bytes' % len(self._buffer)) + valid_requests = self._valid_requests + self._valid_requests = [] + return valid_requests + + @property + def were_all_requests_http_1_1(self): + return self._were_all_requests_http_1_1 + + def _DoLookForGet(self): + """Tries to parse an HTTTP request line. + + Returns: + (Boolean) True if a request was found. + + Raises: + UnexpectedMethodError: On a non-GET method. + """ + m = self.REQUEST_RE.match(self._buffer) + if not m: + return False + method, path, http_major, http_minor = m.groups() + + if method != 'GET': + raise UnexpectedMethodError('Unexpected method: ' + method) + if path in ['/', '/index.htm', '/index.html']: + raise ServeIndexError() + + if http_major != '1' or http_minor != '1': + self._were_all_requests_http_1_1 = False + +# print method, path + + self._pending_request = path + self._buffer = self._buffer[m.end():] + self._state = self.READING_HEADERS + return True + + def _DoReadHeader(self): + """Tries to parse a HTTP header. + + Returns: + (Boolean) True if it found the end of the request or a HTTP header. + """ + if self._buffer.startswith('\n'): + self._buffer = self._buffer[1:] + self._state = self.LOOKING_FOR_GET + self._valid_requests.append((self._pending_request, + self._pending_headers)) + self._pending_headers = {} + self._pending_request = "" + return True + + m = self.HEADER_RE.match(self._buffer) + if not m: + return False + + header = m.group(1).lower() + value = m.group(2).strip().lower() + if header not in self._pending_headers: + self._pending_headers[header] = value + self._buffer = self._buffer[m.end():] + return True + + +class ResponseBuilder(object): + """Builds HTTP responses for a list of accumulated requests.""" + + def __init__(self): + """Initializer.""" + self._max_pipeline_depth = 0 + self._requested_paths = [] + self._processed_end = False + self._were_all_requests_http_1_1 = True + + def QueueRequests(self, requested_paths, were_all_requests_http_1_1): + """Adds requests to the queue of requests. + + Args: + requested_paths: (List of Strings) Requested paths. + """ + self._requested_paths.extend(requested_paths) + self._were_all_requests_http_1_1 = were_all_requests_http_1_1 + + def Chunkify(self, data, chunksize): + """ Divides a string into chunks + """ + return [hex(chunksize)[2:] + "\r\n" + data[i:i+chunksize] + "\r\n" for i in range(0, len(data), chunksize)] + + def BuildResponses(self): + """Converts the queue of requests into responses. + + Returns: + (String) Buffer containing all of the responses. + """ + result = "" + self._max_pipeline_depth = max(self._max_pipeline_depth, + len(self._requested_paths)) + for path, headers in self._requested_paths: + if path == '/verifiedserver': + body = "WE ROOLZ: {}\r\n".format(os.getpid()); + result += self._BuildResponse( + '200 OK', ['Server: Apache', + 'Content-Length: {}'.format(len(body)), + 'Cache-Control: no-store'], body) + + elif path == '/alphabet.txt': + body = 'abcdefghijklmnopqrstuvwxyz' + result += self._BuildResponse( + '200 OK', ['Server: Apache', + 'Content-Length: 26', + 'Cache-Control: no-store'], body) + + elif path == '/reverse.txt': + body = 'zyxwvutsrqponmlkjihgfedcba' + result += self._BuildResponse( + '200 OK', ['Content-Length: 26', 'Cache-Control: no-store'], body) + + elif path == '/chunked.txt': + body = ('7\r\nchunked\r\n' + '8\r\nencoding\r\n' + '2\r\nis\r\n' + '3\r\nfun\r\n' + '0\r\n\r\n') + result += self._BuildResponse( + '200 OK', ['Transfer-Encoding: chunked', 'Cache-Control: no-store'], + body) + + elif path == '/cached.txt': + body = 'azbycxdwevfugthsirjqkplomn' + result += self._BuildResponse( + '200 OK', ['Content-Length: 26', 'Cache-Control: max-age=60'], body) + + elif path == '/connection_close.txt': + body = 'azbycxdwevfugthsirjqkplomn' + result += self._BuildResponse( + '200 OK', ['Content-Length: 26', 'Cache-Control: max-age=60', 'Connection: close'], body) + self._processed_end = True + + elif path == '/1k.txt': + str = '0123456789abcdef' + body = ''.join([str for num in xrange(64)]) + result += self._BuildResponse( + '200 OK', ['Server: Apache', + 'Content-Length: 1024', + 'Cache-Control: max-age=60'], body) + + elif path == '/10k.txt': + str = '0123456789abcdef' + body = ''.join([str for num in xrange(640)]) + result += self._BuildResponse( + '200 OK', ['Server: Apache', + 'Content-Length: 10240', + 'Cache-Control: max-age=60'], body) + + elif path == '/100k.txt': + str = '0123456789abcdef' + body = ''.join([str for num in xrange(6400)]) + result += self._BuildResponse( + '200 OK', + ['Server: Apache', + 'Content-Length: 102400', + 'Cache-Control: max-age=60'], + body) + + elif path == '/100k_chunked.txt': + str = '0123456789abcdef' + moo = ''.join([str for num in xrange(6400)]) + body = self.Chunkify(moo, 20480) + body.append('0\r\n\r\n') + body = ''.join(body) + + result += self._BuildResponse( + '200 OK', ['Transfer-Encoding: chunked', 'Cache-Control: no-store'], body) + + elif path == '/stats.txt': + results = { + 'max_pipeline_depth': self._max_pipeline_depth, + 'were_all_requests_http_1_1': int(self._were_all_requests_http_1_1), + } + body = ','.join(['%s:%s' % (k, v) for k, v in results.items()]) + result += self._BuildResponse( + '200 OK', + ['Content-Length: %s' % len(body), 'Cache-Control: no-store'], body) + self._processed_end = True + + else: + result += self._BuildResponse('404 Not Found', ['Content-Length: 7'], 'Go away') + if self._processed_end: + break + self._requested_paths = [] + return result + + def WriteError(self, status, error): + """Returns an HTTP response for the specified error. + + Args: + status: (String) Response code and descrtion (e.g. "404 Not Found") + + Returns: + (String) Text of HTTP response. + """ + return self._BuildResponse( + status, ['Connection: close', 'Content-Type: text/plain'], error) + + @property + def processed_end(self): + return self._processed_end + + def _BuildResponse(self, status, headers, body): + """Builds an HTTP response. + + Args: + status: (String) Response code and descrtion (e.g. "200 OK") + headers: (List of Strings) Headers (e.g. "Connection: close") + body: (String) Response body. + + Returns: + (String) Text of HTTP response. + """ + return ('HTTP/1.1 %s\r\n' + '%s\r\n' + '\r\n' + '%s' % (status, '\r\n'.join(headers), body)) + + +class PipelineRequestHandler(SocketServer.BaseRequestHandler): + """Called on an incoming TCP connection.""" + + def _GetTimeUntilTimeout(self): + return self._start_time + TIMEOUT - time.time() + + def _GetTimeUntilNextSend(self): + if not self._last_queued_time: + return TIMEOUT + return self._last_queued_time + SEND_BUFFER_TIME - time.time() + + def handle(self): + self._request_parser = RequestParser() + self._response_builder = ResponseBuilder() + self._last_queued_time = 0 + self._num_queued = 0 + self._num_written = 0 + self._send_buffer = "" + self._start_time = time.time() + try: + poller = select.epoll(sizehint=1) + poller.register(self.request.fileno(), select.EPOLLIN) + while not self._response_builder.processed_end or self._send_buffer: + + time_left = self._GetTimeUntilTimeout() + time_until_next_send = self._GetTimeUntilNextSend() + max_poll_time = min(time_left, time_until_next_send) + MIN_POLL_TIME + + events = None + if max_poll_time > 0: + if self._send_buffer: + poller.modify(self.request.fileno(), + select.EPOLLIN | select.EPOLLOUT) + else: + poller.modify(self.request.fileno(), select.EPOLLIN) + events = poller.poll(timeout=max_poll_time) + + if self._GetTimeUntilTimeout() <= 0: + return + + if self._GetTimeUntilNextSend() <= 0: + self._send_buffer += self._response_builder.BuildResponses() + self._num_written = self._num_queued + self._last_queued_time = 0 + + for fd, mode in events: + if mode & select.EPOLLIN: + new_data = self.request.recv(MAX_REQUEST_SIZE, socket.MSG_DONTWAIT) + if not new_data: + return + new_requests = self._request_parser.ParseAdditionalData(new_data) + self._response_builder.QueueRequests( + new_requests, self._request_parser.were_all_requests_http_1_1) + self._num_queued += len(new_requests) + self._last_queued_time = time.time() + elif mode & select.EPOLLOUT: + num_bytes_sent = self.request.send(self._send_buffer[0:4096]) + self._send_buffer = self._send_buffer[num_bytes_sent:] + time.sleep(0.05) + else: + return + + except RequestTooLargeError as e: + self.request.send(self._response_builder.WriteError( + '413 Request Entity Too Large', e)) + raise + except UnexpectedMethodError as e: + self.request.send(self._response_builder.WriteError( + '405 Method Not Allowed', e)) + raise + except ServeIndexError: + self.request.send(self._response_builder.WriteError( + '200 OK', INFO_MESSAGE)) + except Exception as e: + print e + self.request.close() + + +class PipelineServer(SocketServer.ForkingMixIn, SocketServer.TCPServer): + pass + + +parser = argparse.ArgumentParser() +parser.add_argument("--port", action="store", default=0, + type=int, help="port to listen on") +parser.add_argument("--verbose", action="store", default=0, + type=int, help="verbose output") +parser.add_argument("--pidfile", action="store", default=0, + help="file name for the PID") +parser.add_argument("--logfile", action="store", default=0, + help="file name for the log") +parser.add_argument("--srcdir", action="store", default=0, + help="test directory") +parser.add_argument("--id", action="store", default=0, + help="server ID") +parser.add_argument("--ipv4", action="store_true", default=0, + help="IPv4 flag") +args = parser.parse_args() + +if args.pidfile: + pid = os.getpid() + f = open(args.pidfile, 'w') + f.write('{}'.format(pid)) + f.close() + +server = PipelineServer(('0.0.0.0', args.port), PipelineRequestHandler) +server.allow_reuse_address = True +server.serve_forever() diff --git a/tests/libtest/.gitignore b/tests/libtest/.gitignore index 28e83c08d..7fd6e7e30 100644 --- a/tests/libtest/.gitignore +++ b/tests/libtest/.gitignore @@ -1,5 +1,7 @@ chkhostname lib5[0-9][0-9] lib150[0-9] +lib19[0-9][0-9] +lib2033 libauthretry libntlmconnect diff --git a/tests/libtest/Makefile.inc b/tests/libtest/Makefile.inc index 88fc7d8fa..391c6255e 100644 --- a/tests/libtest/Makefile.inc +++ b/tests/libtest/Makefile.inc @@ -23,7 +23,9 @@ noinst_PROGRAMS = chkhostname libauthretry libntlmconnect \ lib582 lib583 lib585 lib586 lib587 \ lib590 lib591 lib597 lib598 lib599 \ \ - lib1500 lib1501 lib1502 lib1503 lib1504 lib1505 lib1506 lib1507 lib1508 + lib1500 lib1501 lib1502 lib1503 lib1504 lib1505 lib1506 lib1507 lib1508 \ + lib1900 \ + lib2033 chkhostname_SOURCES = chkhostname.c ../../lib/curl_gethostname.c chkhostname_LDADD = @CURL_NETWORK_LIBS@ @@ -323,3 +325,9 @@ lib1507_CPPFLAGS = $(AM_CPPFLAGS) -DLIB1507 lib1508_SOURCES = lib1508.c $(SUPPORTFILES) $(TESTUTIL) $(WARNLESS) lib1508_LDADD = $(TESTUTIL_LIBS) lib1508_CPPFLAGS = $(AM_CPPFLAGS) -DLIB1508 + +lib1900_SOURCES = lib1900.c $(SUPPORTFILES) $(TESTUTIL) $(WARNLESS) +lib1900_CPPFLAGS = $(AM_CPPFLAGS) + +lib2033_SOURCES = libntlmconnect.c $(SUPPORTFILES) $(TESTUTIL) $(WARNLESS) +lib2033_CPPFLAGS = $(AM_CPPFLAGS) -DUSE_PIPELINING diff --git a/tests/libtest/lib1900.c b/tests/libtest/lib1900.c new file mode 100644 index 000000000..b2a943440 --- /dev/null +++ b/tests/libtest/lib1900.c @@ -0,0 +1,256 @@ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 2013, Linus Nielsen Feltzing, + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ +#include "test.h" + +#include "testutil.h" +#include "warnless.h" +#include "memdebug.h" + +#define TEST_HANG_TIMEOUT 60 * 1000 +#define MAX_URLS 200 +#define MAX_BLACKLIST 20 + +int urltime[MAX_URLS]; +char *urlstring[MAX_URLS]; +CURL *handles[MAX_URLS]; +char *site_blacklist[MAX_BLACKLIST]; +char *server_blacklist[MAX_BLACKLIST]; +int num_handles; +int blacklist_num_servers; +int blacklist_num_sites; + +int parse_url_file(const char *filename); +void free_urls(void); +int create_handles(void); +void setup_handle(char *base_url, CURLM *m, int handlenum); +void remove_handles(void); + +static size_t +write_callback(void *contents, size_t size, size_t nmemb, void *userp) +{ + size_t realsize = size * nmemb; + (void)contents; + (void)userp; + + return realsize; +} + +int parse_url_file(const char *filename) +{ + FILE *f; + int time; + char buf[200]; + + num_handles = 0; + blacklist_num_sites = 0; + blacklist_num_servers = 0; + + f = fopen(filename, "rb"); + if(!f) + return 0; + + while(!feof(f)) { + if(fscanf(f, "%d %s\n", &time, buf)) { + urltime[num_handles] = time; + urlstring[num_handles] = strdup(buf); + num_handles++; + continue; + } + + if(fscanf(f, "blacklist_site %s\n", buf)) { + site_blacklist[blacklist_num_sites] = strdup(buf); + blacklist_num_sites++; + continue; + } + + break; + } + fclose(f); + + site_blacklist[blacklist_num_sites] = NULL; + server_blacklist[blacklist_num_servers] = NULL; + return num_handles; +} + +void free_urls(void) +{ + int i; + for(i = 0;i < num_handles;i++) { + free(urlstring[i]); + } + for(i = 0;i < blacklist_num_servers;i++) { + free(server_blacklist[i]); + } + for(i = 0;i < blacklist_num_sites;i++) { + free(site_blacklist[i]); + } +} + +int create_handles(void) +{ + int i; + + for(i = 0;i < num_handles;i++) { + handles[i] = curl_easy_init(); + } + return 0; +} + +void setup_handle(char *base_url, CURLM *m, int handlenum) +{ + char urlbuf[256]; + + sprintf(urlbuf, "%s%s", base_url, urlstring[handlenum]); + curl_easy_setopt(handles[handlenum], CURLOPT_URL, urlbuf); + curl_easy_setopt(handles[handlenum], CURLOPT_VERBOSE, 1L); + curl_easy_setopt(handles[handlenum], CURLOPT_FAILONERROR, 1L); + curl_easy_setopt(handles[handlenum], CURLOPT_WRITEFUNCTION, write_callback); + curl_easy_setopt(handles[handlenum], CURLOPT_WRITEDATA, NULL); + curl_multi_add_handle(m, handles[handlenum]); +} + +void remove_handles(void) +{ + int i; + + for(i = 0;i < num_handles;i++) { + if(handles[i]) + curl_easy_cleanup(handles[i]); + } +} + +int test(char *URL) +{ + int res = 0; + CURLM *m = NULL; + CURLMsg *msg; /* for picking up messages with the transfer status */ + int msgs_left; /* how many messages are left */ + int running; + int handlenum = 0; + struct timeval last_handle_add; + + if(parse_url_file("log/urls.txt") <= 0) + goto test_cleanup; + + start_test_timing(); + + curl_global_init(CURL_GLOBAL_ALL); + + m = curl_multi_init(); + + create_handles(); + + multi_setopt(m, CURLMOPT_PIPELINING, 1L); + multi_setopt(m, CURLMOPT_MAX_HOST_CONNECTIONS, 2L); + multi_setopt(m, CURLMOPT_MAX_PIPELINE_LENGTH, 3L); + multi_setopt(m, CURLMOPT_CONTENT_LENGTH_PENALTY_SIZE, 15000L); + multi_setopt(m, CURLMOPT_CHUNK_LENGTH_PENALTY_SIZE, 10000L); + + multi_setopt(m, CURLMOPT_PIPELINING_SITE_BL, site_blacklist); + multi_setopt(m, CURLMOPT_PIPELINING_SERVER_BL, server_blacklist); + + gettimeofday(&last_handle_add, NULL); + + for(;;) { + struct timeval interval; + struct timeval now; + long int msnow, mslast; + fd_set rd, wr, exc; + int maxfd = -99; + long timeout; + + interval.tv_sec = 1; + interval.tv_usec = 0; + + if(handlenum < num_handles) { + gettimeofday(&now, NULL); + msnow = now.tv_sec * 1000 + now.tv_usec / 1000; + mslast = last_handle_add.tv_sec * 1000 + last_handle_add.tv_usec / 1000; + if(msnow - mslast >= urltime[handlenum] && handlenum < num_handles) { + fprintf(stdout, "Adding handle %d\n", handlenum); + setup_handle(URL, m, handlenum); + last_handle_add = now; + handlenum++; + } + } + + curl_multi_perform(m, &running); + + abort_on_test_timeout(); + + /* See how the transfers went */ + while ((msg = curl_multi_info_read(m, &msgs_left))) { + if (msg->msg == CURLMSG_DONE) { + int i, found = 0; + + /* Find out which handle this message is about */ + for (i = 0; i < num_handles; i++) { + found = (msg->easy_handle == handles[i]); + if(found) + break; + } + + printf("Handle %d Completed with status %d\n", i, msg->data.result); + curl_multi_remove_handle(m, handles[i]); + } + } + + if(handlenum == num_handles && !running) { + break; /* done */ + } + + FD_ZERO(&rd); + FD_ZERO(&wr); + FD_ZERO(&exc); + + curl_multi_fdset(m, &rd, &wr, &exc, &maxfd); + + /* At this point, maxfd is guaranteed to be greater or equal than -1. */ + + curl_multi_timeout(m, &timeout); + + if(timeout < 0) + timeout = 1; + + interval.tv_sec = timeout / 1000; + interval.tv_usec = (timeout % 1000) * 1000; + + interval.tv_sec = 0; + interval.tv_usec = 1000; + + select_test(maxfd+1, &rd, &wr, &exc, &interval); + + abort_on_test_timeout(); + } + +test_cleanup: + + remove_handles(); + + /* undocumented cleanup sequence - type UB */ + + curl_multi_cleanup(m); + curl_global_cleanup(); + + free_urls(); + return res; +} diff --git a/tests/libtest/lib530.c b/tests/libtest/lib530.c index ad84ff8a5..06a846439 100644 --- a/tests/libtest/lib530.c +++ b/tests/libtest/lib530.c @@ -37,6 +37,7 @@ int test(char *URL) CURLM *m = NULL; int i; char target_url[256]; + int handles_added = 0; for(i=0; i < NUM_HANDLES; i++) curl[i] = NULL; @@ -59,10 +60,13 @@ int test(char *URL) easy_setopt(curl[i], CURLOPT_VERBOSE, 1L); /* include headers */ easy_setopt(curl[i], CURLOPT_HEADER, 1L); - /* add handle to multi */ - multi_add_handle(m, curl[i]); } + /* Add the first handle to multi. We do this to let libcurl detect + that the server can do pipelining. The rest of the handles will be + added later. */ + multi_add_handle(m, curl[handles_added++]); + multi_setopt(m, CURLMOPT_PIPELINING, 1L); fprintf(stderr, "Start at URL 0\n"); @@ -79,9 +83,14 @@ int test(char *URL) abort_on_test_timeout(); - if(!running) + if(!running && handles_added >= NUM_HANDLES) break; /* done */ + /* Add the rest of the handles now that the first handle has sent the + request. */ + while(handles_added < NUM_HANDLES) + multi_add_handle(m, curl[handles_added++]); + FD_ZERO(&rd); FD_ZERO(&wr); FD_ZERO(&exc); diff --git a/tests/libtest/libntlmconnect.c b/tests/libtest/libntlmconnect.c index b540ebf58..cd507dfa1 100644 --- a/tests/libtest/libntlmconnect.c +++ b/tests/libtest/libntlmconnect.c @@ -125,6 +125,12 @@ int test(char *url) multi_init(multi); +#ifdef USE_PIPELINING + multi_setopt(multi, CURLMOPT_PIPELINING, 1); + multi_setopt(multi, CURLMOPT_MAX_HOST_CONNECTIONS, 5); + multi_setopt(multi, CURLMOPT_MAX_TOTAL_CONNECTIONS, 10); +#endif + for(;;) { struct timeval interval; fd_set fdread; diff --git a/tests/runtests.pl b/tests/runtests.pl index 4915f2e81..cc6999cdc 100755 --- a/tests/runtests.pl +++ b/tests/runtests.pl @@ -140,6 +140,7 @@ my $GOPHER6PORT; # Gopher IPv6 server port my $HTTPTLSPORT; # HTTP TLS (non-stunnel) server port my $HTTPTLS6PORT; # HTTP TLS (non-stunnel) IPv6 server port my $HTTPPROXYPORT; # HTTP proxy port, when using CONNECT +my $HTTPPIPEPORT; # HTTP pipelining port my $srcdir = $ENV{'srcdir'} || '.'; my $CURL="../src/curl".exe_ext(); # what curl executable to run on the tests @@ -339,10 +340,10 @@ delete $ENV{'CURL_CA_BUNDLE'} if($ENV{'CURL_CA_BUNDLE'}); # Load serverpidfile hash with pidfile names for all possible servers. # sub init_serverpidfile_hash { - for my $proto (('ftp', 'http', 'imap', 'pop3', 'smtp')) { + for my $proto (('ftp', 'http', 'imap', 'pop3', 'smtp', 'http')) { for my $ssl (('', 's')) { for my $ipvnum ((4, 6)) { - for my $idnum ((1, 2)) { + for my $idnum ((1, 2, 3)) { my $serv = servername_id("$proto$ssl", $ipvnum, $idnum); my $pidf = server_pidfilename("$proto$ssl", $ipvnum, $idnum); $serverpidfile{$serv} = $pidf; @@ -642,11 +643,11 @@ sub stopserver { # All servers relative to the given one must be stopped also # my @killservers; - if($server =~ /^(ftp|http|imap|pop3|smtp)s((\d*)(-ipv6|))$/) { + if($server =~ /^(ftp|http|imap|pop3|smtp|httppipe)s((\d*)(-ipv6|))$/) { # given a stunnel based ssl server, also kill non-ssl underlying one push @killservers, "${1}${2}"; } - elsif($server =~ /^(ftp|http|imap|pop3|smtp)((\d*)(-ipv6|))$/) { + elsif($server =~ /^(ftp|http|imap|pop3|smtp|httppipe)((\d*)(-ipv6|))$/) { # given a non-ssl server, also kill stunnel based ssl piggybacking one push @killservers, "${1}s${2}"; } @@ -1105,6 +1106,7 @@ my %protofunc = ('http' => \&verifyhttp, 'pop3' => \&verifyftp, 'imap' => \&verifyftp, 'smtp' => \&verifyftp, + 'httppipe' => \&verifyhttp, 'ftps' => \&verifyftp, 'tftp' => \&verifyftp, 'ssh' => \&verifyssh, @@ -1170,6 +1172,7 @@ sub runhttpserver { my $pidfile; my $logfile; my $flags = ""; + my $exe = "$perl $srcdir/httpserver.pl"; if($alt eq "ipv6") { # if IPv6, use a different setup @@ -1180,6 +1183,11 @@ sub runhttpserver { # basically the same, but another ID $idnum = 2; } + elsif($alt eq "pipe") { + # basically the same, but another ID + $idnum = 3; + $exe = "python $srcdir/http_pipe.py"; + } $server = servername_id($proto, $ipvnum, $idnum); @@ -1207,7 +1215,82 @@ sub runhttpserver { $flags .= "--id $idnum " if($idnum > 1); $flags .= "--ipv$ipvnum --port $port --srcdir \"$srcdir\""; - my $cmd = "$perl $srcdir/httpserver.pl $flags"; + my $cmd = "$exe $flags"; + my ($httppid, $pid2) = startnew($cmd, $pidfile, 15, 0); + + if($httppid <= 0 || !kill(0, $httppid)) { + # it is NOT alive + logmsg "RUN: failed to start the $srvrname server\n"; + stopserver($server, "$pid2"); + displaylogs($testnumcheck); + $doesntrun{$pidfile} = 1; + return (0,0); + } + + # Server is up. Verify that we can speak to it. + my $pid3 = verifyserver($proto, $ipvnum, $idnum, $ip, $port); + if(!$pid3) { + logmsg "RUN: $srvrname server failed verification\n"; + # failed to talk to it properly. Kill the server and return failure + stopserver($server, "$httppid $pid2"); + displaylogs($testnumcheck); + $doesntrun{$pidfile} = 1; + return (0,0); + } + $pid2 = $pid3; + + if($verbose) { + logmsg "RUN: $srvrname server is now running PID $httppid\n"; + } + + sleep(1); + + return ($httppid, $pid2); +} + +####################################################################### +# start the http server +# +sub runhttp_pipeserver { + my ($proto, $verbose, $alt, $port) = @_; + my $ip = $HOSTIP; + my $ipvnum = 4; + my $idnum = 1; + my $server; + my $srvrname; + my $pidfile; + my $logfile; + my $flags = ""; + + if($alt eq "ipv6") { + # No IPv6 + } + + $server = servername_id($proto, $ipvnum, $idnum); + + $pidfile = $serverpidfile{$server}; + + # don't retry if the server doesn't work + if ($doesntrun{$pidfile}) { + return (0,0); + } + + my $pid = processexists($pidfile); + if($pid > 0) { + stopserver($server, "$pid"); + } + unlink($pidfile) if(-f $pidfile); + + $srvrname = servername_str($proto, $ipvnum, $idnum); + + $logfile = server_logfilename($LOGDIR, $proto, $ipvnum, $idnum); + + $flags .= "--verbose " if($debugprotocol); + $flags .= "--pidfile \"$pidfile\" --logfile \"$logfile\" "; + $flags .= "--id $idnum " if($idnum > 1); + $flags .= "--port $port --srcdir \"$srcdir\""; + + my $cmd = "$srcdir/http_pipe.py $flags"; my ($httppid, $pid2) = startnew($cmd, $pidfile, 15, 0); if($httppid <= 0 || !kill(0, $httppid)) { @@ -2276,6 +2359,9 @@ sub checksystem { # 'http-proxy' is used in test cases to do CONNECT through push @protocols, 'http-proxy'; + # 'http-pipe' is the special server for testing pipelining + push @protocols, 'http-pipe'; + # 'none' is used in test cases to mean no server push @protocols, 'none'; } @@ -2477,6 +2563,7 @@ sub checksystem { } logmsg "\n"; } + logmsg sprintf("* HTTP-PIPE/%d \n", $HTTPPIPEPORT); $has_textaware = ($^O eq 'MSWin32') || ($^O eq 'msys'); @@ -2505,6 +2592,7 @@ sub subVariables { $$thing =~ s/%HTTP6PORT/$HTTP6PORT/g; $$thing =~ s/%HTTPSPORT/$HTTPSPORT/g; $$thing =~ s/%HTTPPORT/$HTTPPORT/g; + $$thing =~ s/%HTTPPIPEPORT/$HTTPPIPEPORT/g; $$thing =~ s/%PROXYPORT/$HTTPPROXYPORT/g; $$thing =~ s/%IMAP6PORT/$IMAP6PORT/g; @@ -3870,6 +3958,23 @@ sub startservers { $run{'http-ipv6'}="$pid $pid2"; } } + elsif($what eq "http-pipe") { + if($torture && $run{'http-pipe'} && + !responsive_http_server("http", $verbose, "pipe", + $HTTPPIPEPORT)) { + stopserver('http-pipe'); + } + if(!$run{'http-pipe'}) { + ($pid, $pid2) = runhttpserver("http", $verbose, "pipe", + $HTTPPIPEPORT); + if($pid <= 0) { + return "failed starting HTTP-pipe server"; + } + logmsg sprintf ("* pid http-pipe => %d %d\n", $pid, $pid2) + if($verbose); + $run{'http-pipe'}="$pid $pid2"; + } + } elsif($what eq "rtsp") { if($torture && $run{'rtsp'} && !responsive_rtsp_server($verbose)) { @@ -4512,6 +4617,7 @@ $GOPHER6PORT = $base++; # Gopher IPv6 server port $HTTPTLSPORT = $base++; # HTTP TLS (non-stunnel) server port $HTTPTLS6PORT = $base++; # HTTP TLS (non-stunnel) IPv6 server port $HTTPPROXYPORT = $base++; # HTTP proxy port, when using CONNECT +$HTTPPIPEPORT = $base++; # HTTP pipelining port ####################################################################### # clear and create logging directory: diff --git a/tests/serverhelp.pm b/tests/serverhelp.pm index a1d1dc367..b0b5b7492 100644 --- a/tests/serverhelp.pm +++ b/tests/serverhelp.pm @@ -79,7 +79,7 @@ sub serverfactors { my $idnum; if($server =~ - /^((ftp|http|imap|pop3|smtp)s?)(\d*)(-ipv6|)$/) { + /^((ftp|http|imap|pop3|smtp|http-pipe)s?)(\d*)(-ipv6|)$/) { $proto = $1; $idnum = ($3 && ($3 > 1)) ? $3 : 1; $ipvnum = ($4 && ($4 =~ /6$/)) ? 6 : 4; @@ -105,7 +105,7 @@ sub servername_str { $proto = uc($proto) if($proto); die "unsupported protocol: '$proto'" unless($proto && - ($proto =~ /^(((FTP|HTTP|IMAP|POP3|SMTP)S?)|(TFTP|SFTP|SOCKS|SSH|RTSP|GOPHER|HTTPTLS))$/)); + ($proto =~ /^(((FTP|HTTP|IMAP|POP3|SMTP|HTTP-PIPE)S?)|(TFTP|SFTP|SOCKS|SSH|RTSP|GOPHER|HTTPTLS))$/)); $ipver = (not $ipver) ? 'ipv4' : lc($ipver); die "unsupported IP version: '$ipver'" unless($ipver &&