From 6cf8413e31629183b3c749aa2a17d24be14cbca1 Mon Sep 17 00:00:00 2001 From: Daniel Stenberg Date: Sat, 10 Aug 2013 22:55:59 +0200 Subject: [PATCH] curl_easy_perform_ev: debug/test function This function is meant to work *exactly* as curl_easy_perform() but will use the event-based libcurl API internally instead of curl_multi_perform(). To avoid relying on an actual event-based library and to not use non-portable functions (like epoll or similar), there's a rather inefficient emulation layer implemented on top of Curl_poll() instead. There's currently some convenience logging done in curl_easy_perform_ev which helps when tracking down problems. They may be suitable to remove or change once things seem to be fine enough. curl has a new --test-event option when built with debug enabled that then uses curl_easy_perform_ev() instead of curl_easy_perform(). If built without debug, using --test-event will only output a warning message. NOTE: curl_easy_perform_ev() is not part if the public API on purpose. It is only present in debug builds of libcurl and MUST NOT be considered stable even then. Use it for libcurl-testing purposes only. runtests.pl now features an -e command line option that makes it use --test-event for all curl command line tests. The man page is updated. --- lib/easy.c | 414 +++++++++++++++++++++++++++++++++++++------- lib/easyif.h | 3 + src/tool_cfgable.h | 5 +- src/tool_getparam.c | 8 + src/tool_operate.c | 10 ++ tests/runtests.1 | 6 +- tests/runtests.pl | 34 +++- 7 files changed, 414 insertions(+), 66 deletions(-) diff --git a/lib/easy.c b/lib/easy.c index 07a42bb5c..2ac92a2f6 100644 --- a/lib/easy.c +++ b/lib/easy.c @@ -453,70 +453,275 @@ CURLcode curl_easy_setopt(CURL *curl, CURLoption tag, ...) return ret; } -/* - * curl_easy_perform() is the external interface that performs a blocking - * transfer as previously setup. +#ifdef CURLDEBUG + +struct monitor { + struct monitor *next; /* the next node in the list or NULL */ + struct pollfd socket; /* socket info of what to monitor */ +}; + +struct events { + long ms; /* timeout, run the timeout function when reached */ + bool msbump; /* set TRUE when timeout is set by callback */ + int num_sockets; /* number of nodes in the monitor list */ + struct monitor *list; /* list of sockets to monitor */ + int running_handles; /* store the returned number */ +}; + +/* events_timer * - * CONCEPT: This function creates a multi handle, adds the easy handle to it, - * runs curl_multi_perform() until the transfer is done, then detaches the - * easy handle, destroys the multi handle and returns the easy handle's return - * code. - * - * REALITY: it can't just create and destroy the multi handle that easily. It - * needs to keep it around since if this easy handle is used again by this - * function, the same multi handle must be re-used so that the same pools and - * caches can be used. + * Callback that gets called with a new value when the timeout should be + * updated. */ -CURLcode curl_easy_perform(CURL *easy) + +static int events_timer(CURLM *multi, /* multi handle */ + long timeout_ms, /* see above */ + void *userp) /* private callback pointer */ +{ + struct events *ev = userp; + (void)multi; + if(timeout_ms == -1) + /* timeout removed */ + timeout_ms = 0; + else if(timeout_ms == 0) + /* timeout is already reached! */ + timeout_ms = 1; /* trigger asap */ + + ev->ms = timeout_ms; + ev->msbump = TRUE; + /* fprintf(stderr, "%s: timeout %ld\n", __func__, timeout_ms); */ + return 0; +} + + +/* poll2cselect + * + * convert from poll() bit definitions to libcurl's CURL_CSELECT_* ones + */ +static int poll2cselect(int pollmask) +{ + int omask=0; + if(pollmask & POLLIN) + omask |= CURL_CSELECT_IN; + if(pollmask & POLLOUT) + omask |= CURL_CSELECT_OUT; + if(pollmask & POLLERR) + omask |= CURL_CSELECT_ERR; + return omask; +} + + +/* socketcb2poll + * + * convert from libcurl' CURL_POLL_* bit definitions to poll()'s + */ +static short socketcb2poll(int pollmask) +{ + short omask=0; + if(pollmask & CURL_POLL_IN) + omask |= POLLIN; + if(pollmask & CURL_POLL_OUT) + omask |= POLLOUT; + return omask; +} + +/* events_socket + * + * Callback that gets called with information about socket activity to + * monitor. + */ +static int events_socket(CURL *easy, /* easy handle */ + curl_socket_t s, /* socket */ + int what, /* see above */ + void *userp, /* private callback + pointer */ + void *socketp) /* private socket + pointer */ +{ + struct events *ev = userp; + struct monitor *m; + struct monitor *prev=NULL; + (void)socketp; + + m = ev->list; + while(m) { + if(m->socket.fd == s) { + + if(what == CURL_POLL_REMOVE) { + struct monitor *nxt = m->next; + /* remove this node from the list of monitored sockets */ + if(prev) + prev->next = nxt; + else + ev->list = nxt; + free(m); + m = nxt; + infof(easy, "socket cb: socket %d REMOVED\n", s); + } + else { + /* The socket 's' is already being monitored, update the activity + mask. Convert from libcurl bitmask to the poll one. */ + m->socket.events = socketcb2poll(what); + infof(easy, "socket cb: socket %d UPDATED as %s%s\n", s, + what&CURL_POLL_IN?"IN":"", + what&CURL_POLL_OUT?"OUT":""); + } + break; + } + prev = m; + m = m->next; /* move to next node */ + } + if(!m) { + if(what == CURL_POLL_REMOVE) { + /* this happens a bit too often, libcurl fix perhaps? */ + /* fprintf(stderr, + "%s: socket %d asked to be REMOVED but not present!\n", + __func__, s); */ + } + else { + m = malloc(sizeof(struct monitor)); + m->next = ev->list; + m->socket.fd = s; + m->socket.events = socketcb2poll(what); + m->socket.revents = 0; + ev->list = m; + infof(easy, "socket cb: socket %d ADDED as %s%s\n", s, + what&CURL_POLL_IN?"IN":"", + what&CURL_POLL_OUT?"OUT":""); + } + } + + return 0; +} + + +/* + * events_setup() + * + * Do the multi handle setups that only event-based transfers need. + */ +static void events_setup(CURLM *multi, struct events *ev) +{ + /* timer callback */ + curl_multi_setopt(multi, CURLMOPT_TIMERFUNCTION, events_timer); + curl_multi_setopt(multi, CURLMOPT_TIMERDATA, ev); + + /* socket callback */ + curl_multi_setopt(multi, CURLMOPT_SOCKETFUNCTION, events_socket); + curl_multi_setopt(multi, CURLMOPT_SOCKETDATA, ev); +} + + +/* wait_or_timeout() + * + * waits for activity on any of the given sockets, or the timeout to trigger. + */ + +static CURLcode wait_or_timeout(struct Curl_multi *multi, struct events *ev) { - CURLM *multi; - CURLMcode mcode; - CURLcode code = CURLE_OK; - CURLMsg *msg; bool done = FALSE; - int rc; - struct SessionHandle *data = easy; + CURLMcode mcode; + CURLcode rc = CURLE_OK; + + while(!done) { + CURLMsg *msg; + struct monitor *m; + struct pollfd *f; + struct pollfd fds[4]; + int numfds=0; + int pollrc; + int i; + struct timeval before; + struct timeval after; + + /* populate the fds[] array */ + for(m = ev->list, f=&fds[0]; m; m = m->next) { + f->fd = m->socket.fd; + f->events = m->socket.events; + f->revents = 0; + /* fprintf(stderr, "poll() %d check socket %d\n", numfds, f->fd); */ + f++; + numfds++; + } + + /* get the time stamp to use to figure out how long poll takes */ + before = curlx_tvnow(); + + /* fprintf(stderr, "poll(), %d numfds\n", numfds); */ + + /* wait for activity or timeout */ + pollrc = Curl_poll(fds, numfds, (int)ev->ms); + + after = curlx_tvnow(); + + ev->msbump = FALSE; /* reset here */ + + if(0 == pollrc) { + /* timeout! */ + ev->ms = 0; + /* fprintf(stderr, "call curl_multi_socket_action( TIMEOUT )\n"); */ + mcode = curl_multi_socket_action(multi, CURL_SOCKET_TIMEOUT, 0, + &ev->running_handles); + } + else if(pollrc > 0) { + /* loop over the monitored sockets to see which ones had activity */ + for(i = 0; i< numfds; i++) { + if(fds[i].revents) { + /* socket activity, tell libcurl */ + int act = poll2cselect(fds[i].revents); /* convert */ + infof(multi->easyp, "call curl_multi_socket_action( socket %d )\n", + fds[i].fd); + mcode = curl_multi_socket_action(multi, fds[i].fd, act, + &ev->running_handles); + } + } + + if(!ev->msbump) + /* If nothing updated the timeout, we decrease it by the spent time. + * If it was updated, it has the new timeout time stored already. + */ + ev->ms += curlx_tvdiff(after, before); + + } + if(mcode) + return CURLE_URL_MALFORMAT; /* TODO: return a proper error! */ + + /* we don't really care about the "msgs_in_queue" value returned in the + second argument */ + msg = curl_multi_info_read(multi, &pollrc); + if(msg) { + rc = msg->data.result; + done = TRUE; + } + } + + return rc; +} + + +/* easy_events() + * + * Runs a transfer in a blocking manner using the events-based API + */ +static CURLcode easy_events(CURLM *multi) +{ + struct events evs= {2, FALSE, 0, NULL, 0}; + + /* if running event-based, do some further multi inits */ + events_setup(multi, &evs); + + return wait_or_timeout(multi, &evs); +} +#endif + +static CURLcode easy_transfer(CURLM *multi) +{ + bool done = FALSE; + CURLMcode mcode = CURLM_OK; + CURLcode code; + struct timeval before; int without_fds = 0; /* count number of consecutive returns from curl_multi_wait() without any filedescriptors */ - struct timeval before; - SIGPIPE_VARIABLE(pipe_st); - - if(!easy) - return CURLE_BAD_FUNCTION_ARGUMENT; - - if(data->multi) { - failf(data, "easy handled already used in multi handle"); - return CURLE_FAILED_INIT; - } - - if(data->multi_easy) - multi = data->multi_easy; - else { - /* this multi handle will only ever have a single easy handled attached - to it, so make it use minimal hashes */ - multi = Curl_multi_handle(1, 3); - if(!multi) - return CURLE_OUT_OF_MEMORY; - data->multi_easy = multi; - } - - /* Copy the MAXCONNECTS option to the multi handle */ - curl_multi_setopt(multi, CURLMOPT_MAXCONNECTS, data->set.maxconnects); - - mcode = curl_multi_add_handle(multi, easy); - if(mcode) { - curl_multi_cleanup(multi); - if(mcode == CURLM_OUT_OF_MEMORY) - return CURLE_OUT_OF_MEMORY; - else - return CURLE_FAILED_INIT; - } - - sigpipe_ignore(data, &pipe_st); - - /* assign this after curl_multi_add_handle() since that function checks for - it and rejects this handle otherwise */ - data->multi = multi; while(!done && !mcode) { int still_running; @@ -556,13 +761,82 @@ CURLcode curl_easy_perform(CURL *easy) /* only read 'still_running' if curl_multi_perform() return OK */ if((mcode == CURLM_OK) && !still_running) { - msg = curl_multi_info_read(multi, &rc); + int rc; + CURLMsg *msg = curl_multi_info_read(multi, &rc); if(msg) { code = msg->data.result; done = TRUE; } } } + return code; +} + + +/* + * easy_perform() is the external interface that performs a blocking + * transfer as previously setup. + * + * CONCEPT: This function creates a multi handle, adds the easy handle to it, + * runs curl_multi_perform() until the transfer is done, then detaches the + * easy handle, destroys the multi handle and returns the easy handle's return + * code. + * + * REALITY: it can't just create and destroy the multi handle that easily. It + * needs to keep it around since if this easy handle is used again by this + * function, the same multi handle must be re-used so that the same pools and + * caches can be used. + * + * DEBUG: if 'events' is set TRUE, this function will use a replacement engine + * instead of curl_multi_perform() and use curl_multi_socket_action(). + */ +static CURLcode easy_perform(CURL *easy, bool events) +{ + CURLM *multi; + CURLMcode mcode; + CURLcode code = CURLE_OK; + struct SessionHandle *data = easy; + SIGPIPE_VARIABLE(pipe_st); + + if(!easy) + return CURLE_BAD_FUNCTION_ARGUMENT; + + if(data->multi) { + failf(data, "easy handled already used in multi handle"); + return CURLE_FAILED_INIT; + } + + if(data->multi_easy) + multi = data->multi_easy; + else { + /* this multi handle will only ever have a single easy handled attached + to it, so make it use minimal hashes */ + multi = Curl_multi_handle(1, 3); + if(!multi) + return CURLE_OUT_OF_MEMORY; + data->multi_easy = multi; + } + + /* Copy the MAXCONNECTS option to the multi handle */ + curl_multi_setopt(multi, CURLMOPT_MAXCONNECTS, data->set.maxconnects); + + mcode = curl_multi_add_handle(multi, easy); + if(mcode) { + curl_multi_cleanup(multi); + if(mcode == CURLM_OUT_OF_MEMORY) + return CURLE_OUT_OF_MEMORY; + else + return CURLE_FAILED_INIT; + } + + sigpipe_ignore(data, &pipe_st); + + /* assign this after curl_multi_add_handle() since that function checks for + it and rejects this handle otherwise */ + data->multi = multi; + + /* run the transfer */ + code = events ? easy_events(multi) : easy_transfer(multi); /* ignoring the return code isn't nice, but atm we can't really handle a failure here, room for future improvement! */ @@ -574,6 +848,28 @@ CURLcode curl_easy_perform(CURL *easy) return code; } + +/* + * curl_easy_perform() is the external interface that performs a blocking + * transfer as previously setup. + */ +CURLcode curl_easy_perform(CURL *easy) +{ + return easy_perform(easy, FALSE); +} + +#ifdef CURLDEBUG +/* + * curl_easy_perform_ev() is the external interface that performs a blocking + * transfer using the event-based API internally. + */ +CURLcode curl_easy_perform_ev(CURL *easy) +{ + return easy_perform(easy, TRUE); +} + +#endif + /* * curl_easy_cleanup() is the external interface to cleaning/freeing the given * easy handle. diff --git a/lib/easyif.h b/lib/easyif.h index c55819396..eed160577 100644 --- a/lib/easyif.h +++ b/lib/easyif.h @@ -25,6 +25,9 @@ /* * Prototypes for library-wide functions provided by easy.c */ +#ifdef CURLDEBUG +CURLcode curl_easy_perform_ev(CURL *easy); +#endif #endif /* HEADER_CURL_EASYIF_H */ diff --git a/src/tool_cfgable.h b/src/tool_cfgable.h index 9a9b6d8d3..144552e84 100644 --- a/src/tool_cfgable.h +++ b/src/tool_cfgable.h @@ -7,7 +7,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2012, Daniel Stenberg, , et al. + * Copyright (C) 1998 - 2013, Daniel Stenberg, , et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -205,6 +205,9 @@ struct Configurable { bool use_metalink; /* process given URLs as metalink XML file */ metalinkfile *metalinkfile_list; /* point to the first node */ metalinkfile *metalinkfile_last; /* point to the last/current node */ +#ifdef CURLDEBUG + bool test_event_based; +#endif }; /* struct Configurable */ void free_config_fields(struct Configurable *config); diff --git a/src/tool_getparam.c b/src/tool_getparam.c index c3d86931c..d9deb3b2b 100644 --- a/src/tool_getparam.c +++ b/src/tool_getparam.c @@ -174,6 +174,7 @@ static const struct LongShort aliases[]= { {"$I", "post303", FALSE}, {"$J", "metalink", FALSE}, {"$K", "sasl-ir", FALSE}, + {"$L", "test-event", FALSE}, {"0", "http1.0", FALSE}, {"1", "tlsv1", FALSE}, {"2", "sslv2", FALSE}, @@ -961,6 +962,13 @@ ParameterError getparameter(char *flag, /* f or -long-flag */ case 'K': /* --sasl-ir */ config->sasl_ir = toggle; break; + case 'L': /* --test-event */ +#ifdef CURLDEBUG + config->test_event_based = toggle; +#else + warnf(config, "--test-event is ignored unless a debug build!\n"); +#endif + break; } break; case '#': /* --progress-bar */ diff --git a/src/tool_operate.c b/src/tool_operate.c index dbbbc26c2..60d09ff25 100644 --- a/src/tool_operate.c +++ b/src/tool_operate.c @@ -81,6 +81,11 @@ #include "memdebug.h" /* keep this as LAST include */ +#ifdef CURLDEBUG +/* libcurl's debug builds provide an extra function */ +CURLcode curl_easy_perform_ev(CURL *easy); +#endif + #define CURLseparator "--_curl_--" #ifndef O_BINARY @@ -1450,6 +1455,11 @@ int operate(struct Configurable *config, int argc, argv_item_t argv[]) mlfile->filename, this_url); #endif /* USE_METALINK */ +#ifdef CURLDEBUG + if(config->test_event_based) + res = curl_easy_perform_ev(curl); + else +#endif res = curl_easy_perform(curl); if(outs.is_cd_filename && outs.stream && !config->mute && diff --git a/tests/runtests.1 b/tests/runtests.1 index 45f878333..0de38f69a 100644 --- a/tests/runtests.1 +++ b/tests/runtests.1 @@ -5,7 +5,7 @@ .\" * | (__| |_| | _ <| |___ .\" * \___|\___/|_| \_\_____| .\" * -.\" * Copyright (C) 1998 - 2007, Daniel Stenberg, , et al. +.\" * Copyright (C) 1998 - 2013, Daniel Stenberg, , et al. .\" * .\" * This software is licensed as described in the file COPYING, which .\" * you should have received as part of this distribution. The terms @@ -53,6 +53,10 @@ Provide a custom curl binary to run the tests with. Default is the curl executable in the build tree. .IP "-d" Enable protocol debug: have the servers display protocol output. +.IP "-e" +Run the test event-based (if possible). This will make runtests invoke curl +with --test-event option. This option only works if both curl and libcurl were +built debug-enabled. .IP "-g" Run the given test(s) with gdb. This is best used on a single test case and curl built --disable-shared. This then fires up gdb with command line set to diff --git a/tests/runtests.pl b/tests/runtests.pl index 96df8d524..5c50337e3 100755 --- a/tests/runtests.pl +++ b/tests/runtests.pl @@ -274,6 +274,7 @@ my $gdbxwin; # use windowed gdb when using gdb my $keepoutfiles; # keep stdout and stderr files after tests my $listonly; # only list the tests my $postmortem; # display detailed info about failed tests +my $run_event_based; # run curl with --test-event to test the event API my %run; # running server my %doesntrun; # servers that don't work, identified by pidfile @@ -2710,7 +2711,11 @@ sub timestampskippedevents { # Run a single specified test case # sub singletest { - my ($testnum, $count, $total)=@_; + my ($evbased, # 1 means switch on if possible (and "curl" is tested) + # returns "not a test" if it can't be used for this test + $testnum, + $count, + $total)=@_; my @what; my $why; @@ -3127,6 +3132,7 @@ sub singletest { my $CMDLINE; my $cmdargs; my $cmdtype = $cmdhash{'type'} || "default"; + my $fail_due_event_based = $evbased; if($cmdtype eq "perl") { # run the command line prepended with "perl" $cmdargs ="$cmd"; @@ -3142,15 +3148,22 @@ sub singletest { $disablevalgrind=1; } elsif(!$tool) { - # run curl, add --verbose for debug information output + # run curl, add suitable command line options $cmd = "-1 ".$cmd if(exists $feature{"SSL"} && ($has_axtls)); my $inc=""; if((!$cmdhash{'option'}) || ($cmdhash{'option'} !~ /no-include/)) { - $inc = "--include "; + $inc = " --include"; } - $cmdargs ="$out $inc--trace-ascii log/trace$testnum --trace-time $cmd"; + $cmdargs = "$out$inc "; + $cmdargs .= "--trace-ascii log/trace$testnum "; + $cmdargs .= "--trace-time "; + if($evbased) { + $cmdargs .= "--test-event "; + $fail_due_event_based--; + } + $cmdargs .= $cmd; } else { $cmdargs = " $cmd"; # $cmd is the command line for the test file @@ -3171,6 +3184,11 @@ sub singletest { $DBGCURL=$CMDLINE; } + if($fail_due_event_based) { + logmsg "This test cannot run event based\n"; + return -1; + } + my @stdintest = getpart("client", "stdin"); if(@stdintest) { @@ -3755,6 +3773,8 @@ sub singletest { else { $ok .= "-"; # valgrind not checked } + # add 'E' for event-based + $ok .= $evbased ? "E" : "-"; logmsg "$ok " if(!$short); @@ -4454,6 +4474,10 @@ while(@ARGV) { # continue anyway, even if a test fail $anyway=1; } + elsif($ARGV[0] eq "-e") { + # run the tests cases event based if possible + $run_event_based=1; + } elsif($ARGV[0] eq "-p") { $postmortem=1; } @@ -4825,7 +4849,7 @@ foreach $testnum (@at) { $lasttest = $testnum if($testnum > $lasttest); $count++; - my $error = singletest($testnum, $count, scalar(@at)); + my $error = singletest($run_event_based, $testnum, $count, scalar(@at)); if($error < 0) { # not a test we can run next;