- Adam D. Moss made the HTTP CONNECT procedure less blocking when used from

the multi interface. Note that it still does a part of the connection in a
  blocking manner.

CHANGES

@@ -6,6 +6,11 @@
 
                                   Changelog
 
+Daniel (25 February 2007)
+- Adam D. Moss made the HTTP CONNECT procedure less blocking when used from
+  the multi interface. Note that it still does a part of the connection in a
+  blocking manner.
+
 Daniel (23 February 2007)
 - Added warning outputs if the command line uses more than one of the options
   -v, --trace and --trace-ascii, since it could really confuse the user.

RELEASE-NOTES

@@ -33,6 +33,8 @@ This release includes the following bugfixes:
  o curl-config --libs and libcurl.pc no longer list unnecessary dependencies
  o fixed an issue with CCC not working on some servers
  o several HTTP pipelining problems
+ o HTTP CONNECT thru a proxy is now less blocking when the multi interface is
+   used
 
 This release includes the following known bugs:
 
@@ -52,6 +54,6 @@ advice from friends like these:
  Yang Tse, Manfred Schwarb, Michael Wallner, Jeff Pohlmeyer, Shmulik Regev,
  Rob Crittenden, Robert A. Monat, Dan Fandrich, Duncan Mac-Vicar Prett,
  Michal Marek, Robson Braga Araujo, Ian Turner, Linus Nielsen Feltzing,
- Ravi Pratap
+ Ravi Pratap, Adam D. Moss
 
         Thanks! (and sorry if I forgot to mention someone)

lib/http.c

@@ -1115,259 +1115,309 @@ CURLcode Curl_proxyCONNECT(struct connectdata *conn,
   struct Curl_transfer_keeper *k = &data->reqdata.keep;
   CURLcode result;
   int res;
-  size_t nread;   /* total size read */
-  int perline; /* count bytes per line */
-  int keepon=TRUE;
-  ssize_t gotbytes;
-  char *ptr;
   long timeout =
     data->set.timeout?data->set.timeout:3600000; /* in milliseconds */
-  char *line_start;
-  char *host_port;
   curl_socket_t tunnelsocket = conn->sock[sockindex];
-  send_buffer *req_buffer;
   curl_off_t cl=0;
   bool closeConnection = FALSE;
+  long check;
 
 #define SELECT_OK      0
 #define SELECT_ERROR   1
 #define SELECT_TIMEOUT 2
   int error = SELECT_OK;
 
-  infof(data, "Establish HTTP proxy tunnel to %s:%d\n", hostname, remote_port);
   conn->bits.proxy_connect_closed = FALSE;
 
   do {
-    if(data->reqdata.newurl) {
+    if (!conn->bits.tunnel_connecting) { /* BEGIN CONNECT PHASE */
-      /* This only happens if we've looped here due to authentication reasons,
+      char *host_port;
-         and we don't really use the newly cloned URL here then. Just free()
+      send_buffer *req_buffer;
-         it. */
-      free(data->reqdata.newurl);
-      data->reqdata.newurl = NULL;
-    }
 
-    /* initialize a dynamic send-buffer */
+      infof(data, "Establish HTTP proxy tunnel to %s:%d\n",
-    req_buffer = add_buffer_init();
+            hostname, remote_port);
 
-    if(!req_buffer)
+      if(data->reqdata.newurl) {
-      return CURLE_OUT_OF_MEMORY;
+        /* This only happens if we've looped here due to authentication
-
+           reasons, and we don't really use the newly cloned URL here
-    host_port = aprintf("%s:%d", hostname, remote_port);
+           then. Just free() it. */
-    if(!host_port)
+        free(data->reqdata.newurl);
-      return CURLE_OUT_OF_MEMORY;
+        data->reqdata.newurl = NULL;
-
-    /* Setup the proxy-authorization header, if any */
-    result = Curl_http_output_auth(conn, (char *)"CONNECT", host_port, TRUE);
-
-    if(CURLE_OK == result) {
-      char *host=(char *)"";
-      const char *proxyconn="";
-      const char *useragent="";
-
-      if(!checkheaders(data, "Host:")) {
-        host = aprintf("Host: %s\r\n", host_port);
-        if(!host)
-          result = CURLE_OUT_OF_MEMORY;
       }
-      if(!checkheaders(data, "Proxy-Connection:"))
-        proxyconn = "Proxy-Connection: Keep-Alive\r\n";
 
-      if(!checkheaders(data, "User-Agent:") && data->set.useragent)
+      /* initialize a dynamic send-buffer */
-        useragent = conn->allocptr.uagent;
+      req_buffer = add_buffer_init();
+
+      if(!req_buffer)
+        return CURLE_OUT_OF_MEMORY;
+
+      host_port = aprintf("%s:%d", hostname, remote_port);
+      if(!host_port)
+        return CURLE_OUT_OF_MEMORY;
+
+      /* Setup the proxy-authorization header, if any */
+      result = Curl_http_output_auth(conn, (char *)"CONNECT", host_port, TRUE);
 
       if(CURLE_OK == result) {
-        /* Send the connect request to the proxy */
+        char *host=(char *)"";
-        /* BLOCKING */
+        const char *proxyconn="";
-        result =
+        const char *useragent="";
-          add_bufferf(req_buffer,
-                      "CONNECT %s:%d HTTP/1.0\r\n"
-                      "%s"  /* Host: */
-                      "%s"  /* Proxy-Authorization */
-                      "%s"  /* User-Agent */
-                      "%s", /* Proxy-Connection */
-                      hostname, remote_port,
-                      host,
-                      conn->allocptr.proxyuserpwd?
-                      conn->allocptr.proxyuserpwd:"",
-                      useragent,
-                      proxyconn);
 
-        if(CURLE_OK == result)
+        if(!checkheaders(data, "Host:")) {
-          result = add_custom_headers(conn, req_buffer);
+          host = aprintf("Host: %s\r\n", host_port);
+          if(!host)
+            result = CURLE_OUT_OF_MEMORY;
+        }
+        if(!checkheaders(data, "Proxy-Connection:"))
+          proxyconn = "Proxy-Connection: Keep-Alive\r\n";
 
-        if(host && *host)
+        if(!checkheaders(data, "User-Agent:") && data->set.useragent)
-          free(host);
+          useragent = conn->allocptr.uagent;
 
-        if(CURLE_OK == result)
+        if(CURLE_OK == result) {
-          /* CRLF terminate the request */
+          /* Send the connect request to the proxy */
-          result = add_bufferf(req_buffer, "\r\n");
+          /* BLOCKING */
+          result =
+            add_bufferf(req_buffer,
+                        "CONNECT %s:%d HTTP/1.0\r\n"
+                        "%s"  /* Host: */
+                        "%s"  /* Proxy-Authorization */
+                        "%s"  /* User-Agent */
+                        "%s", /* Proxy-Connection */
+                        hostname, remote_port,
+                        host,
+                        conn->allocptr.proxyuserpwd?
+                        conn->allocptr.proxyuserpwd:"",
+                        useragent,
+                        proxyconn);
 
-        if(CURLE_OK == result)
+          if(CURLE_OK == result)
-          /* Now send off the request */
+            result = add_custom_headers(conn, req_buffer);
-          result = add_buffer_send(req_buffer, conn,
+
-                                   &data->info.request_size, 0, sockindex);
+          if(host && *host)
+            free(host);
+
+          if(CURLE_OK == result)
+            /* CRLF terminate the request */
+            result = add_bufferf(req_buffer, "\r\n");
+
+          if(CURLE_OK == result)
+            /* Now send off the request */
+            result = add_buffer_send(req_buffer, conn,
+                                     &data->info.request_size, 0, sockindex);
+        }
+        if(result)
+          failf(data, "Failed sending CONNECT to proxy");
       }
+      free(host_port);
       if(result)
-        failf(data, "Failed sending CONNECT to proxy");
+        return result;
-    }
-    free(host_port);
-    if(result)
-      return result;
 
-    ptr=data->state.buffer;
+      conn->bits.tunnel_connecting = TRUE;
-    line_start = ptr;
+    } /* END CONNECT PHASE */
 
-    nread=0;
+    /* now we've issued the CONNECT and we're waiting to hear back -
-    perline=0;
+       we try not to block here in multi-mode because that might be a LONG
-    keepon=TRUE;
+       wait if the proxy cannot connect-through to the remote host. */
 
-    while((nread<BUFSIZE) && (keepon && !error)) {
+    /* if timeout is requested, find out how much remaining time we have */
-
+    check = timeout - /* timeout time */
-      /* if timeout is requested, find out how much remaining time we have */
+      Curl_tvdiff(Curl_tvnow(), conn->now); /* spent time */
-      long check = timeout - /* timeout time */
+    if(check <=0 ) {
-        Curl_tvdiff(Curl_tvnow(), conn->now); /* spent time */
+      failf(data, "Proxy CONNECT aborted due to timeout");
-      if(check <= 0) {
+      error = SELECT_TIMEOUT; /* already too little time */
-        failf(data, "Proxy CONNECT aborted due to timeout");
-        error = SELECT_TIMEOUT; /* already too little time */
-        break;
-      }
-
-      /* loop every second at least, less if the timeout is near */
-      switch (Curl_select(tunnelsocket, CURL_SOCKET_BAD,
-                          check<1000L?(int)check:1000)) {
-      case -1: /* select() error, stop reading */
-        error = SELECT_ERROR;
-        failf(data, "Proxy CONNECT aborted due to select() error");
-        break;
-      case 0: /* timeout */
-        break;
-      default:
-        res = Curl_read(conn, tunnelsocket, ptr, BUFSIZE-nread, &gotbytes);
-        if(res< 0)
-          /* EWOULDBLOCK */
-          continue; /* go loop yourself */
-        else if(res)
-          keepon = FALSE;
-        else if(gotbytes <= 0) {
-          keepon = FALSE;
-          error = SELECT_ERROR;
-          failf(data, "Proxy CONNECT aborted");
-        }
-        else {
-          /*
-           * We got a whole chunk of data, which can be anything from one byte
-           * to a set of lines and possibly just a piece of the last line.
-           */
-          int i;
-
-          nread += gotbytes;
-
-          if(keepon > TRUE) {
-            /* This means we are currently ignoring a response-body, so we
-               simply count down our counter and make sure to break out of the
-               loop when we're done! */
-            cl -= gotbytes;
-            if(cl<=0) {
-              keepon = FALSE;
-              break;
-            }
-          }
-          else
-          for(i = 0; i < gotbytes; ptr++, i++) {
-            perline++; /* amount of bytes in this line so far */
-            if(*ptr=='\n') {
-              char letter;
-              int writetype;
-
-              /* output debug if that is requested */
-              if(data->set.verbose)
-                Curl_debug(data, CURLINFO_HEADER_IN,
-                           line_start, (size_t)perline, conn);
-
-              /* send the header to the callback */
-              writetype = CLIENTWRITE_HEADER;
-              if(data->set.include_header)
-                writetype |= CLIENTWRITE_BODY;
-
-              result = Curl_client_write(conn, writetype, line_start, perline);
-              if(result)
-                return result;
-
-              /* Newlines are CRLF, so the CR is ignored as the line isn't
-                 really terminated until the LF comes. Treat a following CR
-                 as end-of-headers as well.*/
-
-              if(('\r' == line_start[0]) ||
-                 ('\n' == line_start[0])) {
-                /* end of response-headers from the proxy */
-                if(cl && (407 == k->httpcode) && !data->state.authproblem) {
-                  /* If we get a 407 response code with content length when we
-                   * have no auth problem, we must ignore the whole
-                   * response-body */
-                  keepon = 2;
-                  infof(data, "Ignore %" FORMAT_OFF_T
-                        " bytes of response-body\n", cl);
-                  cl -= (gotbytes - i);/* remove the remaining chunk of what
-                                          we already read */
-                  if(cl<=0)
-                    /* if the whole thing was already read, we are done! */
-                    keepon=FALSE;
-                }
-                else
-                  keepon = FALSE;
-                break; /* breaks out of for-loop, not switch() */
-              }
-
-              /* keep a backup of the position we are about to blank */
-              letter = line_start[perline];
-              line_start[perline]=0; /* zero terminate the buffer */
-              if((checkprefix("WWW-Authenticate:", line_start) &&
-                  (401 == k->httpcode)) ||
-                 (checkprefix("Proxy-authenticate:", line_start) &&
-                  (407 == k->httpcode))) {
-                result = Curl_http_input_auth(conn, k->httpcode, line_start);
-                if(result)
-                  return result;
-              }
-              else if(checkprefix("Content-Length:", line_start)) {
-                cl = curlx_strtoofft(line_start + strlen("Content-Length:"),
-                                     NULL, 10);
-              }
-              else if(Curl_compareheader(line_start,
-                                         "Connection:", "close"))
-                closeConnection = TRUE;
-              else if(2 == sscanf(line_start, "HTTP/1.%d %d",
-                                  &subversion,
-                                  &k->httpcode)) {
-                /* store the HTTP code from the proxy */
-                data->info.httpproxycode = k->httpcode;
-              }
-              /* put back the letter we blanked out before */
-              line_start[perline]= letter;
-
-              perline=0; /* line starts over here */
-              line_start = ptr+1; /* this skips the zero byte we wrote */
-            }
-          }
-        }
-        break;
-      } /* switch */
-    } /* while there's buffer left and loop is requested */
-
-    if(error)
-      return CURLE_RECV_ERROR;
-
-    if(data->info.httpproxycode != 200)
-      /* Deal with the possibly already received authenticate
-         headers. 'newurl' is set to a new URL if we must loop. */
-      Curl_http_auth_act(conn);
-
-    if (closeConnection && data->reqdata.newurl) {
-      /* Connection closed by server. Don't use it anymore */
-      sclose(conn->sock[sockindex]);
-      conn->sock[sockindex] = CURL_SOCKET_BAD;
       break;
     }
+
+    /* if we're in multi-mode and we would block, return instead for a retry */
+    if (Curl_if_multi == data->state.used_interface) {
+      if (0 == Curl_select(tunnelsocket, CURL_SOCKET_BAD, 0))
+        /* return so we'll be called again polling-style */
+        return CURLE_OK;
+      else {
+        DEBUGF(infof(data,
+                     "Multi mode finished polling for response from "
+                     "proxy CONNECT."));
+      }
+    }
+    else {
+      DEBUGF(infof(data, "Easy mode waiting for response from proxy CONNECT."));
+    }
+
+    /* at this point, either:
+       1) we're in easy-mode and so it's okay to block waiting for a CONNECT
+       response
+       2) we're in multi-mode and we didn't block - it's either an error or we
+       now have some data waiting.
+       In any case, the tunnel_connecting phase is over. */
+    conn->bits.tunnel_connecting = FALSE;
+
+    { /* BEGIN NEGOTIATION PHASE */
+      size_t nread;   /* total size read */
+      int perline; /* count bytes per line */
+      int keepon=TRUE;
+      ssize_t gotbytes;
+      char *ptr;
+      char *line_start;
+
+      ptr=data->state.buffer;
+      line_start = ptr;
+
+      nread=0;
+      perline=0;
+      keepon=TRUE;
+
+      while((nread<BUFSIZE) && (keepon && !error)) {
+
+        /* if timeout is requested, find out how much remaining time we have */
+        check = timeout - /* timeout time */
+          Curl_tvdiff(Curl_tvnow(), conn->now); /* spent time */
+        if(check <= 0) {
+          failf(data, "Proxy CONNECT aborted due to timeout");
+          error = SELECT_TIMEOUT; /* already too little time */
+          break;
+        }
+
+        /* loop every second at least, less if the timeout is near */
+        switch (Curl_select(tunnelsocket, CURL_SOCKET_BAD,
+                            check<1000L?(int)check:1000)) {
+        case -1: /* select() error, stop reading */
+          error = SELECT_ERROR;
+          failf(data, "Proxy CONNECT aborted due to select() error");
+          break;
+        case 0: /* timeout */
+          break;
+        default:
+          res = Curl_read(conn, tunnelsocket, ptr, BUFSIZE-nread, &gotbytes);
+          if(res< 0)
+            /* EWOULDBLOCK */
+            continue; /* go loop yourself */
+          else if(res)
+            keepon = FALSE;
+          else if(gotbytes <= 0) {
+            keepon = FALSE;
+            error = SELECT_ERROR;
+            failf(data, "Proxy CONNECT aborted");
+          }
+          else {
+            /*
+             * We got a whole chunk of data, which can be anything from one
+             * byte to a set of lines and possibly just a piece of the last
+             * line.
+             */
+            int i;
+
+            nread += gotbytes;
+
+            if(keepon > TRUE) {
+              /* This means we are currently ignoring a response-body, so we
+                 simply count down our counter and make sure to break out of
+                 the loop when we're done! */
+              cl -= gotbytes;
+              if(cl<=0) {
+                keepon = FALSE;
+                break;
+              }
+            }
+            else
+              for(i = 0; i < gotbytes; ptr++, i++) {
+                perline++; /* amount of bytes in this line so far */
+                if(*ptr=='\n') {
+                  char letter;
+                  int writetype;
+
+                  /* output debug if that is requested */
+                  if(data->set.verbose)
+                    Curl_debug(data, CURLINFO_HEADER_IN,
+                               line_start, (size_t)perline, conn);
+
+                  /* send the header to the callback */
+                  writetype = CLIENTWRITE_HEADER;
+                  if(data->set.include_header)
+                    writetype |= CLIENTWRITE_BODY;
+
+                  result = Curl_client_write(conn, writetype, line_start, perline);
+                  if(result)
+                    return result;
+
+                  /* Newlines are CRLF, so the CR is ignored as the line isn't
+                     really terminated until the LF comes. Treat a following CR
+                     as end-of-headers as well.*/
+
+                  if(('\r' == line_start[0]) ||
+                     ('\n' == line_start[0])) {
+                    /* end of response-headers from the proxy */
+                    if(cl && (407 == k->httpcode) &&
+                       !data->state.authproblem) {
+                      /* If we get a 407 response code with content length
+                       * when we have no auth problem, we must ignore the
+                       * whole response-body */
+                      keepon = 2;
+                      infof(data, "Ignore %" FORMAT_OFF_T
+                            " bytes of response-body\n", cl);
+                      cl -= (gotbytes - i);/* remove the remaining chunk of
+                                              what we already read */
+                      if(cl<=0)
+                        /* if the whole thing was already read, we are done! */
+                        keepon=FALSE;
+                    }
+                    else
+                      keepon = FALSE;
+                    break; /* breaks out of for-loop, not switch() */
+                  }
+
+                  /* keep a backup of the position we are about to blank */
+                  letter = line_start[perline];
+                  line_start[perline]=0; /* zero terminate the buffer */
+                  if((checkprefix("WWW-Authenticate:", line_start) &&
+                      (401 == k->httpcode)) ||
+                     (checkprefix("Proxy-authenticate:", line_start) &&
+                      (407 == k->httpcode))) {
+                    result = Curl_http_input_auth(conn, k->httpcode,
+                                                  line_start);
+                    if(result)
+                      return result;
+                  }
+                  else if(checkprefix("Content-Length:", line_start)) {
+                    cl = curlx_strtoofft(line_start + strlen("Content-Length:"),
+                                         NULL, 10);
+                  }
+                  else if(Curl_compareheader(line_start,
+                                             "Connection:", "close"))
+                    closeConnection = TRUE;
+                  else if(2 == sscanf(line_start, "HTTP/1.%d %d",
+                                      &subversion,
+                                      &k->httpcode)) {
+                    /* store the HTTP code from the proxy */
+                    data->info.httpproxycode = k->httpcode;
+                  }
+                  /* put back the letter we blanked out before */
+                  line_start[perline]= letter;
+
+                  perline=0; /* line starts over here */
+                  line_start = ptr+1; /* this skips the zero byte we wrote */
+                }
+              }
+          }
+          break;
+        } /* switch */
+      } /* while there's buffer left and loop is requested */
+
+      if(error)
+        return CURLE_RECV_ERROR;
+
+      if(data->info.httpproxycode != 200)
+        /* Deal with the possibly already received authenticate
+           headers. 'newurl' is set to a new URL if we must loop. */
+        Curl_http_auth_act(conn);
+
+      if (closeConnection && data->reqdata.newurl) {
+        /* Connection closed by server. Don't use it anymore */
+        sclose(conn->sock[sockindex]);
+        conn->sock[sockindex] = CURL_SOCKET_BAD;
+        break;
+      }
+    } /* END NEGOTIATION PHASE */
   } while(data->reqdata.newurl);
 
   if(200 != k->httpcode) {
@@ -1423,6 +1473,11 @@ CURLcode Curl_http_connect(struct connectdata *conn, bool *done)
       return result;
   }
 
+  if (conn->bits.tunnel_connecting) {
+    /* nothing else to do except wait right now - we're not done here. */
+    return CURLE_OK;
+  }
+
   if(!data->state.this_is_a_follow) {
     /* this is not a followed location, get the original host name */
     if (data->state.first_host)

lib/multi.c

@@ -47,6 +47,7 @@
 #include "multiif.h"
 #include "sendf.h"
 #include "timeval.h"
+#include "http.h"
 
 /* The last #include file should be: */
 #include "memdebug.h"
@@ -62,6 +63,7 @@ typedef enum {
   CURLM_STATE_CONNECT,     /* resolve/connect has been sent off */
   CURLM_STATE_WAITRESOLVE, /* awaiting the resolve to finalize */
   CURLM_STATE_WAITCONNECT, /* awaiting the connect to finalize */
+  CURLM_STATE_WAITPROXYCONNECT, /* awaiting proxy CONNECT to finalize */
   CURLM_STATE_PROTOCONNECT, /* completing the protocol-specific connect
                                phase */
   CURLM_STATE_WAITDO,      /* wait for our turn to send the request */
@@ -791,7 +793,8 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
         multistate(easy, CURLM_STATE_CONNECT);
         result = CURLM_CALL_MULTI_PERFORM;
         easy->result = CURLE_OK;
-      } else {
+      }
+      else {
         easy->result = CURLE_COULDNT_CONNECT;
         multistate(easy, CURLM_STATE_COMPLETED);
       }
@@ -871,10 +874,13 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
              WAITDO! */
           result = CURLM_CALL_MULTI_PERFORM;
 
-          if(protocol_connect) {
+          if(protocol_connect)
             multistate(easy, CURLM_STATE_WAITDO);
-          } else {
+          else {
-            multistate(easy, CURLM_STATE_WAITCONNECT);
+            if (easy->easy_conn->bits.tunnel_connecting)
+              multistate(easy, CURLM_STATE_WAITPROXYCONNECT);
+            else
+              multistate(easy, CURLM_STATE_WAITCONNECT);
           }
         }
       }
@@ -903,8 +909,12 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
           result = CURLM_CALL_MULTI_PERFORM;
           if(protocol_connect)
             multistate(easy, CURLM_STATE_DO);
-          else
+          else {
-            multistate(easy, CURLM_STATE_WAITCONNECT);
+            if (easy->easy_conn->bits.tunnel_connecting)
+              multistate(easy, CURLM_STATE_WAITPROXYCONNECT);
+            else
+              multistate(easy, CURLM_STATE_WAITCONNECT);
+          }
         }
       }
 
@@ -917,6 +927,16 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
     }
     break;
 
+    case CURLM_STATE_WAITPROXYCONNECT:
+      /* this is HTTP-specific, but sending CONNECT to a proxy is HTTP... */
+      easy->result = Curl_http_connect(easy->easy_conn, &protocol_connect);
+
+      if(CURLE_OK == easy->result) {
+        if (!easy->easy_conn->bits.tunnel_connecting)
+          multistate(easy, CURLM_STATE_WAITCONNECT);
+      }
+      break;
+
     case CURLM_STATE_WAITCONNECT:
       /* awaiting a completion of an asynch connect */
       easy->result = Curl_is_connected(easy->easy_conn,

lib/urldata.h

@@ -470,6 +470,8 @@ struct ConnectBits {
                          This is implicit when SSL-protocols are used through
                          proxies, but can also be enabled explicitly by
                          apps */
+  bool tunnel_connecting; /* TRUE while we're still waiting for a proxy CONNECT
+			   */
   bool authneg;       /* TRUE when the auth phase has started, which means
                          that we are creating a request with an auth header,
                          but it is not the final request in the auth