David Phillips' FD_SETSIZE fix

CHANGES

@@ -6,6 +6,10 @@
 
                                   Changelog
 
+Daniel (18 November 2004)
+- David Phillips fixed libcurl to not crash anymore when more than FD_SETSIZE
+  file descriptors are in use. Test case 518 added to verify.
+
 Daniel (15 November 2004)
 - To test my fix for the CURLINFO_REDIRECT_TIME bug, I added time_redirect and
   num_redirects support to the -w writeout option for the command line tool.
@@ -178,7 +182,7 @@ Daniel (11 October 2004)
   send() on other systems. Alan Pinstein verified the fix.
 
 Daniel (10 October 2004)
-- Systems with 64bit longs no longeruse strtoll() or our strtoll- replacement
+- Systems with 64bit longs no longer use strtoll() or our strtoll- replacement
   to parse 64 bit numbers. strtol() works fine. Added a configure check to
   detect if [constant]LL works and if so, use that in the strtoll replacement
   code to work around compiler warnings reported by Andy Cedilnik.

RELEASE-NOTES

@@ -19,6 +19,7 @@ This release includes the following changes:
 
 This release includes the following bugfixes:
 
+ o now gracefully bails out when exceeding FD_SETSIZE file descriptors
  o CURLINFO_REDIRECT_TIME works
  o building with gssapi libs and hdeaders in the default dirs
  o curl_getdate() parsing of dates later than year 2037 with 32 bit time_t
@@ -43,6 +44,7 @@ advice from friends like these:
 
  Peter Wullinger, Guillaume Arluison, Alexander Krasnostavsky, Mohun Biswas,
  Tomas Pospisek, Gisle Vanem, Dan Fandrich, Paul Nolan, Andres Garcia,
- Tim Sneddon, Ian Gulliver, Jean-Philippe Barrette-LaPierre, Jeff Phillips
+ Tim Sneddon, Ian Gulliver, Jean-Philippe Barrette-LaPierre, Jeff Phillips,
+ Wojciech Zwiefka, David Phillips
 
         Thanks! (and sorry if I forgot to mention someone)

lib/Makefile.inc

@@ -7,7 +7,8 @@ CSOURCES = file.c timeval.c base64.c hostip.c progress.c formdata.c	\
   memdebug.c http_chunks.c strtok.c connect.c llist.c hash.c multi.c	\
   content_encoding.c share.c http_digest.c md5.c http_negotiate.c	\
   http_ntlm.c inet_pton.c strtoofft.c strerror.c hostares.c hostasyn.c	\
-  hostip4.c hostip6.c hostsyn.c hostthre.c inet_ntop.c parsedate.c
+  hostip4.c hostip6.c hostsyn.c hostthre.c inet_ntop.c parsedate.c	\
+  select.c
 
 HHEADERS = arpa_telnet.h netrc.h file.h timeval.h base64.h hostip.h	\
   progress.h formdata.h cookie.h http.h sendf.h ftp.h url.h dict.h	\
@@ -16,4 +17,4 @@ HHEADERS = arpa_telnet.h netrc.h file.h timeval.h base64.h hostip.h	\
   http_chunks.h strtok.h connect.h llist.h hash.h content_encoding.h	\
   share.h md5.h http_digest.h http_negotiate.h http_ntlm.h ca-bundle.h	\
   inet_pton.h strtoofft.h strerror.h inet_ntop.h curlx.h memory.h	\
-  setup.h transfer.h
+  setup.h transfer.h select.h

lib/connect.c

@@ -97,6 +97,7 @@
 #include "strerror.h"
 #include "connect.h"
 #include "memory.h"
+#include "select.h"
 
 /* The last #include file should be: */
 #include "memdebug.h"
@@ -202,9 +203,6 @@ static
 int waitconnect(curl_socket_t sockfd, /* socket */
                 long timeout_msec)
 {
-  fd_set fd;
-  fd_set errfd;
-  struct timeval interval;
   int rc;
 #ifdef mpeix
   /* Call this function once now, and ignore the results. We do this to
@@ -214,18 +212,7 @@ int waitconnect(curl_socket_t sockfd, /* socket */
 #endif
 
   /* now select() until we get connect or timeout */
-  FD_ZERO(&fd);
+  rc = Curl_select(CURL_SOCKET_BAD, sockfd, timeout_msec);
-  FD_SET(sockfd, &fd);
-
-  FD_ZERO(&errfd);
-  FD_SET(sockfd, &errfd);
-
-  interval.tv_sec = (int)(timeout_msec/1000);
-  timeout_msec -= interval.tv_sec*1000;
-
-  interval.tv_usec = timeout_msec*1000;
-
-  rc = select(sockfd+1, NULL, &fd, &errfd, &interval);
   if(-1 == rc)
     /* error, no connect here, try next */
     return WAITCONN_SELECT_ERROR;
@@ -234,7 +221,7 @@ int waitconnect(curl_socket_t sockfd, /* socket */
     /* timeout, no connect today */
     return WAITCONN_TIMEOUT;
 
-  if(FD_ISSET(sockfd, &errfd))
+  if(rc & CSELECT_ERR)
     /* error condition caught */
     return WAITCONN_FDSET_ERROR;
 

lib/ftp.c

@@ -34,9 +34,6 @@
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
 #endif
-#ifdef HAVE_SYS_SELECT_H
-#include <sys/select.h>
-#endif
 
 #if defined(WIN32) && !defined(__GNUC__) || defined(__MINGW32__)
 
@@ -96,6 +93,7 @@
 #include "strerror.h"
 #include "memory.h"
 #include "inet_ntop.h"
+#include "select.h"
 
 #if defined(HAVE_INET_NTOA_R) && !defined(HAVE_INET_NTOA_R_DECL)
 #include "inet_ntoa_r.h"
@@ -162,8 +160,7 @@ static void freedirs(struct FTP *ftp)
  */
 static CURLcode AllowServerConnect(struct connectdata *conn)
 {
-  fd_set rdset;
+  int timeout_ms;
-  struct timeval dt;
   struct SessionHandle *data = conn->data;
   curl_socket_t sock = conn->sock[SECONDARYSOCKET];
   struct timeval now = Curl_tvnow();
@@ -171,10 +168,6 @@ static CURLcode AllowServerConnect(struct connectdata *conn)
   long timeout = data->set.connecttimeout?data->set.connecttimeout:
     (data->set.timeout?data->set.timeout: 0);
 
-  FD_ZERO(&rdset);
-
-  FD_SET(sock, &rdset);
-
   if(timeout) {
     timeout -= timespent;
     if(timeout<=0) {
@@ -184,10 +177,9 @@ static CURLcode AllowServerConnect(struct connectdata *conn)
   }
 
   /* we give the server 60 seconds to connect to us, or a custom timeout */
-  dt.tv_sec = (int)(timeout?timeout:60);
+  timeout_ms = (timeout?timeout:60) * 1000;
-  dt.tv_usec = 0;
 
-  switch (select(sock+1, &rdset, NULL, NULL, &dt)) {
+  switch (Curl_select(sock, CURL_SOCKET_BAD, timeout_ms)) {
   case -1: /* error */
     /* let's die here */
     failf(data, "Error while waiting for server connect");
@@ -250,9 +242,7 @@ CURLcode Curl_GetFTPResponse(ssize_t *nreadp, /* return number of bytes read */
   ssize_t gotbytes;
   char *ptr;
   long timeout;              /* timeout in seconds */
-  struct timeval interval;
+  int interval_ms;
-  fd_set rkeepfd;
-  fd_set readfd;
   struct SessionHandle *data = conn->data;
   char *line_start;
   int code=0; /* default ftp "error code" to return */
@@ -264,13 +254,6 @@ CURLcode Curl_GetFTPResponse(ssize_t *nreadp, /* return number of bytes read */
   if (ftpcode)
     *ftpcode = 0; /* 0 for errors */
 
-  FD_ZERO (&readfd);            /* clear it */
-  FD_SET (sockfd, &readfd);     /* read socket */
-
-  /* get this in a backup variable to be able to restore it on each lap in the
-     select() loop */
-  rkeepfd = readfd;
-
   ptr=buf;
   line_start = buf;
 
@@ -304,11 +287,9 @@ CURLcode Curl_GetFTPResponse(ssize_t *nreadp, /* return number of bytes read */
     }
 
     if(!ftp->cache) {
-      readfd = rkeepfd;            /* set every lap */
+      interval_ms = 1 * 1000;  /* use 1 second timeout intervals */
-      interval.tv_sec = 1; /* use 1 second timeout intervals */
-      interval.tv_usec = 0;
 
-      switch (select (sockfd+1, &readfd, NULL, NULL, &interval)) {
+      switch (Curl_select(sockfd, CURL_SOCKET_BAD, interval_ms)) {
       case -1: /* select() error, stop reading */
         result = CURLE_RECV_ERROR;
         failf(data, "FTP response aborted due to select() error: %d", errno);

lib/http.c

@@ -74,10 +74,6 @@
 #include <sys/param.h>
 #endif
 
-#ifdef HAVE_SYS_SELECT_H
-#include <sys/select.h>
-#endif
-
 #endif
 
 #include "urldata.h"
@@ -98,6 +94,7 @@
 #include "hostip.h"
 #include "http.h"
 #include "memory.h"
+#include "select.h"
 
 #define _MPRINTF_REPLACE /* use our functions only */
 #include <curl/mprintf.h>
@@ -935,9 +932,7 @@ CURLcode Curl_ConnectHTTPProxyTunnel(struct connectdata *conn,
   ssize_t gotbytes;
   char *ptr;
   long timeout = 3600; /* default timeout in seconds */
-  struct timeval interval;
+  int interval_ms;
-  fd_set rkeepfd;
-  fd_set readfd;
   char *line_start;
   char *host_port;
   curl_socket_t tunnelsocket = conn->sock[sockindex];
@@ -985,13 +980,6 @@ CURLcode Curl_ConnectHTTPProxyTunnel(struct connectdata *conn,
     if(result)
       return result;
 
-    FD_ZERO (&readfd);          /* clear it */
-    FD_SET (tunnelsocket, &readfd);     /* read socket */
-
-    /* get this in a backup variable to be able to restore it on each lap in
-       the select() loop */
-    rkeepfd = readfd;
-
     ptr=data->state.buffer;
     line_start = ptr;
 
@@ -1000,9 +988,7 @@ CURLcode Curl_ConnectHTTPProxyTunnel(struct connectdata *conn,
     keepon=TRUE;
 
     while((nread<BUFSIZE) && (keepon && !error)) {
-      readfd = rkeepfd;     /* set every lap */
+      interval_ms = 1;  /* timeout each second and check the timeout */
-      interval.tv_sec = 1;  /* timeout each second and check the timeout */
-      interval.tv_usec = 0;
 
       if(data->set.timeout) {
         /* if timeout is requested, find out how much remaining time we have */
@@ -1015,7 +1001,7 @@ CURLcode Curl_ConnectHTTPProxyTunnel(struct connectdata *conn,
         }
       }
 
-      switch (select (tunnelsocket+1, &readfd, NULL, NULL, &interval)) {
+      switch (Curl_select(tunnelsocket, CURL_SOCKET_BAD, interval_ms)) {
       case -1: /* select() error, stop reading */
         error = SELECT_ERROR;
         failf(data, "Proxy CONNECT aborted due to select() error");

lib/select.c

@@ -0,0 +1,231 @@
+/***************************************************************************
+ *                                  _   _ ____  _
+ *  Project                     ___| | | |  _ \| |
+ *                             / __| | | | |_) | |
+ *                            | (__| |_| |  _ <| |___
+ *                             \___|\___/|_| \_\_____|
+ *
+ * Copyright (C) 1998 - 2004, Daniel Stenberg, <daniel@haxx.se>, et al.
+ *
+ * This software is licensed as described in the file COPYING, which
+ * you should have received as part of this distribution. The terms
+ * are also available at http://curl.haxx.se/docs/copyright.html.
+ *
+ * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+ * copies of the Software, and permit persons to whom the Software is
+ * furnished to do so, under the terms of the COPYING file.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ * $Id$
+ ***************************************************************************/
+
+#include "setup.h"
+
+#ifdef HAVE_SYS_SELECT_H
+#include <sys/select.h>
+#endif
+
+#ifndef HAVE_SELECT
+#error "We can't compile without select() support!"
+#endif
+
+#include "select.h"
+
+/*
+ * This is an internal function used for waiting for read or write
+ * events on single file descriptors.  It attempts to replace select()
+ * in order to avoid limits with FD_SETSIZE.
+ *
+ * Return values:
+ *   -1 = system call error
+ *    0 = timeout
+ *    CSELECT_IN | CSELECT_OUT | CSELECT_ERR
+ */
+int Curl_select(int readfd, int writefd, int timeout_ms)
+{
+#ifdef HAVE_POLL_FINE
+  struct pollfd pfd[2];
+  int num;
+  int r;
+  int ret;
+
+  num = 0;
+  if (readfd != CURL_SOCKET_BAD) {
+    pfd[num].fd = readfd;
+    pfd[num].events = POLLIN;
+    num++;
+  }
+  if (writefd != CURL_SOCKET_BAD) {
+    pfd[num].fd = writefd;
+    pfd[num].events = POLLOUT;
+    num++;
+  }
+
+  r = poll(pfd, num, timeout_ms);
+
+  if (r < 0)
+    return -1;
+  if (r == 0)
+    return 0;
+
+  ret = 0;
+  num = 0;
+  if (readfd != CURL_SOCKET_BAD) {
+    if (pfd[num].revents & POLLIN)
+      ret |= CSELECT_IN;
+    if (pfd[num].revents & POLLERR)
+      ret |= CSELECT_ERR;
+    num++;
+  }
+  if (writefd != CURL_SOCKET_BAD) {
+    if (pfd[num].revents & POLLOUT)
+      ret |= CSELECT_OUT;
+    if (pfd[num].revents & POLLERR)
+      ret |= CSELECT_ERR;
+  }
+
+  return ret;
+#else
+  struct timeval timeout;
+  fd_set fds_read;
+  fd_set fds_write;
+  fd_set fds_err;
+  int maxfd;
+  int r;
+  int ret;
+
+  timeout.tv_sec = timeout_ms / 1000;
+  timeout.tv_usec = (timeout_ms % 1000) * 1000;
+
+  FD_ZERO(&fds_err);
+  maxfd = -1;
+
+  FD_ZERO(&fds_read);
+  if (readfd != CURL_SOCKET_BAD) {
+    if ((readfd < 0) || (readfd >= FD_SETSIZE)) {
+      errno = EINVAL;
+      return -1;
+    }
+    FD_SET(readfd, &fds_read);
+    FD_SET(readfd, &fds_err);
+    maxfd = readfd;
+  }
+
+  FD_ZERO(&fds_write);
+  if (writefd != CURL_SOCKET_BAD) {
+    if ((writefd < 0) || (writefd >= FD_SETSIZE)) {
+      errno = EINVAL;
+      return -1;
+    }
+    FD_SET(writefd, &fds_write);
+    FD_SET(writefd, &fds_err);
+    if (writefd > maxfd)
+      maxfd = writefd;
+  }
+
+  r = select(maxfd + 1, &fds_read, &fds_write, &fds_err, &timeout);
+
+  if (r < 0)
+    return -1;
+  if (r == 0)
+    return 0;
+
+  ret = 0;
+  if (readfd != CURL_SOCKET_BAD) {
+    if (FD_ISSET(readfd, &fds_read))
+      ret |= CSELECT_IN;
+    if (FD_ISSET(readfd, &fds_err))
+      ret |= CSELECT_ERR;
+  }
+  if (writefd != CURL_SOCKET_BAD) {
+    if (FD_ISSET(writefd, &fds_write))
+      ret |= CSELECT_OUT;
+    if (FD_ISSET(writefd, &fds_err))
+      ret |= CSELECT_ERR;
+  }
+
+  return ret;
+#endif
+}
+
+/*
+ * This is a wrapper around poll().  If poll() does not exist, then
+ * select() is used instead.  An error is returned if select() is
+ * being used and a file descriptor too large for FD_SETSIZE.
+ *
+ * Return values:
+ *   -1 = system call error or fd >= FD_SETSIZE
+ *    0 = timeout
+ *    1 = number of structures with non zero revent fields
+ */
+int Curl_poll(struct pollfd ufds[], unsigned int nfds, int timeout_ms)
+{
+#ifdef HAVE_POLL_FINE
+    return poll(ufds, nfds, timeout_ms);
+#else
+  struct timeval timeout;
+  struct timeval *ptimeout;
+  fd_set fds_read;
+  fd_set fds_write;
+  fd_set fds_err;
+  int maxfd;
+  int r;
+  unsigned int i;
+
+  FD_ZERO(&fds_read);
+  FD_ZERO(&fds_write);
+  FD_ZERO(&fds_err);
+  maxfd = -1;
+
+  for (i = 0; i < nfds; i++) {
+    if (ufds[i].fd < 0)
+      continue;
+    if (ufds[i].fd >= FD_SETSIZE) {
+      errno = EINVAL;
+      return -1;
+    }
+    if (ufds[i].fd > maxfd)
+      maxfd = ufds[i].fd;
+    if (ufds[i].events & POLLIN)
+      FD_SET(ufds[i].fd, &fds_read);
+    if (ufds[i].events & POLLOUT)
+      FD_SET(ufds[i].fd, &fds_write);
+    if (ufds[i].events & POLLERR)
+      FD_SET(ufds[i].fd, &fds_err);
+  }
+
+  if (timeout_ms < 0) {
+    ptimeout = NULL;      /* wait forever */
+  } else {
+    timeout.tv_sec = timeout_ms / 1000;
+    timeout.tv_usec = (timeout_ms % 1000) * 1000;
+    ptimeout = &timeout;
+  }
+
+  r = select(maxfd + 1, &fds_read, &fds_write, &fds_err, ptimeout);
+
+  if (r < 0)
+    return -1;
+  if (r == 0)
+    return 0;
+
+  r = 0;
+  for (i = 0; i < nfds; i++) {
+    ufds[i].revents = 0;
+    if (ufds[i].fd < 0)
+      continue;
+    if (FD_ISSET(ufds[i].fd, &fds_read))
+      ufds[i].revents |= POLLIN;
+    if (FD_ISSET(ufds[i].fd, &fds_write))
+      ufds[i].revents |= POLLOUT;
+    if (FD_ISSET(ufds[i].fd, &fds_err))
+      ufds[i].revents |= POLLERR;
+    if (ufds[i].revents != 0)
+      r++;
+  }
+
+  return r;
+#endif
+}

lib/select.h

@@ -0,0 +1,57 @@
+#ifndef __SELECT_H
+#define __SELECT_H
+/***************************************************************************
+ *                                  _   _ ____  _
+ *  Project                     ___| | | |  _ \| |
+ *                             / __| | | | |_) | |
+ *                            | (__| |_| |  _ <| |___
+ *                             \___|\___/|_| \_\_____|
+ *
+ * Copyright (C) 1998 - 2004, Daniel Stenberg, <daniel@haxx.se>, et al.
+ *
+ * This software is licensed as described in the file COPYING, which
+ * you should have received as part of this distribution. The terms
+ * are also available at http://curl.haxx.se/docs/copyright.html.
+ *
+ * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+ * copies of the Software, and permit persons to whom the Software is
+ * furnished to do so, under the terms of the COPYING file.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ * $Id$
+ ***************************************************************************/
+
+
+#ifdef HAVE_SYS_POLL_H
+#include <sys/poll.h>
+#else
+
+#define POLLIN      0x01
+#define POLLPRI     0x02
+#define POLLOUT     0x04
+#define POLLERR     0x08
+#define POLLHUP     0x10
+#define POLLNVAL    0x20
+
+struct pollfd
+{
+    int     fd;
+    short   events;
+    short   revents;
+};
+
+#endif
+
+
+#define CSELECT_IN   0x01
+#define CSELECT_OUT  0x02
+#define CSELECT_ERR  0x04
+
+int Curl_select(int readfd, int writefd, int timeout_ms);
+
+int Curl_poll(struct pollfd ufds[], unsigned int nfds, int timeout_ms);
+
+
+#endif

lib/ssluse.c

@@ -46,6 +46,7 @@
 #include "ssluse.h"
 #include "connect.h" /* Curl_ourerrno() proto */
 #include "strequal.h"
+#include "select.h"
 
 #define _MPRINTF_REPLACE /* use the internal *printf() functions */
 #include <curl/mprintf.h>
@@ -1260,9 +1261,8 @@ Curl_SSLConnect(struct connectdata *conn,
   SSL_set_fd(connssl->handle, sockfd);
 
   while(1) {
-    fd_set writefd;
+    int writefd;
-    fd_set readfd;
+    int readfd;
-    struct timeval interval;
     long timeout_ms;
 
     /* Find out if any timeout is set. If not, use 300 seconds.
@@ -1296,8 +1296,8 @@ Curl_SSLConnect(struct connectdata *conn,
       timeout_ms= DEFAULT_CONNECT_TIMEOUT;
 
 
-    FD_ZERO(&writefd);
+    readfd = CURL_SOCKET_BAD;
-    FD_ZERO(&readfd);
+    writefd = CURL_SOCKET_BAD;
 
     err = SSL_connect(connssl->handle);
 
@@ -1308,9 +1308,9 @@ Curl_SSLConnect(struct connectdata *conn,
       int detail = SSL_get_error(connssl->handle, err);
 
       if(SSL_ERROR_WANT_READ == detail)
-        FD_SET(sockfd, &readfd);
+        readfd = sockfd;
       else if(SSL_ERROR_WANT_WRITE == detail)
-        FD_SET(sockfd, &writefd);
+        writefd = sockfd;
       else {
         /* untreated error */
         unsigned long errdetail;
@@ -1373,13 +1373,8 @@ Curl_SSLConnect(struct connectdata *conn,
       /* we have been connected fine, get out of the connect loop */
       break;
 
-    interval.tv_sec = (int)(timeout_ms/1000);
-    timeout_ms -= interval.tv_sec*1000;
-
-    interval.tv_usec = timeout_ms*1000;
-
     while(1) {
-      what = select(sockfd+1, &readfd, &writefd, NULL, &interval);
+      what = Curl_select(readfd, writefd, timeout_ms);
       if(what > 0)
         /* reabable or writable, go loop in the outer loop */
         break;

lib/telnet.c

@@ -64,10 +64,6 @@
 #include <sys/param.h>
 #endif
 
-#ifdef HAVE_SYS_SELECT_H
-#include <sys/select.h>
-#endif
-
 
 #endif
 
@@ -85,6 +81,7 @@
 
 #include "arpa_telnet.h"
 #include "memory.h"
+#include "select.h"
 
 /* The last #include file should be: */
 #include "memdebug.h"
@@ -1088,8 +1085,8 @@ CURLcode Curl_telnet(struct connectdata *conn)
   DWORD waitret;
   DWORD readfile_read;
 #else
-  fd_set readfd;
+  int interval_ms;
-  fd_set keepfd;
+  struct pollfd pfd[2];
 #endif
   ssize_t nread;
   bool keepon = TRUE;
@@ -1308,27 +1305,21 @@ CURLcode Curl_telnet(struct connectdata *conn)
   if (!FreeLibrary(wsock2))
     infof(data,"FreeLibrary(wsock2) failed (%d)",GetLastError());
 #else
-  FD_ZERO (&readfd);            /* clear it */
+  pfd[0].fd = sockfd;
-  FD_SET (sockfd, &readfd);
+  pfd[0].events = POLLIN;
-  FD_SET (0, &readfd);
+  pfd[1].fd = 0;
-
+  pfd[1].events = POLLIN;
-  keepfd = readfd;
+  interval_ms = 1 * 1000;
 
   while (keepon) {
-    struct timeval interval;
+    switch (Curl_poll(pfd, 2, interval_ms)) {
-
-    readfd = keepfd;            /* set this every lap in the loop */
-    interval.tv_sec = 1;
-    interval.tv_usec = 0;
-
-    switch (select (sockfd + 1, &readfd, NULL, NULL, &interval)) {
     case -1:                    /* error, stop reading */
       keepon = FALSE;
       continue;
     case 0:                     /* timeout */
       break;
     default:                    /* read! */
-      if(FD_ISSET(0, &readfd)) { /* read from stdin */
+      if(pfd[1].revents & POLLIN) { /* read from stdin */
         unsigned char outbuf[2];
         int out_count = 0;
         ssize_t bytes_written;
@@ -1347,7 +1338,7 @@ CURLcode Curl_telnet(struct connectdata *conn)
         }
       }
 
-      if(FD_ISSET(sockfd, &readfd)) {
+      if(pfd[0].revents & POLLIN) {
         /* This OUGHT to check the return code... */
         (void)Curl_read(conn, sockfd, buf, BUFSIZE - 1, &nread);
 

lib/transfer.c

@@ -75,9 +75,6 @@
 #include <sys/select.h>
 #endif
 
-#ifndef HAVE_SELECT
-#error "We can't compile without select() support!"
-#endif
 #ifndef HAVE_SOCKET
 #error "We can't compile without socket() support!"
 #endif
@@ -103,6 +100,7 @@
 #include "http_negotiate.h"
 #include "share.h"
 #include "memory.h"
+#include "select.h"
 
 #define _MPRINTF_REPLACE /* use our functions only */
 #include <curl/mprintf.h>
@@ -118,10 +116,6 @@ enum {
   KEEP_WRITE
 };
 
-/* We keep this static and global since this is read-only and NEVER
-   changed. It should just remain a blanked-out timeout value. */
-static struct timeval notimeout={0,0};
-
 /*
  * This function will call the read callback to fill our buffer with data
  * to upload.
@@ -213,43 +207,28 @@ CURLcode Curl_readwrite(struct connectdata *conn,
   ssize_t nread; /* number of bytes read */
   int didwhat=0;
 
-  /* These two are used only if no other select() or _fdset() have been
+  int fd_read;
-     invoked before this. This typicly happens if you use the multi interface
+  int fd_write;
-     and call curl_multi_perform() without calling curl_multi_fdset()
+  int select_res;
-     first. */
-  fd_set extrareadfd;
-  fd_set extrawritefd;
 
-  fd_set *readfdp = k->readfdp;
-  fd_set *writefdp = k->writefdp;
   curl_off_t contentlength;
 
-  if((k->keepon & KEEP_READ) && !readfdp) {
+  if(k->keepon & KEEP_READ)
-    /* reading is requested, but no socket descriptor pointer was set */
+    fd_read = conn->sockfd;
-    FD_ZERO(&extrareadfd);
+  else
-    FD_SET(conn->sockfd, &extrareadfd);
+    fd_read = CURL_SOCKET_BAD;
-    readfdp = &extrareadfd;
 
-    /* no write, no exceptions, no timeout */
+  if(k->keepon & KEEP_WRITE)
-    select(conn->sockfd+1, readfdp, NULL, NULL, &notimeout);
+    fd_write = conn->writesockfd;
-  }
+  else
-  if((k->keepon & KEEP_WRITE) && !writefdp) {
+    fd_write = CURL_SOCKET_BAD;
-    /* writing is requested, but no socket descriptor pointer was set */
-    FD_ZERO(&extrawritefd);
-    FD_SET(conn->writesockfd, &extrawritefd);
-    writefdp = &extrawritefd;
 
-    /* no read, no exceptions, no timeout */
+  select_res = Curl_select(fd_read, fd_write, 0);
-    select(conn->writesockfd+1, NULL, writefdp, NULL, &notimeout);
-  }
 
   do {
     /* If we still have reading to do, we check if we have a readable
-       socket. Sometimes the reafdp is NULL, if no fd_set was done using
+       socket. */
-       the multi interface and then we can do nothing but to attempt a
+    if((k->keepon & KEEP_READ) && (select_res & CSELECT_IN)) {
-       read to be sure. */
-    if((k->keepon & KEEP_READ) &&
-       (!readfdp || FD_ISSET(conn->sockfd, readfdp))) {
 
       bool is_empty_data = FALSE;
 
@@ -291,7 +270,6 @@ CURLcode Curl_readwrite(struct connectdata *conn,
            we bail out from this! */
         else if (0 >= nread) {
           k->keepon &= ~KEEP_READ;
-          FD_ZERO(&k->rkeepfd);
           break;
         }
 
@@ -436,7 +414,6 @@ CURLcode Curl_readwrite(struct connectdata *conn,
                 if (k->write_after_100_header) {
 
                   k->write_after_100_header = FALSE;
-                  FD_SET (conn->writesockfd, &k->writefd); /* write */
                   k->keepon |= KEEP_WRITE;
                   k->wkeepfd = k->writefd;
                 }
@@ -453,7 +430,6 @@ CURLcode Curl_readwrite(struct connectdata *conn,
                  */
                 k->write_after_100_header = FALSE;
                 k->keepon &= ~KEEP_WRITE;
-                FD_ZERO(&k->wkeepfd);
               }
 
 #ifndef CURL_DISABLE_HTTP
@@ -550,7 +526,6 @@ CURLcode Curl_readwrite(struct connectdata *conn,
                 if(stop_reading) {
                   /* we make sure that this socket isn't read more now */
                   k->keepon &= ~KEEP_READ;
-                  FD_ZERO(&k->rkeepfd);
                 }
 
                 break;          /* exit header line loop */
@@ -951,7 +926,6 @@ CURLcode Curl_readwrite(struct connectdata *conn,
                   /* Abort after the headers if "follow Location" is set
                      and we're set to close anyway. */
                   k->keepon &= ~KEEP_READ;
-                  FD_ZERO(&k->rkeepfd);
                   *done = TRUE;
                   return CURLE_OK;
                 }
@@ -1040,7 +1014,6 @@ CURLcode Curl_readwrite(struct connectdata *conn,
             else if(CHUNKE_STOP == res) {
               /* we're done reading chunks! */
               k->keepon &= ~KEEP_READ; /* read no more */
-              FD_ZERO(&k->rkeepfd);
 
               /* There are now possibly N number of bytes at the end of the
                  str buffer that weren't written to the client, but we don't
@@ -1057,7 +1030,6 @@ CURLcode Curl_readwrite(struct connectdata *conn,
               nread = 0;
 
             k->keepon &= ~KEEP_READ; /* we're done reading */
-            FD_ZERO(&k->rkeepfd);
           }
 
           k->bytecount += nread;
@@ -1125,7 +1097,6 @@ CURLcode Curl_readwrite(struct connectdata *conn,
           /* if we received nothing, the server closed the connection and we
              are done */
           k->keepon &= ~KEEP_READ;
-          FD_ZERO(&k->rkeepfd);
         }
 
       } while(0);
@@ -1133,11 +1104,8 @@ CURLcode Curl_readwrite(struct connectdata *conn,
     } /* if( read from socket ) */
 
     /* If we still have writing to do, we check if we have a writable
-       socket. Sometimes the writefdp is NULL, if no fd_set was done using
+       socket. */
-       the multi interface and then we can do nothing but to attempt a
+    if((k->keepon & KEEP_WRITE) && (select_res & CSELECT_OUT)) {
-       write to be sure. */
-    if((k->keepon & KEEP_WRITE) &&
-       (!writefdp || FD_ISSET(conn->writesockfd, writefdp)) ) {
       /* write */
 
       int i, si;
@@ -1173,7 +1141,6 @@ CURLcode Curl_readwrite(struct connectdata *conn,
                  go into the Expect: 100 state and await such a header */
               k->wait100_after_headers = FALSE; /* headers sent */
               k->write_after_100_header = TRUE; /* wait for the header */
-              FD_ZERO (&k->writefd);            /* clear it */
               k->wkeepfd = k->writefd;          /* set the keeper variable */
               k->keepon &= ~KEEP_WRITE;         /* disable writing */
               k->start100 = Curl_tvnow();       /* timeout count starts now */
@@ -1195,7 +1162,6 @@ CURLcode Curl_readwrite(struct connectdata *conn,
           if (nread<=0) {
             /* done */
             k->keepon &= ~KEEP_WRITE; /* we're done writing */
-            FD_ZERO(&k->wkeepfd);
             writedone = TRUE;
             break;
           }
@@ -1271,7 +1237,6 @@ CURLcode Curl_readwrite(struct connectdata *conn,
           if(k->upload_done) {
             /* switch off writing, we're done! */
             k->keepon &= ~KEEP_WRITE; /* we're done writing */
-            FD_ZERO(&k->wkeepfd);
             writedone = TRUE;
           }
         }
@@ -1313,7 +1278,6 @@ CURLcode Curl_readwrite(struct connectdata *conn,
       if(ms > CURL_TIMEOUT_EXPECT_100) {
         /* we've waited long enough, continue anyway */
         k->write_after_100_header = FALSE;
-        FD_SET (conn->writesockfd, &k->writefd); /* write socket */
         k->keepon |= KEEP_WRITE;
         k->wkeepfd = k->writefd;
       }
@@ -1405,13 +1369,10 @@ CURLcode Curl_readwrite_init(struct connectdata *conn)
   /* we want header and/or body, if neither then don't do this! */
   if(conn->bits.getheader || !conn->bits.no_body) {
 
-    FD_ZERO (&k->readfd);               /* clear it */
+    if(conn->sockfd != CURL_SOCKET_BAD) { 
-    if(conn->sockfd != CURL_SOCKET_BAD) {
-      FD_SET (conn->sockfd, &k->readfd); /* read socket */
       k->keepon |= KEEP_READ;
     }
 
-    FD_ZERO (&k->writefd);              /* clear it */
     if(conn->writesockfd != CURL_SOCKET_BAD) {
       /* HTTP 1.1 magic:
 
@@ -1433,7 +1394,6 @@ CURLcode Curl_readwrite_init(struct connectdata *conn)
           /* when we've sent off the rest of the headers, we must await a
              100-continue */
           k->wait100_after_headers = TRUE;
-        FD_SET (conn->writesockfd, &k->writefd); /* write socket */
         k->keepon |= KEEP_WRITE;
       }
     }
@@ -1521,13 +1481,23 @@ Transfer(struct connectdata *conn)
   k->readfdp = &k->readfd;   /* store the address of the set */
 
   while (!done) {
-    struct timeval interval;
+    int fd_read;
-    k->readfd = k->rkeepfd;  /* set these every lap in the loop */
+    int fd_write;
-    k->writefd = k->wkeepfd;
+    int interval_ms;
-    interval.tv_sec = 1;
-    interval.tv_usec = 0;
 
-    switch (select (k->maxfd, k->readfdp, k->writefdp, NULL, &interval)) {
+    interval_ms = 1 * 1000;
+
+    if(k->keepon & KEEP_READ)
+      fd_read = conn->sockfd;
+    else
+      fd_read = CURL_SOCKET_BAD;
+
+    if(k->keepon & KEEP_WRITE)
+      fd_write = conn->writesockfd;
+    else
+      fd_write = CURL_SOCKET_BAD;
+
+    switch (Curl_select(fd_read, fd_write, interval_ms)) {
     case -1: /* select() error, stop reading */
 #ifdef EINTR
       /* The EINTR is not serious, and it seems you might get this more

lib/url.c

@@ -64,10 +64,6 @@
 #include <sys/param.h>
 #endif
 
-#ifdef HAVE_SYS_SELECT_H
-#include <sys/select.h>
-#endif
-
 #ifdef VMS
 #include <in.h>
 #include <inet.h>
@@ -77,9 +73,6 @@
 #include <setjmp.h>
 #endif
 
-#ifndef HAVE_SELECT
-#error "We can't compile without select() support!"
-#endif
 #ifndef HAVE_SOCKET
 #error "We can't compile without socket() support!"
 #endif
@@ -127,6 +120,7 @@ void idn_free (void *ptr); /* prototype from idn-free.h, not provided by
 #include "content_encoding.h"
 #include "http_digest.h"
 #include "http_negotiate.h"
+#include "select.h"
 
 /* And now for the protocols */
 #include "ftp.h"
@@ -1552,16 +1546,8 @@ static bool SocketIsDead(curl_socket_t sock)
 {
   int sval;
   bool ret_val = TRUE;
-  fd_set check_set;
-  struct timeval to;
 
-  FD_ZERO(&check_set);
+  sval = Curl_select(sock, CURL_SOCKET_BAD, 0);
-  FD_SET(sock, &check_set);
-
-  to.tv_sec = 0;
-  to.tv_usec = 0;
-
-  sval = select(sock + 1, &check_set, 0, 0, &to);
   if(sval == 0)
     /* timeout */
     ret_val = FALSE;

tests/data/Makefile.am

@@ -28,7 +28,7 @@ EXTRA_DIST = test1 test108 test117 test127 test20 test27 test34 test46	\
  test513 test514 test178 test179 test180 test181 test182 test183	\
  test184 test185 test186 test187 test188 test189 test191 test192	\
  test193 test194 test195 test196 test197 test198 test515 test516	\
- test517
+ test517 test518
 
 # The following tests have been removed from the dist since they no longer
 # work. We need to fix the test suite's FTPS server first, then bring them

tests/data/test518

@@ -0,0 +1,45 @@
+#
+# Server-side
+<reply name="1">
+<data>
+HTTP/1.1 200 OK
+Date: Thu, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 6
+Connection: close
+Content-Type: text/html
+Funny-head: yesyes
+
+<foo>
+</data>
+</reply>
+
+# Client-side
+<client>
+<server>
+http
+</server>
+# tool is what to use instead of 'curl'
+<tool>
+lib518
+</tool>
+
+ <name>
+HTTP GET with more than FD_SETSIZE descriptors open
+ </name>
+ <command>
+http://%HOSTIP:%HTTPPORT/518
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+# CURLE_FAILED_INIT (2)
+<errorcode>
+2
+</errorcode>
+</verify>

tests/libtest/Makefile.am

@@ -39,7 +39,8 @@ SUPPORTFILES = first.c test.h
 
 # These are all libcurl test programs
 noinst_PROGRAMS = lib500 lib501 lib502 lib503 lib504 lib505 lib506 lib507 \
-  lib508 lib509 lib510 lib511 lib512 lib513 lib514 lib515 lib516 lib517
+  lib508 lib509 lib510 lib511 lib512 lib513 lib514 lib515 lib516 lib517 \
+  lib518
 
 lib500_SOURCES = lib500.c $(SUPPORTFILES)
 lib500_LDADD = $(LIBDIR)/libcurl.la
@@ -112,3 +113,7 @@ lib516_DEPENDENCIES = $(LIBDIR)/libcurl.la
 lib517_SOURCES = lib517.c $(SUPPORTFILES)
 lib517_LDADD = $(LIBDIR)/libcurl.la
 lib517_DEPENDENCIES = $(LIBDIR)/libcurl.la
+
+lib518_SOURCES = lib518.c $(SUPPORTFILES)
+lib518_LDADD = $(LIBDIR)/libcurl.la
+lib518_DEPENDENCIES = $(LIBDIR)/libcurl.la

tests/libtest/lib518.c

@@ -0,0 +1,47 @@
+#include "test.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+#include <mprintf.h>
+
+#ifdef HAVE_SYS_SELECT_H
+#include <sys/select.h>
+#endif
+
+#ifndef FD_SETSIZE
+#error "this test requires FD_SETSIZE"
+#endif
+
+#define NUM_OPEN (FD_SETSIZE + 10)
+
+int test(char *URL)
+{
+  CURLcode res;
+  CURL *curl;
+  int fd[NUM_OPEN];
+  int i;
+
+  /* open a lot of file descriptors */
+  for (i = 0; i < NUM_OPEN; i++) {
+    fd[i] = open("/dev/null", O_RDONLY);
+    if (fd[i] == -1) {
+      fprintf(stderr, "open: attempt #%i: failed to open /dev/null\n", i);
+      for (i--; i >= 0; i--)
+        close(fd[i]);
+      return CURLE_FAILED_INIT;
+    }
+  }
+
+  curl = curl_easy_init();
+  curl_easy_setopt(curl, CURLOPT_URL, URL);
+  curl_easy_setopt(curl, CURLOPT_HEADER, TRUE);
+  res = curl_easy_perform(curl);
+  curl_easy_cleanup(curl);
+
+  for (i = 0; i < NUM_OPEN; i++)
+    close(fd[i]);
+
+  return (int)res;
+}