diff --git a/lib/connect.c b/lib/connect.c
index 57e203657..0aeaf7959 100644
--- a/lib/connect.c
+++ b/lib/connect.c
@@ -781,6 +781,7 @@ CURLcode Curl_connecthost(struct connectdata *conn,  /* context */
   int num_addr;
   Curl_addrinfo *ai;
   Curl_addrinfo *curr_addr;
+  int timeout_set = 0;
 
   struct timeval after;
   struct timeval before = Curl_tvnow();
@@ -788,36 +789,39 @@ CURLcode Curl_connecthost(struct connectdata *conn,  /* context */
   /*************************************************************
    * Figure out what maximum time we have left
    *************************************************************/
-  long timeout_ms= DEFAULT_CONNECT_TIMEOUT;
+  long timeout_ms;
   long timeout_per_addr;
 
   *connected = FALSE; /* default to not connected */
 
-  if(data->set.timeout || data->set.connecttimeout) {
-    long has_passed;
+  /* if a timeout is set, use the most restrictive one */
 
-    /* Evaluate in milliseconds how much time that has passed */
-    has_passed = Curl_tvdiff(Curl_tvnow(), data->progress.t_startsingle);
+  if (data->set.timeout > 0)
+    timeout_set += 1;
+  if (data->set.connecttimeout > 0)
+    timeout_set += 2;
 
-#ifndef min
-#define min(a, b)   ((a) < (b) ? (a) : (b))
-#endif
-
-    /* get the most strict timeout of the ones converted to milliseconds */
-    if(data->set.timeout && data->set.connecttimeout) {
-      if (data->set.timeout < data->set.connecttimeout)
-        timeout_ms = data->set.timeout;
-      else
-        timeout_ms = data->set.connecttimeout;
-    }
-    else if(data->set.timeout)
+  switch (timeout_set) {
+  case 1:
+    timeout_ms = data->set.timeout;
+    break;
+  case 2:
+    timeout_ms = data->set.connecttimeout;
+    break;
+  case 3:
+    if (data->set.timeout < data->set.connecttimeout)
       timeout_ms = data->set.timeout;
     else
       timeout_ms = data->set.connecttimeout;
+    break;
+  default:
+    timeout_ms = DEFAULT_CONNECT_TIMEOUT;
+    break;
+  }
 
-    /* subtract the passed time */
-    timeout_ms -= has_passed;
-
+  if (timeout_set > 0) {
+    /* if a timeout was already set, substract elapsed time */
+    timeout_ms -= Curl_tvdiff(before, data->progress.t_startsingle);
     if(timeout_ms < 0) {
       /* a precaution, no need to continue if time already is up */
       failf(data, "Connection time-out");
diff --git a/lib/ftp.c b/lib/ftp.c
index ecbfd12f0..c502baab8 100644
--- a/lib/ftp.c
+++ b/lib/ftp.c
@@ -184,27 +184,46 @@ static bool isBadFtpString(const char *string)
  */
 static CURLcode AllowServerConnect(struct connectdata *conn)
 {
-  int timeout_ms;
+  long timeout_ms;
   struct SessionHandle *data = conn->data;
   curl_socket_t sock = conn->sock[SECONDARYSOCKET];
-  struct timeval now = Curl_tvnow();
-  long timespent = Curl_tvdiff(Curl_tvnow(), now);
-  long timeout = data->set.connecttimeout?data->set.connecttimeout:
-    (data->set.timeout?data->set.timeout: 0);
+  int timeout_set = 0;
 
-  if(timeout) {
-    timeout -= timespent;
-    if(timeout<=0) {
+  /* if a timeout is set, use the most restrictive one */
+
+  if (data->set.timeout > 0)
+    timeout_set += 1;
+  if (data->set.connecttimeout > 0)
+    timeout_set += 2;
+
+  switch (timeout_set) {
+  case 1:
+    timeout_ms = data->set.timeout;
+    break;
+  case 2:
+    timeout_ms = data->set.connecttimeout;
+    break;
+  case 3:
+    if (data->set.timeout < data->set.connecttimeout)
+      timeout_ms = data->set.timeout;
+    else
+      timeout_ms = data->set.connecttimeout;
+    break;
+  default:
+    timeout_ms = 60000; /* 60 seconds default timeout */
+    break;
+  }
+
+  if (timeout_set > 0) {
+    /* if a timeout was already set, substract elapsed time */
+    timeout_ms -= Curl_tvdiff(Curl_tvnow(), conn->now);
+    if(timeout_ms < 0) {
       failf(data, "Timed out before server could connect to us");
       return CURLE_OPERATION_TIMEDOUT;
     }
   }
 
-  /* We allow the server 60 seconds to connect to us, or a custom timeout.
-     Note the typecast here. */
-  timeout_ms = (timeout?(int)timeout:60000);
-
-  switch (Curl_socket_ready(sock, CURL_SOCKET_BAD, timeout_ms)) {
+  switch (Curl_socket_ready(sock, CURL_SOCKET_BAD, (int)timeout_ms)) {
   case -1: /* error */
     /* let's die here */
     failf(data, "Error while waiting for server connect");