upkeep: add a connection upkeep API: curl_easy_conn_upkeep()

Add functionality so that protocols can do custom keepalive on their
connections, when an external API function is called.

Add docs for the new options in 7.62.0

Closes #1641

docs/libcurl/Makefile.inc

@@ -22,4 +22,4 @@ man_MANS = curl_easy_cleanup.3 curl_easy_getinfo.3 curl_easy_init.3      \
   curl_mime_data.3 curl_mime_data_cb.3 curl_mime_filedata.3              \
   curl_mime_filename.3 curl_mime_subparts.3                              \
   curl_mime_type.3 curl_mime_headers.3 curl_mime_encoder.3 libcurl-env.3 \
-  libcurl-security.3
+  libcurl-security.3 curl_easy_conn_upkeep.3

docs/libcurl/curl_easy_conn_upkeep.3

@@ -0,0 +1,77 @@
+.\" **************************************************************************
+.\" *                                  _   _ ____  _
+.\" *  Project                     ___| | | |  _ \| |
+.\" *                             / __| | | | |_) | |
+.\" *                            | (__| |_| |  _ <| |___
+.\" *                             \___|\___/|_| \_\_____|
+.\" *
+.\" * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al.
+.\" *
+.\" * This software is licensed as described in the file COPYING, which
+.\" * you should have received as part of this distribution. The terms
+.\" * are also available at https://curl.haxx.se/docs/copyright.html.
+.\" *
+.\" * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+.\" * copies of the Software, and permit persons to whom the Software is
+.\" * furnished to do so, under the terms of the COPYING file.
+.\" *
+.\" * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+.\" * KIND, either express or implied.
+.\" *
+.\" **************************************************************************
+.\"
+.TH curl_easy_conn_upkeep 3 "31 Oct 2018" "libcurl 7.62.0" "libcurl Manual"
+.SH NAME
+curl_easy_conn_upkeep - Perform any connection upkeep checks.
+.SH SYNOPSIS
+.B #include <curl/curl.h>
+
+.BI "CURLcode curl_easy_conn_upkeep(CURL *" handle ");"
+.SH DESCRIPTION
+
+Some protocols have "connection upkeep" mechanisms. These mechanisms usually
+send some traffic on existing connections in order to keep them alive; this
+can prevent connections from being closed due to overzealous firewalls, for
+example.
+
+Currently the only protocol with a connection upkeep mechanism is HTTP/2: when
+the connection upkeep interval is exceeded and \fIcurl_easy_conn_upkeep(3)\fP
+is called, an HTTP/2 PING frame is sent on the connection.
+
+This function must be explicitly called in order to perform the upkeep work.
+The connection upkeep interval is set with
+\fICURLOPT_CONN_UPKEEP_INTERVAL_MS(3)\fP.
+
+.SH AVAILABILITY
+Added in 7.62.0.
+.SH RETURN VALUE
+On success, returns \fBCURLE_OK\fP.
+
+On failure, returns the appropriate error code.
+
+.SH EXAMPLE
+.nf
+CURL *curl = curl_easy_init();
+if(curl) {
+  /* Make a connection to an HTTP/2 server. */
+  curl_easy_setopt(curl, CURLOPT_URL, "https://example.com");
+
+  /* Set the interval to 30000ms / 30s */
+  curl_easy_setopt(curl, CURLOPT_CONN_UPKEEP_INTERVAL_MS, 30000L);
+
+  curl_easy_perform(curl);
+
+  /* Perform more work here. */
+
+  /* While the connection is being held open, curl_easy_conn_upkeep() can be
+     called. If curl_easy_conn_upkeep() is called and the time since the last
+     upkeep exceeds the interval, then an HTTP/2 PING is sent. */
+  curl_easy_conn_upkeep(curl);
+
+  /* Perform more work here. */
+
+  /* always cleanup */
+  curl_easy_cleanup(curl);
+}
+
+.fi

docs/libcurl/curl_easy_setopt.3

@@ -480,6 +480,9 @@ Shuffle addresses before use. See \fICURLOPT_DNS_SHUFFLE_ADDRESSES(3)\fP
 Timeout for waiting for the server's connect back to be accepted. See \fICURLOPT_ACCEPTTIMEOUT_MS(3)\fP
 .IP CURLOPT_HAPPY_EYEBALLS_TIMEOUT_MS
 Timeout for happy eyeballs. See \fICURLOPT_HAPPY_EYEBALLS_TIMEOUT_MS(3)\fP
+.IP CURLOPT_CONN_UPKEEP_INTERVAL_MS
+Sets the interval at which connection upkeep are performed. See
+\fICURLOPT_CONN_UPKEEP_INTERVAL_MS(3)\fP
 .SH SSL and SECURITY OPTIONS
 .IP CURLOPT_SSLCERT
 Client cert. See \fICURLOPT_SSLCERT(3)\fP

docs/libcurl/opts/CURLOPT_CONN_UPKEEP_INTERVAL_MS.3

@@ -0,0 +1,73 @@
+.\" **************************************************************************
+.\" *                                  _   _ ____  _
+.\" *  Project                     ___| | | |  _ \| |
+.\" *                             / __| | | | |_) | |
+.\" *                            | (__| |_| |  _ <| |___
+.\" *                             \___|\___/|_| \_\_____|
+.\" *
+.\" * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al.
+.\" *
+.\" * This software is licensed as described in the file COPYING, which
+.\" * you should have received as part of this distribution. The terms
+.\" * are also available at https://curl.haxx.se/docs/copyright.html.
+.\" *
+.\" * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+.\" * copies of the Software, and permit persons to whom the Software is
+.\" * furnished to do so, under the terms of the COPYING file.
+.\" *
+.\" * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+.\" * KIND, either express or implied.
+.\" *
+.\" **************************************************************************
+.\"
+.TH CURLOPT_CONN_UPKEEP_INTERVAL_MS 3 "31 Oct 2018" "libcurl 7.62.0" "curl_easy_setopt options"
+.SH NAME
+CURLOPT_CONN_UPKEEP_INTERVAL_MS \- connection upkeep interval
+.SH SYNOPSIS
+#include <curl/curl.h>
+
+CURLcode curl_easy_setopt(CURL *handle, CURLOPT_CONN_UPKEEP_INTERVAL_MS, long upkeep_interval_ms);
+.SH DESCRIPTION
+Some protocols have "connection upkeep" mechanisms. These mechanisms usually
+send some traffic on existing connections in order to keep them alive; this
+can prevent connections from being closed due to overzealous firewalls, for
+example.
+
+The user needs to explicitly call \fIcurl_easy_conn_upkeep(3)\fP in order to
+perform the upkeep work.
+
+Currently the only protocol with a connection upkeep mechanism is HTTP/2: when
+the connection upkeep interval is exceeded and \fIcurl_easy_conn_upkeep(3)\fP
+is called, an HTTP/2 PING frame is sent on the connection.
+
+.SH DEFAULT
+CURL_UPKEEP_INTERVAL_DEFAULT (currently defined as 60000L, which is 60 seconds)
+.SH EXAMPLE
+.nf
+CURL *curl = curl_easy_init();
+if(curl) {
+  /* Make a connection to an HTTP/2 server. */
+  curl_easy_setopt(curl, CURLOPT_URL, "https://example.com");
+
+  /* Set the interval to 30000ms / 30s */
+  curl_easy_setopt(curl, CURLOPT_CONN_UPKEEP_INTERVAL_MS, 30000L);
+
+  curl_easy_perform(curl);
+
+  /* Perform more work here. */
+
+  /* While the connection is being held open, curl_easy_conn_upkeep() can be
+     called. If curl_easy_conn_upkeep() is called and the time since the last
+     upkeep exceeds the interval, then an HTTP/2 PING is sent. */
+  curl_easy_conn_upkeep(curl);
+
+  /* Perform more work here. */
+
+  /* always cleanup */
+  curl_easy_cleanup(curl);
+}
+.fi
+.SH AVAILABILITY
+Added in 7.62.0
+.SH RETURN VALUE
+Returns CURLE_OK

docs/libcurl/opts/Makefile.inc

@@ -94,6 +94,7 @@ man_MANS =                                      \
   CURLOPT_CHUNK_END_FUNCTION.3                  \
   CURLOPT_CLOSESOCKETDATA.3                     \
   CURLOPT_CLOSESOCKETFUNCTION.3                 \
+  CURLOPT_CONN_UPKEEP_INTERVAL_MS.3             \
   CURLOPT_CONNECTTIMEOUT.3                      \
   CURLOPT_CONNECTTIMEOUT_MS.3                   \
   CURLOPT_CONNECT_ONLY.3                        \

docs/libcurl/symbols-in-versions

@@ -356,6 +356,7 @@ CURLOPT_CLOSEFUNCTION           7.7           7.11.1      7.15.5
 CURLOPT_CLOSEPOLICY             7.7           7.16.1
 CURLOPT_CLOSESOCKETDATA         7.21.7
 CURLOPT_CLOSESOCKETFUNCTION     7.21.7
+CURLOPT_CONN_UPKEEP_INTERVAL_MS 7.62.0
 CURLOPT_CONNECTTIMEOUT          7.7
 CURLOPT_CONNECTTIMEOUT_MS       7.16.2
 CURLOPT_CONNECT_ONLY            7.15.2
@@ -856,6 +857,7 @@ CURL_TIMECOND_LASTMOD           7.9.7
 CURL_TIMECOND_NONE              7.9.7
 CURL_TLSAUTH_NONE               7.21.4
 CURL_TLSAUTH_SRP                7.21.4
+CURL_UPKEEP_INTERVAL_DEFAULT    7.62.0
 CURL_VERSION_ASYNCHDNS          7.10.7
 CURL_VERSION_BROTLI             7.57.0
 CURL_VERSION_CONV               7.15.4

include/curl/curl.h

@@ -803,6 +803,9 @@ typedef enum {
    this value, keep them in sync. */
 #define CURL_HET_DEFAULT 200L
 
+/* The default connection upkeep interval in milliseconds. */
+#define CURL_UPKEEP_INTERVAL_DEFAULT 60000L
+
 #ifndef CURL_NO_OLDIES /* define this to test if your app builds with all
                           the obsolete stuff removed! */
 
@@ -1865,6 +1868,9 @@ typedef enum {
   /* Preferred buffer size to use for uploads */
   CINIT(UPLOAD_BUFFERSIZE, LONG, 280),
 
+  /* Time in ms between connection upkeep calls for long-lived connections. */
+  CINIT(CONN_UPKEEP_INTERVAL_MS, LONG, 281),
+
   CURLOPT_LASTENTRY /* the last unused */
 } CURLoption;
 

include/curl/easy.h

@@ -95,6 +95,16 @@ CURL_EXTERN CURLcode curl_easy_recv(CURL *curl, void *buffer, size_t buflen,
 CURL_EXTERN CURLcode curl_easy_send(CURL *curl, const void *buffer,
                                     size_t buflen, size_t *n);
 
+
+/*
+ * NAME curl_easy_conn_upkeep()
+ *
+ * DESCRIPTION
+ *
+ * Performs connection upkeep for the given session handle.
+ */
+CURL_EXTERN CURLcode curl_easy_conn_upkeep(CURL *curl);
+
 #ifdef  __cplusplus
 }
 #endif

lib/easy.c

@@ -1197,3 +1197,23 @@ CURLcode curl_easy_send(struct Curl_easy *data, const void *buffer,
 
   return result;
 }
+
+/*
+ * Performs connection upkeep for the given session handle.
+ */
+CURLcode curl_easy_conn_upkeep(struct Curl_easy *data)
+{
+  /* Verify that we got an easy handle we can work with. */
+  if(!GOOD_EASY_HANDLE(data))
+    return CURLE_BAD_FUNCTION_ARGUMENT;
+
+  if(data->multi_easy) {
+    /* Use the common function to keep connections alive. */
+    return Curl_conn_upkeep(&data->multi_easy->conn_cache, data);
+  }
+  else {
+    /* No connections, so just return success */
+    return CURLE_OK;
+  }
+}
+

lib/http2.c

@@ -233,12 +233,43 @@ static unsigned int http2_conncheck(struct connectdata *check,
                                     unsigned int checks_to_perform)
 {
   unsigned int ret_val = CONNRESULT_NONE;
+  struct http_conn *c = &check->proto.httpc;
+  int rc;
+  bool send_frames = false;
 
   if(checks_to_perform & CONNCHECK_ISDEAD) {
     if(http2_connisdead(check))
       ret_val |= CONNRESULT_DEAD;
   }
 
+  if(checks_to_perform & CONNCHECK_KEEPALIVE) {
+    struct curltime now = Curl_now();
+    time_t elapsed = Curl_timediff(now, check->keepalive);
+
+    if(elapsed > check->upkeep_interval_ms) {
+      /* Perform an HTTP/2 PING */
+      rc = nghttp2_submit_ping(c->h2, 0, ZERO_NULL);
+      if(!rc) {
+        /* Successfully added a PING frame to the session. Need to flag this
+           so the frame is sent. */
+        send_frames = true;
+      }
+      else {
+       failf(check->data, "nghttp2_submit_ping() failed: %s(%d)",
+             nghttp2_strerror(rc), rc);
+      }
+
+      check->keepalive = now;
+    }
+  }
+
+  if(send_frames) {
+    rc = nghttp2_session_send(c->h2);
+    if(rc)
+      failf(check->data, "nghttp2_session_send() failed: %s(%d)",
+            nghttp2_strerror(rc), rc);
+  }
+
   return ret_val;
 }
 

lib/setopt.c

@@ -2624,6 +2624,12 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option,
                             va_arg(param, char *));
     data->set.doh = data->set.str[STRING_DOH]?TRUE:FALSE;
     break;
+  case CURLOPT_CONN_UPKEEP_INTERVAL_MS:
+    arg = va_arg(param, long);
+    if(arg < 0)
+      return CURLE_BAD_FUNCTION_ARGUMENT;
+    data->set.upkeep_interval_ms = arg;
+    break;
   default:
     /* unknown tag and its companion, just ignore: */
     result = CURLE_UNKNOWN_OPTION;

lib/url.c

@@ -529,6 +529,7 @@ CURLcode Curl_init_userdefined(struct Curl_easy *data)
   set->upload_buffer_size = UPLOADBUFFER_DEFAULT;
   set->happy_eyeballs_timeout = CURL_HET_DEFAULT;
   set->fnmatch = ZERO_NULL;
+  set->upkeep_interval_ms = CURL_UPKEEP_INTERVAL_DEFAULT;
   set->maxconnects = DEFAULT_CONNCACHE_SIZE; /* for easy handles */
   set->httpversion =
 #ifdef USE_NGHTTP2
@@ -1831,6 +1832,12 @@ static struct connectdata *allocate_conn(struct Curl_easy *data)
   /* Store creation time to help future close decision making */
   conn->created = Curl_now();
 
+  /* Store current time to give a baseline to keepalive connection times. */
+  conn->keepalive = Curl_now();
+
+  /* Store off the configured connection upkeep time. */
+  conn->upkeep_interval_ms = data->set.upkeep_interval_ms;
+
   conn->data = data; /* Setup the association between this connection
                         and the Curl_easy */
 
@@ -4843,3 +4850,34 @@ static unsigned int get_protocol_family(unsigned int protocol)
 
   return family;
 }
+
+
+/*
+ * Wrapper to call functions in Curl_conncache_foreach()
+ *
+ * Returns always 0.
+ */
+static int conn_upkeep(struct connectdata *conn,
+                       void *param)
+{
+  /* Param is unused. */
+  (void)param;
+
+  if(conn->handler->connection_check) {
+    /* Do a protocol-specific keepalive check on the connection. */
+    conn->handler->connection_check(conn, CONNCHECK_KEEPALIVE);
+  }
+
+  return 0; /* continue iteration */
+}
+
+CURLcode Curl_conn_upkeep(struct conncache *conn_cache,
+                          void *data)
+{
+  /* Loop over every connection and make connection alive. */
+  Curl_conncache_foreach(data,
+                         conn_cache,
+                         data,
+                         conn_upkeep);
+  return CURLE_OK;
+}

lib/url.h

@@ -77,6 +77,7 @@ int Curl_removeHandleFromPipeline(struct Curl_easy *handle,
 void Curl_getoff_all_pipelines(struct Curl_easy *data,
                                struct connectdata *conn);
 
+CURLcode Curl_conn_upkeep(struct conncache *conn_cache, void *data);
 
 #define CURL_DEFAULT_PROXY_PORT 1080 /* default proxy port unless specified */
 #define CURL_DEFAULT_HTTPS_PROXY_PORT 443 /* default https proxy port unless

lib/urldata.h

@@ -729,6 +729,7 @@ struct Curl_handler {
 
 #define CONNCHECK_NONE 0                 /* No checks */
 #define CONNCHECK_ISDEAD (1<<0)          /* Check if the connection is dead. */
+#define CONNCHECK_KEEPALIVE (1<<1)       /* Perform any keepalive function. */
 
 #define CONNRESULT_NONE 0                /* No extra information. */
 #define CONNRESULT_DEAD (1<<0)           /* The connection is dead. */
@@ -905,6 +906,13 @@ struct connectdata {
 
   long ip_version; /* copied from the Curl_easy at creation time */
 
+  /* Protocols can use a custom keepalive mechanism to keep connections alive.
+     This allows those protocols to track the last time the keepalive mechanism
+     was used on this connection. */
+  struct curltime keepalive;
+
+  long upkeep_interval_ms;      /* Time between calls for connection upkeep. */
+
   /**** curl_get() phase fields */
 
   curl_socket_t sockfd;   /* socket to read from or CURL_SOCKET_BAD */
@@ -1704,6 +1712,7 @@ struct UserDefined {
                                                   before resolver start */
   void *resolver_start_client; /* pointer to pass to resolver start callback */
   bool disallow_username_in_url; /* disallow username in url */
+  long upkeep_interval_ms;      /* Time between calls for connection upkeep. */
   bool doh; /* DNS-over-HTTPS enabled */
   bool doh_get; /* use GET for DoH requests, instead of POST */
   multidone_func fmultidone;

tests/data/test1135

@@ -75,6 +75,7 @@ CURL_EXTERN CURL *curl_easy_duphandle(CURL *curl);
 CURL_EXTERN void curl_easy_reset(CURL *curl);
 CURL_EXTERN CURLcode curl_easy_recv(CURL *curl, void *buffer, size_t buflen,
 CURL_EXTERN CURLcode curl_easy_send(CURL *curl, const void *buffer,
+CURL_EXTERN CURLcode curl_easy_conn_upkeep(CURL *curl);
 CURL_EXTERN int curl_mprintf(const char *format, ...);
 CURL_EXTERN int curl_mfprintf(FILE *fd, const char *format, ...);
 CURL_EXTERN int curl_msprintf(char *buffer, const char *format, ...);