curl: support parallel transfers

This is done by making sure each individual transfer is first added to a
linked list as then they can be performed serially, or at will, in
parallel.

Closes #3804

CMakeLists.txt

@@ -860,6 +860,7 @@ check_symbol_exists(strlcat       "${CURL_INCLUDES}" HAVE_STRLCAT)
 check_symbol_exists(getpwuid      "${CURL_INCLUDES}" HAVE_GETPWUID)
 check_symbol_exists(getpwuid_r    "${CURL_INCLUDES}" HAVE_GETPWUID_R)
 check_symbol_exists(geteuid       "${CURL_INCLUDES}" HAVE_GETEUID)
+check_symbol_exists(usleep        "${CURL_INCLUDES}" HAVE_USLEEP)
 check_symbol_exists(utime         "${CURL_INCLUDES}" HAVE_UTIME)
 check_symbol_exists(gmtime_r      "${CURL_INCLUDES}" HAVE_GMTIME_R)
 check_symbol_exists(localtime_r   "${CURL_INCLUDES}" HAVE_LOCALTIME_R)

configure.ac

@@ -3671,6 +3671,7 @@ AC_CHECK_FUNCS([fnmatch \
   setlocale \
   setmode \
   setrlimit \
+  usleep \
   utime \
   utimes
 ],[

docs/cmdline-opts/Makefile.inc

@@ -100,7 +100,10 @@ DPAGES =					\
   noproxy.d					\
   ntlm.d ntlm-wb.d				\
   oauth2-bearer.d				\
-  output.d pass.d				\
+  output.d                                      \
+  pass.d                			\
+  parallel.d                                    \
+  parallel-max.d                                \
   path-as-is.d					\
   pinnedpubkey.d				\
   post301.d					\

docs/cmdline-opts/parallel-max.d

@@ -0,0 +1,9 @@
+Long: parallel-max
+Help: Maximum concurrency for parallel transfers
+Added: 7.66.0
+See-also: parallel
+---
+When asked to do parallel transfers, using --parallel, this option controls
+the maximum amount of transfers to do simultaneously.
+
+The default is 50.

docs/cmdline-opts/parallel.d

@@ -0,0 +1,7 @@
+Short: Z
+Long: parallel
+Help: Perform transfers in parallel
+Added: 7.66.0
+---
+Makes curl perform its transfers in parallel as compared to the regular serial
+manner.

src/Makefile.inc

@@ -54,6 +54,7 @@ CURL_CFILES = \
   tool_panykey.c \
   tool_paramhlp.c \
   tool_parsecfg.c \
+  tool_progress.c \
   tool_strdup.c \
   tool_setopt.c \
   tool_sleep.c \
@@ -95,6 +96,7 @@ CURL_HFILES = \
   tool_panykey.h \
   tool_paramhlp.h \
   tool_parsecfg.h \
+  tool_progress.h \
   tool_sdecls.h \
   tool_setopt.h \
   tool_setup.h \

src/tool_cb_hdr.c

@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -32,6 +32,7 @@
 #include "tool_msgs.h"
 #include "tool_cb_hdr.h"
 #include "tool_cb_wrt.h"
+#include "tool_operate.h"
 
 #include "memdebug.h" /* keep this as LAST include */
 
@@ -54,9 +55,10 @@ static char *parse_filename(const char *ptr, size_t len);
 
 size_t tool_header_cb(char *ptr, size_t size, size_t nmemb, void *userdata)
 {
-  struct HdrCbData *hdrcbdata = userdata;
+  struct per_transfer *per = userdata;
-  struct OutStruct *outs = hdrcbdata->outs;
+  struct HdrCbData *hdrcbdata = &per->hdrcbdata;
-  struct OutStruct *heads = hdrcbdata->heads;
+  struct OutStruct *outs = &per->outs;
+  struct OutStruct *heads = &per->heads;
   const char *str = ptr;
   const size_t cb = size * nmemb;
   const char *end = (char *)ptr + cb;
@@ -100,7 +102,7 @@ size_t tool_header_cb(char *ptr, size_t size, size_t nmemb, void *userdata)
    * Content-Disposition header specifying a filename property.
    */
 
-  curl_easy_getinfo(outs->config->easy, CURLINFO_PROTOCOL, &protocol);
+  curl_easy_getinfo(per->curl, CURLINFO_PROTOCOL, &protocol);
   if(hdrcbdata->honor_cd_filename &&
      (cb > 20) && checkprefix("Content-disposition:", str) &&
      (protocol & (CURLPROTO_HTTPS|CURLPROTO_HTTP))) {

src/tool_cb_wrt.c

@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -28,6 +28,7 @@
 #include "tool_cfgable.h"
 #include "tool_msgs.h"
 #include "tool_cb_wrt.h"
+#include "tool_operate.h"
 
 #include "memdebug.h" /* keep this as LAST include */
 
@@ -75,7 +76,8 @@ bool tool_create_output_file(struct OutStruct *outs)
 size_t tool_write_cb(char *buffer, size_t sz, size_t nmemb, void *userdata)
 {
   size_t rc;
-  struct OutStruct *outs = userdata;
+  struct per_transfer *per = userdata;
+  struct OutStruct *outs = &per->outs;
   struct OperationConfig *config = outs->config;
   size_t bytes = sz * nmemb;
   bool is_tty = config->global->isatty;
@@ -202,7 +204,7 @@ size_t tool_write_cb(char *buffer, size_t sz, size_t nmemb, void *userdata)
 
   if(config->readbusy) {
     config->readbusy = FALSE;
-    curl_easy_pause(config->easy, CURLPAUSE_CONT);
+    curl_easy_pause(per->curl, CURLPAUSE_CONT);
   }
 
   if(config->nobuffer) {

src/tool_cfgable.h

@@ -38,7 +38,6 @@ typedef enum {
 struct GlobalConfig;
 
 struct OperationConfig {
-  CURL *easy;               /* A copy of the handle from GlobalConfig */
   bool remote_time;
   char *random_file;
   char *egd_file;
@@ -242,9 +241,6 @@ struct OperationConfig {
   bool use_metalink;        /* process given URLs as metalink XML file */
   metalinkfile *metalinkfile_list; /* point to the first node */
   metalinkfile *metalinkfile_last; /* point to the last/current node */
-#ifdef CURLDEBUG
-  bool test_event_based;
-#endif
   char *oauth_bearer;             /* OAuth 2.0 bearer token */
   bool nonpn;                     /* enable/disable TLS NPN extension */
   bool noalpn;                    /* enable/disable TLS ALPN extension */
@@ -268,7 +264,6 @@ struct OperationConfig {
 };
 
 struct GlobalConfig {
-  CURL *easy;                     /* Once we have one, we keep it here */
   int showerror;                  /* -1 == unset, default => show errors
                                       0 => -s is used to NOT show errors
                                       1 => -S has been used to show errors */
@@ -286,6 +281,11 @@ struct GlobalConfig {
   char *libcurl;                  /* Output libcurl code to this file name */
   bool fail_early;                /* exit on first transfer error */
   bool styled_output;             /* enable fancy output style detection */
+#ifdef CURLDEBUG
+  bool test_event_based;
+#endif
+  bool parallel;
+  long parallel_max;
   struct OperationConfig *first;
   struct OperationConfig *current;
   struct OperationConfig *last;   /* Always last in the struct */

src/tool_getparam.c

@@ -40,6 +40,7 @@
 #include "tool_msgs.h"
 #include "tool_paramhlp.h"
 #include "tool_parsecfg.h"
+#include "tool_main.h"
 
 #include "memdebug.h" /* keep this as LAST include */
 
@@ -316,6 +317,8 @@ static const struct LongShort aliases[]= {
   {"Y",  "speed-limit",              ARG_STRING},
   {"y",  "speed-time",               ARG_STRING},
   {"z",  "time-cond",                ARG_STRING},
+  {"Z",  "parallel",                 ARG_BOOL},
+  {"Zb", "parallel-max",             ARG_STRING},
   {"#",  "progress-bar",             ARG_BOOL},
   {":",  "next",                     ARG_NONE},
 };
@@ -1104,7 +1107,7 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
         break;
       case 'L': /* --test-event */
 #ifdef CURLDEBUG
-        config->test_event_based = toggle;
+        global->test_event_based = toggle;
 #else
         warnf(global, "--test-event is ignored unless a debug build!\n");
 #endif
@@ -1356,7 +1359,7 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
           size = 0;
         }
         else {
-          char *enc = curl_easy_escape(config->easy, postdata, (int)size);
+          char *enc = curl_easy_escape(NULL, postdata, (int)size);
           Curl_safefree(postdata); /* no matter if it worked or not */
           if(enc) {
             /* now make a string with the name from above and append the
@@ -2127,6 +2130,21 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
       if(!config->low_speed_time)
         config->low_speed_time = 30;
       break;
+    case 'Z':
+      switch(subletter) {
+      case '\0':  /* --parallel */
+        global->parallel = toggle;
+        break;
+      case 'b':   /* --parallel-max */
+        err = str2unum(&global->parallel_max, nextarg);
+        if(err)
+          return err;
+        if((global->parallel_max > MAX_PARALLEL) ||
+           (global->parallel_max < 1))
+          global->parallel_max = PARALLEL_DEFAULT;
+        break;
+      }
+      break;
     case 'z': /* time condition coming up */
       switch(*nextarg) {
       case '+':
@@ -2176,14 +2194,14 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
   return PARAM_OK;
 }
 
-ParameterError parse_args(struct GlobalConfig *config, int argc,
+ParameterError parse_args(struct GlobalConfig *global, int argc,
                           argv_item_t argv[])
 {
   int i;
   bool stillflags;
   char *orig_opt = NULL;
   ParameterError result = PARAM_OK;
-  struct OperationConfig *operation = config->first;
+  struct OperationConfig *config = global->first;
 
   for(i = 1, stillflags = TRUE; i < argc && !result; i++) {
     orig_opt = argv[i];
@@ -2199,31 +2217,28 @@ ParameterError parse_args(struct GlobalConfig *config, int argc,
       else {
         char *nextarg = (i < (argc - 1)) ? argv[i + 1] : NULL;
 
-        result = getparameter(flag, nextarg, &passarg, config, operation);
+        result = getparameter(flag, nextarg, &passarg, global, config);
         if(result == PARAM_NEXT_OPERATION) {
           /* Reset result as PARAM_NEXT_OPERATION is only used here and not
              returned from this function */
           result = PARAM_OK;
 
-          if(operation->url_list && operation->url_list->url) {
+          if(config->url_list && config->url_list->url) {
             /* Allocate the next config */
-            operation->next = malloc(sizeof(struct OperationConfig));
+            config->next = malloc(sizeof(struct OperationConfig));
-            if(operation->next) {
+            if(config->next) {
               /* Initialise the newly created config */
-              config_init(operation->next);
+              config_init(config->next);
-
-              /* Copy the easy handle */
-              operation->next->easy = config->easy;
 
               /* Set the global config pointer */
-              operation->next->global = config;
+              config->next->global = global;
 
-              /* Update the last operation pointer */
+              /* Update the last config pointer */
-              config->last = operation->next;
+              global->last = config->next;
 
               /* Move onto the new config */
-              operation->next->prev = operation;
+              config->next->prev = config;
-              operation = operation->next;
+              config = config->next;
             }
             else
               result = PARAM_NO_MEM;
@@ -2237,8 +2252,8 @@ ParameterError parse_args(struct GlobalConfig *config, int argc,
       bool used;
 
       /* Just add the URL please */
-      result = getparameter((char *)"--url", argv[i], &used, config,
+      result = getparameter((char *)"--url", argv[i], &used, global,
-                            operation);
+                            config);
     }
   }
 
@@ -2249,9 +2264,9 @@ ParameterError parse_args(struct GlobalConfig *config, int argc,
     const char *reason = param2text(result);
 
     if(orig_opt && strcmp(":", orig_opt))
-      helpf(config->errors, "option %s: %s\n", orig_opt, reason);
+      helpf(global->errors, "option %s: %s\n", orig_opt, reason);
     else
-      helpf(config->errors, "%s\n", reason);
+      helpf(global->errors, "%s\n", reason);
   }
 
   return result;

src/tool_help.c

@@ -273,6 +273,10 @@ static const struct helptxt helptext[] = {
    "OAuth 2 Bearer Token"},
   {"-o, --output <file>",
    "Write to file instead of stdout"},
+  {"-Z, --parallel",
+   "Perform transfers in parallel"},
+  {"    --parallel-max",
+   "Maximum concurrency for parallel transfers"},
   {"    --pass <phrase>",
    "Pass phrase for the private key"},
   {"    --path-as-is",
@@ -602,8 +606,9 @@ void tool_version_info(void)
   }
 }
 
-void tool_list_engines(CURL *curl)
+void tool_list_engines(void)
 {
+  CURL *curl = curl_easy_init();
   struct curl_slist *engines = NULL;
 
   /* Get the list of engines */
@@ -620,4 +625,5 @@ void tool_list_engines(CURL *curl)
 
   /* Cleanup the list of engines */
   curl_slist_free_all(engines);
+  curl_easy_cleanup(curl);
 }

src/tool_help.h

@@ -7,7 +7,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2014, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -24,7 +24,7 @@
 #include "tool_setup.h"
 
 void tool_help(void);
-void tool_list_engines(CURL *curl);
+void tool_list_engines(void);
 void tool_version_info(void);
 
 #endif /* HEADER_CURL_TOOL_HELP_H */

src/tool_main.c

@@ -149,6 +149,7 @@ static CURLcode main_init(struct GlobalConfig *config)
   config->showerror = -1;             /* Will show errors */
   config->errors = stderr;            /* Default errors to stderr */
   config->styled_output = TRUE;       /* enable detection */
+  config->parallel_max = PARALLEL_DEFAULT;
 
   /* Allocate the initial operate config */
   config->first = config->last = malloc(sizeof(struct OperationConfig));
@@ -160,20 +161,10 @@ static CURLcode main_init(struct GlobalConfig *config)
       result = get_libcurl_info();
 
       if(!result) {
-        /* Get a curl handle to use for all forthcoming curl transfers */
-        config->easy = curl_easy_init();
-        if(config->easy) {
         /* Initialise the config */
         config_init(config->first);
-          config->first->easy = config->easy;
         config->first->global = config;
       }
-        else {
-          helpf(stderr, "error initializing curl easy handle\n");
-          result = CURLE_FAILED_INIT;
-          free(config->first);
-        }
-      }
       else {
         helpf(stderr, "error retrieving curl library information\n");
         free(config->first);
@@ -214,9 +205,6 @@ static void free_globalconfig(struct GlobalConfig *config)
 static void main_free(struct GlobalConfig *config)
 {
   /* Cleanup the easy handle */
-  curl_easy_cleanup(config->easy);
-  config->easy = NULL;
-
   /* Main cleanup */
   curl_global_cleanup();
   convert_cleanup();

src/tool_main.h

@@ -7,7 +7,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2012, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -28,6 +28,9 @@
 #define RETRY_SLEEP_DEFAULT 1000L   /* ms */
 #define RETRY_SLEEP_MAX     600000L /* ms == 10 minutes */
 
+#define MAX_PARALLEL 300 /* conservative */
+#define PARALLEL_DEFAULT 50
+
 #ifndef STDIN_FILENO
 #  define STDIN_FILENO  fileno(stdin)
 #endif

src/tool_metalink.c

@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -104,6 +104,7 @@ struct win32_crypto_hash {
 #include "tool_paramhlp.h"
 #include "tool_cfgable.h"
 #include "tool_metalink.h"
+#include "tool_operate.h"
 #include "tool_msgs.h"
 
 #include "memdebug.h" /* keep this as LAST include */
@@ -674,8 +675,9 @@ int metalink_check_hash(struct GlobalConfig *config,
   return rv;
 }
 
-static metalink_checksum *new_metalink_checksum_from_hex_digest
+static metalink_checksum *
-(const metalink_digest_def *digest_def, const char *hex_digest)
+checksum_from_hex_digest(const metalink_digest_def *digest_def,
+                         const char *hex_digest)
 {
   metalink_checksum *chksum;
   unsigned char *digest;
@@ -754,7 +756,7 @@ static metalinkfile *new_metalinkfile(metalink_file_t *fileinfo)
         if(curl_strequal(digest_alias->alias_name, (*p)->type) &&
            check_hex_digest((*p)->hash, digest_alias->digest_def)) {
           f->checksum =
-            new_metalink_checksum_from_hex_digest(digest_alias->digest_def,
+            checksum_from_hex_digest(digest_alias->digest_def,
                                      (*p)->hash);
           break;
         }
@@ -891,7 +893,8 @@ int parse_metalink(struct OperationConfig *config, struct OutStruct *outs,
 size_t metalink_write_cb(void *buffer, size_t sz, size_t nmemb,
                          void *userdata)
 {
-  struct OutStruct *outs = userdata;
+  struct per_transfer *per = userdata;
+  struct OutStruct *outs = &per->outs;
   struct OperationConfig *config = outs->config;
   int rv;
 

src/tool_metalink.h

@@ -7,7 +7,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2014, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2014, 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -22,6 +22,7 @@
  *
  ***************************************************************************/
 #include "tool_setup.h"
+#include "tool_sdecls.h"
 
 struct GlobalConfig;
 struct OperationConfig;

src/tool_operate.h

@@ -7,7 +7,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2014, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -22,7 +22,53 @@
  *
  ***************************************************************************/
 #include "tool_setup.h"
+#include "tool_cb_hdr.h"
+#include "tool_cb_prg.h"
+#include "tool_sdecls.h"
+
+struct per_transfer {
+  /* double linked */
+  struct per_transfer *next;
+  struct per_transfer *prev;
+  struct OperationConfig *config; /* for this transfer */
+  CURL *curl;
+  long retry_numretries;
+  long retry_sleep_default;
+  long retry_sleep;
+  struct timeval retrystart;
+  bool metalink; /* nonzero for metalink download. */
+  bool metalink_next_res;
+  metalinkfile *mlfile;
+  metalink_resource *mlres;
+  char *this_url;
+  char *outfile;
+  bool infdopen; /* TRUE if infd needs closing */
+  int infd;
+  struct ProgressData progressbar;
+  struct OutStruct outs;
+  struct OutStruct heads;
+  struct InStruct input;
+  struct HdrCbData hdrcbdata;
+  char errorbuffer[CURL_ERROR_SIZE];
+
+  bool added; /* set TRUE when added to the multi handle */
+
+  /* for parallel progress bar */
+  curl_off_t dltotal;
+  curl_off_t dlnow;
+  curl_off_t ultotal;
+  curl_off_t ulnow;
+  bool dltotal_added; /* if the total has been added from this */
+  bool ultotal_added;
+
+  /* NULL or malloced */
+  char *separator_err;
+  char *separator;
+  char *uploadfile;
+};
 
 CURLcode operate(struct GlobalConfig *config, int argc, argv_item_t argv[]);
 
+extern struct per_transfer *transfers; /* first node */
+
 #endif /* HEADER_CURL_TOOL_OPERATE_H */

src/tool_operhlp.c

@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -71,10 +71,13 @@ bool stdin_upload(const char *uploadfile)
  * Adds the file name to the URL if it doesn't already have one.
  * url will be freed before return if the returned pointer is different
  */
-char *add_file_name_to_url(CURL *curl, char *url, const char *filename)
+char *add_file_name_to_url(char *url, const char *filename)
 {
   /* If no file name part is given in the URL, we add this file name */
   char *ptr = strstr(url, "://");
+  CURL *curl = curl_easy_init(); /* for url escaping */
+  if(!curl)
+    return NULL; /* error! */
   if(ptr)
     ptr += 3;
   else
@@ -120,6 +123,7 @@ char *add_file_name_to_url(CURL *curl, char *url, const char *filename)
     else
       Curl_safefree(url);
   }
+  curl_easy_cleanup(curl);
   return url;
 }
 

src/tool_operhlp.h

@@ -7,7 +7,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2014, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -31,7 +31,7 @@ bool output_expected(const char *url, const char *uploadfile);
 
 bool stdin_upload(const char *uploadfile);
 
-char *add_file_name_to_url(CURL *curl, char *url, const char *filename);
+char *add_file_name_to_url(char *url, const char *filename);
 
 CURLcode get_url_file_name(char **filename, const char *url);
 

src/tool_parsecfg.c

@@ -230,9 +230,6 @@ int parseconfig(const char *filename, struct GlobalConfig *global)
             /* Initialise the newly created config */
             config_init(operation->next);
 
-            /* Copy the easy handle */
-            operation->next->easy = global->easy;
-
             /* Set the global config pointer */
             operation->next->global = global;
 

src/tool_progress.c

@@ -0,0 +1,314 @@
+/***************************************************************************
+ *                                  _   _ ____  _
+ *  Project                     ___| | | |  _ \| |
+ *                             / __| | | | |_) | |
+ *                            | (__| |_| |  _ <| |___
+ *                             \___|\___/|_| \_\_____|
+ *
+ * Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
+ *
+ * This software is licensed as described in the file COPYING, which
+ * you should have received as part of this distribution. The terms
+ * are also available at https://curl.haxx.se/docs/copyright.html.
+ *
+ * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+ * copies of the Software, and permit persons to whom the Software is
+ * furnished to do so, under the terms of the COPYING file.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ ***************************************************************************/
+#include "tool_setup.h"
+#include "tool_operate.h"
+#include "tool_progress.h"
+#include "tool_util.h"
+
+#define ENABLE_CURLX_PRINTF
+/* use our own printf() functions */
+#include "curlx.h"
+
+/* The point of this function would be to return a string of the input data,
+   but never longer than 5 columns (+ one zero byte).
+   Add suffix k, M, G when suitable... */
+static char *max5data(curl_off_t bytes, char *max5)
+{
+#define ONE_KILOBYTE  CURL_OFF_T_C(1024)
+#define ONE_MEGABYTE (CURL_OFF_T_C(1024) * ONE_KILOBYTE)
+#define ONE_GIGABYTE (CURL_OFF_T_C(1024) * ONE_MEGABYTE)
+#define ONE_TERABYTE (CURL_OFF_T_C(1024) * ONE_GIGABYTE)
+#define ONE_PETABYTE (CURL_OFF_T_C(1024) * ONE_TERABYTE)
+
+  if(bytes < CURL_OFF_T_C(100000))
+    msnprintf(max5, 6, "%5" CURL_FORMAT_CURL_OFF_T, bytes);
+
+  else if(bytes < CURL_OFF_T_C(10000) * ONE_KILOBYTE)
+    msnprintf(max5, 6, "%4" CURL_FORMAT_CURL_OFF_T "k", bytes/ONE_KILOBYTE);
+
+  else if(bytes < CURL_OFF_T_C(100) * ONE_MEGABYTE)
+    /* 'XX.XM' is good as long as we're less than 100 megs */
+    msnprintf(max5, 6, "%2" CURL_FORMAT_CURL_OFF_T ".%0"
+              CURL_FORMAT_CURL_OFF_T "M", bytes/ONE_MEGABYTE,
+              (bytes%ONE_MEGABYTE) / (ONE_MEGABYTE/CURL_OFF_T_C(10)) );
+
+#if (CURL_SIZEOF_CURL_OFF_T > 4)
+
+  else if(bytes < CURL_OFF_T_C(10000) * ONE_MEGABYTE)
+    /* 'XXXXM' is good until we're at 10000MB or above */
+    msnprintf(max5, 6, "%4" CURL_FORMAT_CURL_OFF_T "M", bytes/ONE_MEGABYTE);
+
+  else if(bytes < CURL_OFF_T_C(100) * ONE_GIGABYTE)
+    /* 10000 MB - 100 GB, we show it as XX.XG */
+    msnprintf(max5, 6, "%2" CURL_FORMAT_CURL_OFF_T ".%0"
+              CURL_FORMAT_CURL_OFF_T "G", bytes/ONE_GIGABYTE,
+              (bytes%ONE_GIGABYTE) / (ONE_GIGABYTE/CURL_OFF_T_C(10)) );
+
+  else if(bytes < CURL_OFF_T_C(10000) * ONE_GIGABYTE)
+    /* up to 10000GB, display without decimal: XXXXG */
+    msnprintf(max5, 6, "%4" CURL_FORMAT_CURL_OFF_T "G", bytes/ONE_GIGABYTE);
+
+  else if(bytes < CURL_OFF_T_C(10000) * ONE_TERABYTE)
+    /* up to 10000TB, display without decimal: XXXXT */
+    msnprintf(max5, 6, "%4" CURL_FORMAT_CURL_OFF_T "T", bytes/ONE_TERABYTE);
+
+  else
+    /* up to 10000PB, display without decimal: XXXXP */
+    msnprintf(max5, 6, "%4" CURL_FORMAT_CURL_OFF_T "P", bytes/ONE_PETABYTE);
+
+    /* 16384 petabytes (16 exabytes) is the maximum a 64 bit unsigned number
+       can hold, but our data type is signed so 8192PB will be the maximum. */
+
+#else
+
+  else
+    msnprintf(max5, 6, "%4" CURL_FORMAT_CURL_OFF_T "M", bytes/ONE_MEGABYTE);
+
+#endif
+
+  return max5;
+}
+
+int xferinfo_cb(void *clientp,
+                curl_off_t dltotal,
+                curl_off_t dlnow,
+                curl_off_t ultotal,
+                curl_off_t ulnow)
+{
+  struct per_transfer *per = clientp;
+  per->dltotal = dltotal;
+  per->dlnow = dlnow;
+  per->ultotal = ultotal;
+  per->ulnow = ulnow;
+  return 0;
+}
+
+/* Provide a string that is 2 + 1 + 2 + 1 + 2 = 8 letters long (plus the zero
+   byte) */
+static void time2str(char *r, curl_off_t seconds)
+{
+  curl_off_t h;
+  if(seconds <= 0) {
+    strcpy(r, "--:--:--");
+    return;
+  }
+  h = seconds / CURL_OFF_T_C(3600);
+  if(h <= CURL_OFF_T_C(99)) {
+    curl_off_t m = (seconds - (h*CURL_OFF_T_C(3600))) / CURL_OFF_T_C(60);
+    curl_off_t s = (seconds - (h*CURL_OFF_T_C(3600))) - (m*CURL_OFF_T_C(60));
+    msnprintf(r, 9, "%2" CURL_FORMAT_CURL_OFF_T ":%02" CURL_FORMAT_CURL_OFF_T
+              ":%02" CURL_FORMAT_CURL_OFF_T, h, m, s);
+  }
+  else {
+    /* this equals to more than 99 hours, switch to a more suitable output
+       format to fit within the limits. */
+    curl_off_t d = seconds / CURL_OFF_T_C(86400);
+    h = (seconds - (d*CURL_OFF_T_C(86400))) / CURL_OFF_T_C(3600);
+    if(d <= CURL_OFF_T_C(999))
+      msnprintf(r, 9, "%3" CURL_FORMAT_CURL_OFF_T
+                "d %02" CURL_FORMAT_CURL_OFF_T "h", d, h);
+    else
+      msnprintf(r, 9, "%7" CURL_FORMAT_CURL_OFF_T "d", d);
+  }
+}
+
+static curl_off_t all_dltotal = 0;
+static curl_off_t all_ultotal = 0;
+static curl_off_t all_dlalready = 0;
+static curl_off_t all_ulalready = 0;
+
+curl_off_t all_xfers = 0;   /* current total */
+
+struct speedcount {
+  curl_off_t dl;
+  curl_off_t ul;
+  struct timeval stamp;
+};
+#define SPEEDCNT 10
+static unsigned int speedindex;
+static bool indexwrapped;
+static struct speedcount speedstore[SPEEDCNT];
+
+/*
+  |DL% UL%  Dled  Uled  Xfers  Live   Qd Total     Current  Left    Speed
+  |  6 --   9.9G     0     2     2     0  0:00:40  0:00:02  0:00:37 4087M
+*/
+bool progress_meter(struct GlobalConfig *global,
+                    struct timeval *start,
+                    bool final)
+{
+  static struct timeval stamp;
+  static bool header = FALSE;
+  struct timeval now;
+  long diff;
+
+  if(global->noprogress)
+    return FALSE;
+
+  now = tvnow();
+  diff = tvdiff(now, stamp);
+
+  if(!header) {
+    header = TRUE;
+    fputs("DL% UL%  Dled  Uled  Xfers  Live   Qd "
+          "Total     Current  Left    Speed\n",
+          global->errors);
+  }
+  if(final || (diff > 500)) {
+    char time_left[10];
+    char time_total[10];
+    char time_spent[10];
+    char buffer[3][6];
+    curl_off_t spent = tvdiff(now, *start)/1000;
+    char dlpercen[4]="--";
+    char ulpercen[4]="--";
+    struct per_transfer *per;
+    curl_off_t all_dlnow = 0;
+    curl_off_t all_ulnow = 0;
+    bool dlknown = TRUE;
+    bool ulknown = TRUE;
+    curl_off_t all_running = 0; /* in progress */
+    curl_off_t all_queued = 0;  /* pending */
+    curl_off_t speed = 0;
+    unsigned int i;
+    stamp = now;
+
+    /* first add the amounts of the already completed transfers */
+    all_dlnow += all_dlalready;
+    all_ulnow += all_ulalready;
+
+    for(per = transfers; per; per = per->next) {
+      all_dlnow += per->dlnow;
+      all_ulnow += per->ulnow;
+      if(!per->dltotal)
+        dlknown = FALSE;
+      else if(!per->dltotal_added) {
+        /* only add this amount once */
+        all_dltotal += per->dltotal;
+        per->dltotal_added = TRUE;
+      }
+      if(!per->ultotal)
+        ulknown = FALSE;
+      else if(!per->ultotal_added) {
+        /* only add this amount once */
+        all_ultotal += per->ultotal;
+        per->ultotal_added = TRUE;
+      }
+      if(!per->added)
+        all_queued++;
+      else
+        all_running++;
+    }
+    if(dlknown && all_dltotal)
+      /* TODO: handle integer overflow */
+      msnprintf(dlpercen, sizeof(dlpercen), "%3d",
+                all_dlnow * 100 / all_dltotal);
+    if(ulknown && all_ultotal)
+      /* TODO: handle integer overflow */
+      msnprintf(ulpercen, sizeof(ulpercen), "%3d",
+                all_ulnow * 100 / all_ultotal);
+
+    /* get the transfer speed, the higher of the two */
+
+    i = speedindex;
+    speedstore[i].dl = all_dlnow;
+    speedstore[i].ul = all_ulnow;
+    speedstore[i].stamp = now;
+    if(++speedindex >= SPEEDCNT) {
+      indexwrapped = TRUE;
+      speedindex = 0;
+    }
+
+    {
+      long deltams;
+      curl_off_t dl;
+      curl_off_t ul;
+      curl_off_t dls;
+      curl_off_t uls;
+      if(indexwrapped) {
+        /* 'speedindex' is the oldest stored data */
+        deltams = tvdiff(now, speedstore[speedindex].stamp);
+        dl = all_dlnow - speedstore[speedindex].dl;
+        ul = all_ulnow - speedstore[speedindex].ul;
+      }
+      else {
+        /* since the beginning */
+        deltams = tvdiff(now, *start);
+        dl = all_dlnow;
+        ul = all_ulnow;
+      }
+      dls = (curl_off_t)((double)dl / ((double)deltams/1000.0));
+      uls = (curl_off_t)((double)ul / ((double)deltams/1000.0));
+      speed = dls > uls ? dls : uls;
+    }
+
+
+    if(dlknown && speed) {
+      curl_off_t est = all_dltotal / speed;
+      curl_off_t left = (all_dltotal - all_dlnow) / speed;
+      time2str(time_left, left);
+      time2str(time_total, est);
+    }
+    else {
+      time2str(time_left, 0);
+      time2str(time_total, 0);
+    }
+    time2str(time_spent, spent);
+
+    fprintf(global->errors,
+            "\r"
+            "%-3s " /* percent downloaded */
+            "%-3s " /* percent uploaded */
+            "%s " /* Dled */
+            "%s " /* Uled */
+            "%5" CURL_FORMAT_CURL_OFF_T " " /* Xfers */
+            "%5" CURL_FORMAT_CURL_OFF_T " " /* Live */
+            "%5" CURL_FORMAT_CURL_OFF_T " " /* Queued */
+            "%s "  /* Total time */
+            "%s "  /* Current time */
+            "%s "  /* Time left */
+            "%s "  /* Speed */
+            "%5s" /* final newline */,
+
+            dlpercen,  /* 3 letters */
+            ulpercen,  /* 3 letters */
+            max5data(all_dlnow, buffer[0]),
+            max5data(all_ulnow, buffer[1]),
+            all_xfers,
+            all_running,
+            all_queued,
+            time_total,
+            time_spent,
+            time_left,
+            max5data(speed, buffer[2]), /* speed */
+            final ? "\n" :"");
+    return TRUE;
+  }
+  return FALSE;
+}
+
+void progress_finalize(struct per_transfer *per)
+{
+  /* get the numbers before this transfer goes away */
+  all_dlalready += per->dlnow;
+  all_ulalready += per->ulnow;
+}

src/tool_progress.h

@@ -0,0 +1,39 @@
+#ifndef HEADER_CURL_TOOL_PROGRESS_H
+#define HEADER_CURL_TOOL_PROGRESS_H
+/***************************************************************************
+ *                                  _   _ ____  _
+ *  Project                     ___| | | |  _ \| |
+ *                             / __| | | | |_) | |
+ *                            | (__| |_| |  _ <| |___
+ *                             \___|\___/|_| \_\_____|
+ *
+ * Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
+ *
+ * This software is licensed as described in the file COPYING, which
+ * you should have received as part of this distribution. The terms
+ * are also available at https://curl.haxx.se/docs/copyright.html.
+ *
+ * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+ * copies of the Software, and permit persons to whom the Software is
+ * furnished to do so, under the terms of the COPYING file.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ ***************************************************************************/
+#include "tool_setup.h"
+
+int xferinfo_cb(void *clientp,
+                curl_off_t dltotal,
+                curl_off_t dlnow,
+                curl_off_t ultotal,
+                curl_off_t ulnow);
+
+bool progress_meter(struct GlobalConfig *global,
+                    struct timeval *start,
+                    bool final);
+void progress_finalize(struct per_transfer *per);
+
+extern curl_off_t all_xfers;   /* total number */
+
+#endif /* HEADER_CURL_TOOL_PROGRESS_H */

tests/data/test1002

@@ -101,6 +101,14 @@ Content-Length: 3
 Expect: 100-continue
 
 st
+GET http://%HOSTIP:%HTTPPORT/1002.upload2 HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+Content-Range: bytes 2-4/5
+User-Agent: curl/7.16.1
+Accept: */*
+Proxy-Connection: Keep-Alive
+Content-Length: 0
+
 GET http://%HOSTIP:%HTTPPORT/1002.upload2 HTTP/1.1
 Host: %HOSTIP:%HTTPPORT
 Authorization: Digest username="auser", realm="testrealm", nonce="1053604144", uri="/1002.upload2", response="d711f0d2042786d930de635ba0d1a1d0"

tests/data/test1291

@@ -21,7 +21,7 @@ HTTP PUT
 none
 </server>
 <name>
-Attempt to upload 100K files but fail immediately
+Attempt to upload 1000 files but fail immediately
 </name>
 <command>
 -K log/cmd1291 --fail-early
@@ -31,7 +31,7 @@ XXXXXXXx
 </file>
 # generate the config file
 <precheck>
-perl -e 'for(1 .. 100000) { printf("upload-file=log/upload-this\nurl=htttttp://non-existing-host.haxx.se/upload/1291\n", $_);}' > log/cmd1291;
+perl -e 'for(1 .. 1000) { printf("upload-file=log/upload-this\nurl=htttttp://non-existing-host.haxx.se/upload/1291\n", $_);}' > log/cmd1291;
 </precheck>
 </client>
 
@@ -40,11 +40,5 @@ perl -e 'for(1 .. 100000) { printf("upload-file=log/upload-this\nurl=htttttp://n
 <errorcode>
 1
 </errorcode>
-
-# we disable valgrind here since it takes 40+ seconds even on a fairly snappy
-# machine
-<valgrind>
-disable
-</valgrind>
 </verify>
 </testcase>

tests/data/test1406

@@ -76,13 +76,13 @@ int main(int argc, char *argv[])
 
   hnd = curl_easy_init();
   curl_easy_setopt(hnd, CURLOPT_BUFFERSIZE, 102400L);
-  curl_easy_setopt(hnd, CURLOPT_INFILESIZE_LARGE, (curl_off_t)38);
   curl_easy_setopt(hnd, CURLOPT_URL, "smtp://%HOSTIP:%SMTPPORT/1406");
   curl_easy_setopt(hnd, CURLOPT_UPLOAD, 1L);
   curl_easy_setopt(hnd, CURLOPT_VERBOSE, 1L);
   curl_easy_setopt(hnd, CURLOPT_TCP_KEEPALIVE, 1L);
   curl_easy_setopt(hnd, CURLOPT_MAIL_FROM, "sender@example.com");
   curl_easy_setopt(hnd, CURLOPT_MAIL_RCPT, slist1);
+  curl_easy_setopt(hnd, CURLOPT_INFILESIZE_LARGE, (curl_off_t)38);
 
   /* Here is a list of options the curl code used that cannot get generated
      as source easily. You may select to either not use them or implement

tests/data/test1412

@@ -25,6 +25,19 @@ Connection: close
 This is not the real page
 </data>
 
+# The second URL will get this response
+<data1>
+HTTP/1.1 401 Authorization Required swsclose
+Server: Apache/1.3.27 (Darwin) PHP/4.1.2
+WWW-Authenticate: Blackmagic realm="gimme all yer s3cr3ts"
+WWW-Authenticate: Basic realm="gimme all yer s3cr3ts"
+WWW-Authenticate: Digest realm="gimme all yer s3cr3ts", nonce="11223344"
+Content-Type: text/html; charset=iso-8859-1
+Connection: close
+
+This is not the real page
+</data1>
+
 # This is supposed to be returned when the server gets a
 # Authorization: Digest line passed-in from the client
 <data1000>
@@ -107,6 +120,11 @@ Authorization: Digest username="testuser", realm="gimme all yer s3cr3ts", nonce=
 User-Agent: curl/7.10.5 (i686-pc-linux-gnu) libcurl/7.10.5 OpenSSL/0.9.7a ipv6 zlib/1.1.3
 Accept: */*
 
+GET /14120001 HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/7.10.5 (i686-pc-linux-gnu) libcurl/7.10.5 OpenSSL/0.9.7a ipv6 zlib/1.1.3
+Accept: */*
+
 GET /14120001 HTTP/1.1
 Host: %HOSTIP:%HTTPPORT
 Authorization: Digest username="testuser", realm="gimme all yer s3cr3ts", nonce="11223344", uri="/14120001", response="0085df91870374c8bf4e94415e7fbf8e"

tests/data/test1418

@@ -22,6 +22,15 @@ WWW-Authenticate: Basic
 Please auth with me
 </data>
 
+<data3>
+HTTP/1.1 401 Authentication please!
+Content-Length: 20
+WWW-Authenticate: Digest realm="loonie", nonce="314156592"
+WWW-Authenticate: Basic
+
+Please auth with me
+</data3>
+
 # This is supposed to be returned when the server gets the second
 # Authorization: NTLM line passed-in from the client
 <data1000>
@@ -97,6 +106,10 @@ Host: %HOSTIP:%HTTPPORT
 Authorization: Digest username="testuser", realm="loonie", nonce="314156592", uri="/1418", response="986238b7e0077754944c966f56d9bc77"
 Accept: */*
 
+GET /14180003 HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+Accept: */*
+
 GET /14180003 HTTP/1.1
 Host: %HOSTIP:%HTTPPORT
 Authorization: Digest username="testuser", realm="loonie", nonce="314156592", uri="/14180003", response="1c6390a67bac3283a9b023402f3b3540"

tests/data/test153

@@ -9,7 +9,7 @@ HTTP Digest auth
 
 # Server-side
 <reply>
-# reply back and ask for Digest auth
+# First reply back and ask for Digest auth
 <data1>
 HTTP/1.1 401 Authorization Required swsclose
 Server: Apache/1.3.27 (Darwin) PHP/4.1.2
@@ -20,6 +20,17 @@ Content-Length: 26
 This is not the real page
 </data1>
 
+# second reply back
+<data2>
+HTTP/1.1 401 Authorization Required swsclose
+Server: Apache/1.3.27 (Darwin) PHP/4.1.2
+WWW-Authenticate: Digest realm="testrealm", nonce="1053604145"
+Content-Type: text/html; charset=iso-8859-1
+Content-Length: 26
+
+This is not the real page
+</data2>
+
 # This is supposed to be returned when the server gets a
 # Authorization: Digest line passed-in from the client
 <data1001>
@@ -91,6 +102,11 @@ Authorization: Digest username="testuser", realm="testrealm", nonce="1053604145"
 User-Agent: curl/7.10.5 (i686-pc-linux-gnu) libcurl/7.10.5 OpenSSL/0.9.7a ipv6 zlib/1.1.3
 Accept: */*
 
+GET /1530002 HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/7.11.0-CVS (i686-pc-linux-gnu) libcurl/7.11.0-CVS OpenSSL/0.9.6b ipv6 zlib/1.1.4 GSS
+Accept: */*
+
 GET /1530002 HTTP/1.1
 Host: %HOSTIP:%HTTPPORT
 Authorization: Digest username="testuser", realm="testrealm", nonce="1053604145", uri="/1530002", response="f84511b014fdd0ba6494f42871079c32"
@@ -117,6 +133,12 @@ Content-Type: text/html; charset=iso-8859-1
 Content-Length: 23
 
 This IS the real page!
+HTTP/1.1 401 Authorization Required swsclose
+Server: Apache/1.3.27 (Darwin) PHP/4.1.2
+WWW-Authenticate: Digest realm="testrealm", nonce="1053604145"
+Content-Type: text/html; charset=iso-8859-1
+Content-Length: 26
+
 HTTP/1.1 401 Authorization re-negotiation please swsbounce
 Server: Apache/1.3.27 (Darwin) PHP/4.1.2
 WWW-Authenticate: Digest realm="testrealm", algorithm=MD5, nonce="999999", stale=true, qop="auth"

tests/data/test2006

@@ -86,10 +86,6 @@ Accept: */*
 Some data delivered from an HTTP resource
 </file1>
 <file2 name="log/heads2006">
-Content-Length: 496
-Accept-ranges: bytes
-
-
 HTTP/1.1 200 OK
 Date: Thu, 21 Jun 2012 14:49:01 GMT
 Server: test-server/fake

tests/data/test2007

@@ -90,10 +90,6 @@ Something delivered from an HTTP resource
 s/Last-Modified:.*//
 </stripfile2>
 <file2 name="log/heads2007">
-Content-Length: 496
-Accept-ranges: bytes
-
-
 HTTP/1.1 200 OK
 Date: Thu, 21 Jun 2012 14:50:02 GMT
 Server: test-server/fake

tests/data/test2008

@@ -82,10 +82,6 @@ Some stuff delivered from an HTTP resource
 s/Last-Modified:.*//
 </stripfile2>
 <file2 name="log/heads2008">
-Content-Length: 496
-Accept-ranges: bytes
-
-
 HTTP/1.1 200 OK
 Date: Thu, 21 Jun 2012 15:23:48 GMT
 Server: test-server/fake

tests/data/test2009

@@ -83,10 +83,6 @@ Some contents delivered from an HTTP resource
 s/Last-Modified:.*//
 </stripfile2>
 <file2 name="log/heads2009">
-Content-Length: 496
-Accept-ranges: bytes
-
-
 HTTP/1.1 200 OK
 Date: Thu, 21 Jun 2012 16:27:17 GMT
 Server: test-server/fake

tests/data/test2010

@@ -82,10 +82,6 @@ Contents delivered from an HTTP resource
 s/Last-Modified:.*//
 </stripfile2>
 <file2 name="log/heads2010">
-Content-Length: 496
-Accept-ranges: bytes
-
-
 HTTP/1.1 200 OK
 Date: Thu, 21 Jun 2012 17:37:27 GMT
 Server: test-server/fake