curl: show headers in bold

The feature is only enabled if the output is believed to be a tty.

-J: There's some minor differences and improvements in -J handling, as
now J should work with -i and it actually creates a file first using the
initial name and then *renames* that to the one found in
Content-Disposition (if any).

-i: only shows headers for HTTP transfers now (as documented).
Previously it would also show for pieces of the transfer that were HTTP
(for example when doing FTP over a HTTP proxy).

-i: now shows trailers as well. Previously they were not shown at all.

--libcurl: the CURLOPT_HEADER is no longer set, as the header output is
now done in the header callback.

src/tool_cb_hdr.c

@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -31,11 +31,20 @@
 #include "tool_doswin.h"
 #include "tool_msgs.h"
 #include "tool_cb_hdr.h"
+#include "tool_cb_wrt.h"
 
 #include "memdebug.h" /* keep this as LAST include */
 
 static char *parse_filename(const char *ptr, size_t len);
 
+#ifdef WIN32
+#define BOLD
+#define BOLDOFF
+#else
+#define BOLD "\x1b[1m"
+#define BOLDOFF "\x1b[21m"
+#endif
+
 /*
 ** callback for CURLOPT_HEADERFUNCTION
 */
@@ -48,7 +57,7 @@ size_t tool_header_cb(char *ptr, size_t size, size_t nmemb, void *userdata)
   const char *str = ptr;
   const size_t cb = size * nmemb;
   const char *end = (char *)ptr + cb;
-  char *url = NULL;
+  long protocol = 0;
 
   /*
    * Once that libcurl has called back tool_header_cb() the returned value
@@ -88,12 +97,15 @@ size_t tool_header_cb(char *ptr, size_t size, size_t nmemb, void *userdata)
    * Content-Disposition header specifying a filename property.
    */
 
+  curl_easy_getinfo(outs->config->easy, CURLINFO_PROTOCOL, &protocol);
   if(hdrcbdata->honor_cd_filename &&
      (cb > 20) && checkprefix("Content-disposition:", str) &&
-     !curl_easy_getinfo(outs->config->easy, CURLINFO_EFFECTIVE_URL, &url) &&
+     (protocol & (CURLPROTO_HTTPS|CURLPROTO_HTTP))) {
-     url && (checkprefix("http://", url) || checkprefix("https://", url))) {
     const char *p = str + 20;
 
+    if(!outs->stream && !tool_create_output_file(outs, FALSE))
+      return failure;
+
     /* look for the 'filename=' parameter
        (encoded filenames (*=) are not supported) */
     for(;;) {
@@ -119,19 +131,49 @@ size_t tool_header_cb(char *ptr, size_t size, size_t nmemb, void *userdata)
       len = (ssize_t)cb - (p - str);
       filename = parse_filename(p, len);
       if(filename) {
-        outs->filename = filename;
+        if(outs->stream) {
-        outs->alloc_filename = TRUE;
+          /* already opened and possibly written to */
+          if(outs->fopened)
+            fclose(outs->stream);
+          outs->stream = NULL;
+
+          /* rename the initial file name to the new file name */
+          rename(outs->filename, filename);
+          if(outs->alloc_filename)
+            free(outs->filename);
+        }
         outs->is_cd_filename = TRUE;
         outs->s_isreg = TRUE;
         outs->fopened = FALSE;
-        outs->stream = NULL;
+        outs->filename = filename;
-        hdrcbdata->honor_cd_filename = FALSE;
+        outs->alloc_filename = TRUE;
-        break;
+        hdrcbdata->honor_cd_filename = FALSE; /* done now! */
+        if(!tool_create_output_file(outs, TRUE))
+          return failure;
       }
-      return failure;
+      break;
     }
   }
 
+  if(hdrcbdata->config->show_headers &&
+     (protocol & (CURLPROTO_HTTP|CURLPROTO_HTTPS|CURLPROTO_RTSP))) {
+    /* bold headers only happen for HTTP(S) and RTSP */
+    char *value = NULL;
+
+    if(!outs->stream && !tool_create_output_file(outs, FALSE))
+      return failure;
+
+    if(hdrcbdata->global->isatty)
+      value = memchr(ptr, ':', cb);
+    if(value) {
+      size_t namelen = value - ptr;
+      fprintf(outs->stream, BOLD "%.*s" BOLDOFF ":", namelen, ptr);
+      fwrite(&value[1], cb - namelen - 1, 1, outs->stream);
+    }
+    else
+      /* not "handled", just show it */
+      fwrite(ptr, cb, 1, outs->stream);
+  }
   return cb;
 }
 
@@ -233,4 +275,3 @@ static char *parse_filename(const char *ptr, size_t len)
 
   return copy;
 }
-

src/tool_cb_hdr.h

@@ -7,7 +7,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2012, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -39,6 +39,8 @@
  */
 
 struct HdrCbData {
+  struct GlobalConfig *global;
+  struct OperationConfig *config;
   struct OutStruct *outs;
   struct OutStruct *heads;
   bool honor_cd_filename;

src/tool_cb_wrt.c

@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -32,7 +32,8 @@
 #include "memdebug.h" /* keep this as LAST include */
 
 /* create a local file for writing, return TRUE on success */
-bool tool_create_output_file(struct OutStruct *outs)
+bool tool_create_output_file(struct OutStruct *outs,
+                             bool append)
 {
   struct GlobalConfig *global = outs->config->global;
   FILE *file;
@@ -42,7 +43,7 @@ bool tool_create_output_file(struct OutStruct *outs)
     return FALSE;
   }
 
-  if(outs->is_cd_filename) {
+  if(outs->is_cd_filename && !append) {
     /* don't overwrite existing files */
     file = fopen(outs->filename, "rb");
     if(file) {
@@ -54,7 +55,7 @@ bool tool_create_output_file(struct OutStruct *outs)
   }
 
   /* open file for writing */
-  file = fopen(outs->filename, "wb");
+  file = fopen(outs->filename, append?"ab":"wb");
   if(!file) {
     warnf(global, "Failed to create the file %s: %s\n", outs->filename,
           strerror(errno));
@@ -97,7 +98,7 @@ size_t tool_write_cb(char *buffer, size_t sz, size_t nmemb, void *userdata)
     }
   }
 
-  if(config->include_headers) {
+  if(config->show_headers) {
     if(bytes > (size_t)CURL_MAX_HTTP_HEADER) {
       warnf(config->global, "Header data size exceeds single call write "
             "limit!\n");
@@ -141,7 +142,7 @@ size_t tool_write_cb(char *buffer, size_t sz, size_t nmemb, void *userdata)
   }
 #endif
 
-  if(!outs->stream && !tool_create_output_file(outs))
+  if(!outs->stream && !tool_create_output_file(outs, FALSE))
     return failure;
 
   if(is_tty && (outs->bytes < 2000) && !config->terminal_binary_ok) {

src/tool_cb_wrt.h

@@ -7,7 +7,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2012, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -30,7 +30,7 @@
 size_t tool_write_cb(char *buffer, size_t sz, size_t nmemb, void *userdata);
 
 /* create a local file for writing, return TRUE on success */
-bool tool_create_output_file(struct OutStruct *outs);
+bool tool_create_output_file(struct OutStruct *outs, bool append);
 
 #endif /* HEADER_CURL_TOOL_CB_WRT_H */
 

src/tool_cfgable.h

@@ -100,7 +100,7 @@ struct OperationConfig {
   bool use_ascii;           /* select ascii or text transfer */
   bool autoreferer;         /* automatically set referer */
   bool failonerror;         /* fail on (HTTP) errors */
-  bool include_headers;     /* send headers to data output */
+  bool show_headers;        /* show headers to data output */
   bool no_body;             /* don't get the body */
   bool dirlistonly;         /* only get the FTP dir list */
   bool followlocation;      /* follow http redirects */

src/tool_getparam.c

@@ -1722,24 +1722,22 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
       }
       break;
     case 'i':
-      config->include_headers = toggle; /* include the headers as well in the
+      config->show_headers = toggle; /* show the headers as well in the
-                                           general output stream */
+                                        general output stream */
       break;
     case 'j':
       config->cookiesession = toggle;
       break;
-    case 'I':
+    case 'I': /* --head */
-      /*
-       * no_body will imply include_headers later on
-       */
       config->no_body = toggle;
+      config->show_headers = toggle;
       if(SetHTTPrequest(config,
                         (config->no_body)?HTTPREQ_HEAD:HTTPREQ_GET,
                         &config->httpreq))
         return PARAM_BAD_USE;
       break;
     case 'J': /* --remote-header-name */
-      if(config->include_headers) {
+      if(config->show_headers) {
         warnf(global,
               "--include and --remote-header-name cannot be combined.\n");
         return PARAM_BAD_USE;

src/tool_operate.c

@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -851,15 +851,8 @@ static CURLcode operate_do(struct GlobalConfig *global,
           my_setopt(curl, CURLOPT_INFILESIZE_LARGE, uploadfilesize);
         my_setopt_str(curl, CURLOPT_URL, this_url);     /* what to fetch */
         my_setopt(curl, CURLOPT_NOPROGRESS, global->noprogress?1L:0L);
-        if(config->no_body) {
+        if(config->no_body)
           my_setopt(curl, CURLOPT_NOBODY, 1L);
-          my_setopt(curl, CURLOPT_HEADER, 1L);
-        }
-        /* If --metalink is used, we ignore --include (headers in
-           output) option because mixing headers to the body will
-           confuse XML parser and/or hash check will fail. */
-        else if(!config->use_metalink)
-          my_setopt(curl, CURLOPT_HEADER, config->include_headers?1L:0L);
 
         if(config->oauth_bearer)
           my_setopt_str(curl, CURLOPT_XOAUTH2_BEARER, config->oauth_bearer);
@@ -1373,6 +1366,8 @@ static CURLcode operate_do(struct GlobalConfig *global,
 
         hdrcbdata.outs = &outs;
         hdrcbdata.heads = &heads;
+        hdrcbdata.global = global;
+        hdrcbdata.config = config;
 
         my_setopt(curl, CURLOPT_HEADERFUNCTION, tool_header_cb);
         my_setopt(curl, CURLOPT_HEADERDATA, &hdrcbdata);
@@ -1523,7 +1518,7 @@ static CURLcode operate_do(struct GlobalConfig *global,
             /* do not create (or even overwrite) the file in case we get no
                data because of unmet condition */
             curl_easy_getinfo(curl, CURLINFO_CONDITION_UNMET, &cond_unmet);
-            if(!cond_unmet && !tool_create_output_file(&outs))
+            if(!cond_unmet && !tool_create_output_file(&outs, FALSE))
               result = CURLE_WRITE_ERROR;
           }
 

tests/data/test1116

@@ -34,6 +34,8 @@ Transfer-Encoding: chunked
 Connection: mooo
 
 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbcccccccccccccccccccccccccccccccc
+chunky-trailer: header data
+another-header: yes
 </datacheck>
 </reply>
 

tests/data/test1319

@@ -31,9 +31,6 @@ body
 </data>
 
 <datacheck>
-HTTP/1.1 200 Mighty fine indeed
-pop3: sure hit me
-
 From: me@somewhere
 To: fake@nowhere
 

tests/data/test1321

@@ -27,9 +27,6 @@ body
   yours sincerely
 </data>
 <datacheck>
-HTTP/1.1 200 Mighty fine indeed
-imap: sure hit me
-
 From: me@somewhere
 To: fake@nowhere
 

tests/data/test1400

@@ -70,7 +70,6 @@ int main(int argc, char *argv[])
   hnd = curl_easy_init();
   curl_easy_setopt(hnd, CURLOPT_BUFFERSIZE, 102400L);
   curl_easy_setopt(hnd, CURLOPT_URL, "http://%HOSTIP:%HTTPPORT/we/want/1400");
-  curl_easy_setopt(hnd, CURLOPT_HEADER, 1L);
   curl_easy_setopt(hnd, CURLOPT_USERAGENT, "stripped");
   curl_easy_setopt(hnd, CURLOPT_MAXREDIRS, 50L);
   curl_easy_setopt(hnd, CURLOPT_VERBOSE, 1L);

tests/data/test1401

@@ -82,7 +82,6 @@ int main(int argc, char *argv[])
   hnd = curl_easy_init();
   curl_easy_setopt(hnd, CURLOPT_BUFFERSIZE, 102400L);
   curl_easy_setopt(hnd, CURLOPT_URL, "http://%HOSTIP:%HTTPPORT/we/want/1401");
-  curl_easy_setopt(hnd, CURLOPT_HEADER, 1L);
   curl_easy_setopt(hnd, CURLOPT_USERPWD, "fake:user");
   curl_easy_setopt(hnd, CURLOPT_HTTPAUTH, (long)CURLAUTH_BASIC);
   curl_easy_setopt(hnd, CURLOPT_HTTPHEADER, slist1);

tests/data/test1402

@@ -75,7 +75,6 @@ int main(int argc, char *argv[])
   hnd = curl_easy_init();
   curl_easy_setopt(hnd, CURLOPT_BUFFERSIZE, 102400L);
   curl_easy_setopt(hnd, CURLOPT_URL, "http://%HOSTIP:%HTTPPORT/we/want/1402");
-  curl_easy_setopt(hnd, CURLOPT_HEADER, 1L);
   curl_easy_setopt(hnd, CURLOPT_POSTFIELDS, "foo=bar&baz=quux");
   curl_easy_setopt(hnd, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t)16);
   curl_easy_setopt(hnd, CURLOPT_USERAGENT, "stripped");

tests/data/test1403

@@ -72,7 +72,6 @@ int main(int argc, char *argv[])
   hnd = curl_easy_init();
   curl_easy_setopt(hnd, CURLOPT_BUFFERSIZE, 102400L);
   curl_easy_setopt(hnd, CURLOPT_URL, "http://%HOSTIP:%HTTPPORT/we/want/1403?foo=bar&baz=quux");
-  curl_easy_setopt(hnd, CURLOPT_HEADER, 1L);
   curl_easy_setopt(hnd, CURLOPT_USERAGENT, "stripped");
   curl_easy_setopt(hnd, CURLOPT_MAXREDIRS, 50L);
   curl_easy_setopt(hnd, CURLOPT_VERBOSE, 1L);

tests/data/test1404

@@ -121,7 +121,6 @@ int main(int argc, char *argv[])
   hnd = curl_easy_init();
   curl_easy_setopt(hnd, CURLOPT_BUFFERSIZE, 102400L);
   curl_easy_setopt(hnd, CURLOPT_URL, "http://%HOSTIP:%HTTPPORT/we/want/1404");
-  curl_easy_setopt(hnd, CURLOPT_HEADER, 1L);
   mime1 = curl_mime_init(hnd);
   part1 = curl_mime_addpart(mime1);
   curl_mime_data(part1, "value", CURL_ZERO_TERMINATED);

tests/data/test1405

@@ -85,7 +85,6 @@ int main(int argc, char *argv[])
   hnd = curl_easy_init();
   curl_easy_setopt(hnd, CURLOPT_BUFFERSIZE, 102400L);
   curl_easy_setopt(hnd, CURLOPT_URL, "ftp://%HOSTIP:%FTPPORT/1405");
-  curl_easy_setopt(hnd, CURLOPT_HEADER, 1L);
   curl_easy_setopt(hnd, CURLOPT_QUOTE, slist1);
   curl_easy_setopt(hnd, CURLOPT_POSTQUOTE, slist2);
   curl_easy_setopt(hnd, CURLOPT_PREQUOTE, slist3);

tests/data/test1406

@@ -78,7 +78,6 @@ int main(int argc, char *argv[])
   curl_easy_setopt(hnd, CURLOPT_BUFFERSIZE, 102400L);
   curl_easy_setopt(hnd, CURLOPT_INFILESIZE_LARGE, (curl_off_t)38);
   curl_easy_setopt(hnd, CURLOPT_URL, "smtp://%HOSTIP:%SMTPPORT/1406");
-  curl_easy_setopt(hnd, CURLOPT_HEADER, 1L);
   curl_easy_setopt(hnd, CURLOPT_UPLOAD, 1L);
   curl_easy_setopt(hnd, CURLOPT_VERBOSE, 1L);
   curl_easy_setopt(hnd, CURLOPT_TCP_KEEPALIVE, 1L);

tests/data/test1407

@@ -59,7 +59,6 @@ int main(int argc, char *argv[])
   hnd = curl_easy_init();
   curl_easy_setopt(hnd, CURLOPT_BUFFERSIZE, 102400L);
   curl_easy_setopt(hnd, CURLOPT_URL, "pop3://%HOSTIP:%POP3PORT/1407");
-  curl_easy_setopt(hnd, CURLOPT_HEADER, 1L);
   curl_easy_setopt(hnd, CURLOPT_DIRLISTONLY, 1L);
   curl_easy_setopt(hnd, CURLOPT_USERPWD, "user:secret");
   curl_easy_setopt(hnd, CURLOPT_VERBOSE, 1L);

tests/data/test1417

@@ -35,6 +35,7 @@ Trailer: chunky-trailer
 Connection: mooo
 
 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbcccccccccccccccccccccccccccccccc
+chunky-trailer: header data
 </datacheck>
 </reply>
 

tests/data/test1420

@@ -65,7 +65,6 @@ int main(int argc, char *argv[])
   hnd = curl_easy_init();
   curl_easy_setopt(hnd, CURLOPT_BUFFERSIZE, 102400L);
   curl_easy_setopt(hnd, CURLOPT_URL, "imap://%HOSTIP:%IMAPPORT/1420/;UID=1");
-  curl_easy_setopt(hnd, CURLOPT_HEADER, 1L);
   curl_easy_setopt(hnd, CURLOPT_USERPWD, "user:secret");
   curl_easy_setopt(hnd, CURLOPT_VERBOSE, 1L);
   curl_easy_setopt(hnd, CURLOPT_TCP_KEEPALIVE, 1L);

tests/data/test2010

@@ -35,10 +35,10 @@ Metalink
 http
 </server>
  <name>
-Metalink local XML file, HTTP resource, using -o fname -i -D file
+Metalink local XML file, HTTP resource, using -o fname -D file
  </name>
 <command option="no-output,no-include">
---metalink file://%PWD/log/test2010.metalink -i -o log/outfile2010 -D log/heads2010
+--metalink file://%PWD/log/test2010.metalink -o log/outfile2010 -D log/heads2010
 </command>
 # local metalink file written before test command runs
 <file name="log/test2010.metalink">

tests/data/test266

@@ -35,6 +35,7 @@ Trailer: chunky-trailer
 Connection: mooo
 
 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbcccccccccccccccccccccccccccccccc
+chunky-trailer: header data
 </datacheck>
 </reply>