writeout: support to generate JSON output

This commit adds support to generate JSON via the writeout feature:

    -w "%{json}"

It leverages the existing infrastructure as much as possible. Thus,
generating the JSON on STDERR is possible by:

    -w "%{stderr}%{json}"

This implements a variant of
https://github.com/curl/curl/wiki/JSON#--write-out-json.

Closes #4870

docs/cmdline-opts/write-out.d

@@ -50,6 +50,9 @@ curl CONNECT request. (Added in 7.12.4)
 .B http_version
 The http version that was effectively used. (Added in 7.50.0)
 .TP
+.B json
+A JSON object with all available keys.
+.TP
 .B local_ip
 The IP address of the local end of the most recently done connection - can be
 either IPv4 or IPv6 (Added in 7.29.0)

lib/version.c

@@ -127,6 +127,16 @@ char *curl_version(void)
   int i = 0;
   int j;
 
+#ifdef DEBUGBUILD
+  /* Override version string when environment variable CURL_VERSION is set */
+  const char *debugversion = getenv("CURL_VERSION");
+  if(debugversion) {
+    strncpy(out, debugversion, sizeof(out)-1);
+    out[sizeof(out)-1] = '\0';
+    return out;
+  }
+#endif
+
   src[i++] = LIBCURL_NAME "/" LIBCURL_VERSION;
 #ifdef USE_SSL
   Curl_ssl_version(ssl_version, sizeof(ssl_version));

packages/Symbian/group/curl.mmp

@@ -45,6 +45,7 @@ SOURCE \
     tool_vms.c \
     tool_writeenv.c \
     tool_writeout.c \
+    tool_writeout_json.c \
     tool_xattr.c
 
 SOURCEPATH  ../../../lib

src/Makefile.inc

@@ -62,6 +62,7 @@ CURL_CFILES = \
   tool_util.c \
   tool_vms.c \
   tool_writeout.c \
+  tool_writeout_json.c \
   tool_xattr.c
 
 CURL_HFILES = \
@@ -107,6 +108,7 @@ CURL_HFILES = \
   tool_version.h \
   tool_vms.h \
   tool_writeout.h \
+  tool_writeout_json.h \
   tool_xattr.h
 
 CURL_RCFILES = curl.rc

src/tool_writeout.c

@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -25,87 +25,81 @@
 #include "curlx.h"
 #include "tool_cfgable.h"
 #include "tool_writeout.h"
+#include "tool_writeout_json.h"
 
 #include "memdebug.h" /* keep this as LAST include */
 
-typedef enum {
+static const struct writeoutvar variables[] = {
-  VAR_NONE,       /* must be the first */
+  {"url_effective", VAR_EFFECTIVE_URL, 0,
-  VAR_TOTAL_TIME,
+   CURLINFO_EFFECTIVE_URL, JSON_STRING},
-  VAR_NAMELOOKUP_TIME,
+  {"http_code", VAR_HTTP_CODE, 0,
-  VAR_CONNECT_TIME,
+   CURLINFO_RESPONSE_CODE, JSON_LONG},
-  VAR_APPCONNECT_TIME,
+  {"response_code", VAR_HTTP_CODE, 0,
-  VAR_PRETRANSFER_TIME,
+   CURLINFO_RESPONSE_CODE, JSON_LONG},
-  VAR_STARTTRANSFER_TIME,
+  {"http_connect", VAR_HTTP_CODE_PROXY, 0,
-  VAR_SIZE_DOWNLOAD,
+   CURLINFO_HTTP_CONNECTCODE, JSON_LONG},
-  VAR_SIZE_UPLOAD,
+  {"time_total", VAR_TOTAL_TIME, 0,
-  VAR_SPEED_DOWNLOAD,
+   CURLINFO_TOTAL_TIME_T, JSON_TIME},
-  VAR_SPEED_UPLOAD,
+  {"time_namelookup", VAR_NAMELOOKUP_TIME, 0,
-  VAR_HTTP_CODE,
+   CURLINFO_NAMELOOKUP_TIME_T, JSON_TIME},
-  VAR_HTTP_CODE_PROXY,
+  {"time_connect", VAR_CONNECT_TIME, 0,
-  VAR_HEADER_SIZE,
+   CURLINFO_CONNECT_TIME_T, JSON_TIME},
-  VAR_REQUEST_SIZE,
+  {"time_appconnect", VAR_APPCONNECT_TIME, 0,
-  VAR_EFFECTIVE_URL,
+   CURLINFO_APPCONNECT_TIME_T, JSON_TIME},
-  VAR_CONTENT_TYPE,
+  {"time_pretransfer", VAR_PRETRANSFER_TIME, 0,
-  VAR_NUM_CONNECTS,
+   CURLINFO_PRETRANSFER_TIME_T, JSON_TIME},
-  VAR_REDIRECT_TIME,
+  {"time_starttransfer", VAR_STARTTRANSFER_TIME, 0,
-  VAR_REDIRECT_COUNT,
+   CURLINFO_STARTTRANSFER_TIME_T, JSON_TIME},
-  VAR_FTP_ENTRY_PATH,
+  {"size_header", VAR_HEADER_SIZE, 0,
-  VAR_REDIRECT_URL,
+   CURLINFO_HEADER_SIZE, JSON_LONG},
-  VAR_SSL_VERIFY_RESULT,
+  {"size_request", VAR_REQUEST_SIZE, 0,
-  VAR_PROXY_SSL_VERIFY_RESULT,
+   CURLINFO_REQUEST_SIZE, JSON_LONG},
-  VAR_EFFECTIVE_FILENAME,
+  {"size_download", VAR_SIZE_DOWNLOAD, 0,
-  VAR_PRIMARY_IP,
+   CURLINFO_SIZE_DOWNLOAD_T, JSON_LONG},
-  VAR_PRIMARY_PORT,
+  {"size_upload", VAR_SIZE_UPLOAD, 0,
-  VAR_LOCAL_IP,
+   CURLINFO_SIZE_UPLOAD_T, JSON_LONG},
-  VAR_LOCAL_PORT,
+  {"speed_download", VAR_SPEED_DOWNLOAD, 0,
-  VAR_HTTP_VERSION,
+   CURLINFO_SPEED_DOWNLOAD_T, JSON_TIME},
-  VAR_SCHEME,
+  {"speed_upload", VAR_SPEED_UPLOAD, 0,
-  VAR_STDOUT,
+   CURLINFO_SPEED_UPLOAD_T, JSON_TIME},
-  VAR_STDERR,
+  {"content_type", VAR_CONTENT_TYPE, 0,
-  VAR_NUM_OF_VARS /* must be the last */
+   CURLINFO_CONTENT_TYPE, JSON_STRING},
-} replaceid;
+  {"num_connects", VAR_NUM_CONNECTS, 0,
-
+   CURLINFO_NUM_CONNECTS, JSON_LONG},
-struct variable {
+  {"time_redirect", VAR_REDIRECT_TIME, 0,
-  const char *name;
+   CURLINFO_REDIRECT_TIME_T, JSON_TIME},
-  replaceid id;
+  {"num_redirects", VAR_REDIRECT_COUNT, 0,
-};
+   CURLINFO_REDIRECT_COUNT, JSON_LONG},
-
+  {"ftp_entry_path", VAR_FTP_ENTRY_PATH, 0,
-
+   CURLINFO_FTP_ENTRY_PATH, JSON_STRING},
-static const struct variable replacements[]={
+  {"redirect_url", VAR_REDIRECT_URL, 0,
-  {"url_effective", VAR_EFFECTIVE_URL},
+   CURLINFO_REDIRECT_URL, JSON_STRING},
-  {"http_code", VAR_HTTP_CODE},
+  {"ssl_verify_result", VAR_SSL_VERIFY_RESULT, 0,
-  {"response_code", VAR_HTTP_CODE},
+   CURLINFO_SSL_VERIFYRESULT, JSON_LONG},
-  {"http_connect", VAR_HTTP_CODE_PROXY},
+  {"proxy_ssl_verify_result", VAR_PROXY_SSL_VERIFY_RESULT, 0,
-  {"time_total", VAR_TOTAL_TIME},
+   CURLINFO_PROXY_SSL_VERIFYRESULT, JSON_LONG},
-  {"time_namelookup", VAR_NAMELOOKUP_TIME},
+  {"filename_effective", VAR_EFFECTIVE_FILENAME, 0,
-  {"time_connect", VAR_CONNECT_TIME},
+   0, JSON_FILENAME},
-  {"time_appconnect", VAR_APPCONNECT_TIME},
+  {"remote_ip", VAR_PRIMARY_IP, 0,
-  {"time_pretransfer", VAR_PRETRANSFER_TIME},
+   CURLINFO_PRIMARY_IP, JSON_STRING},
-  {"time_starttransfer", VAR_STARTTRANSFER_TIME},
+  {"remote_port", VAR_PRIMARY_PORT, 0,
-  {"size_header", VAR_HEADER_SIZE},
+   CURLINFO_PRIMARY_PORT, JSON_LONG},
-  {"size_request", VAR_REQUEST_SIZE},
+  {"local_ip", VAR_LOCAL_IP, 0,
-  {"size_download", VAR_SIZE_DOWNLOAD},
+   CURLINFO_LOCAL_IP, JSON_STRING},
-  {"size_upload", VAR_SIZE_UPLOAD},
+  {"local_port", VAR_LOCAL_PORT, 0,
-  {"speed_download", VAR_SPEED_DOWNLOAD},
+   CURLINFO_LOCAL_PORT, JSON_LONG},
-  {"speed_upload", VAR_SPEED_UPLOAD},
+  {"http_version", VAR_HTTP_VERSION, 0,
-  {"content_type", VAR_CONTENT_TYPE},
+   CURLINFO_HTTP_VERSION, JSON_VERSION},
-  {"num_connects", VAR_NUM_CONNECTS},
+  {"scheme", VAR_SCHEME, 0,
-  {"time_redirect", VAR_REDIRECT_TIME},
+   CURLINFO_SCHEME, JSON_STRING},
-  {"num_redirects", VAR_REDIRECT_COUNT},
+  {"stdout", VAR_STDOUT, 1,
-  {"ftp_entry_path", VAR_FTP_ENTRY_PATH},
+   0, JSON_NONE},
-  {"redirect_url", VAR_REDIRECT_URL},
+  {"stderr", VAR_STDERR, 1,
-  {"ssl_verify_result", VAR_SSL_VERIFY_RESULT},
+   0, JSON_NONE},
-  {"proxy_ssl_verify_result", VAR_PROXY_SSL_VERIFY_RESULT},
+  {"json", VAR_JSON, 1,
-  {"filename_effective", VAR_EFFECTIVE_FILENAME},
+   0, JSON_NONE},
-  {"remote_ip", VAR_PRIMARY_IP},
+  {NULL, VAR_NONE, 1,
-  {"remote_port", VAR_PRIMARY_PORT},
+   0, JSON_NONE}
-  {"local_ip", VAR_LOCAL_IP},
-  {"local_port", VAR_LOCAL_PORT},
-  {"http_version", VAR_HTTP_VERSION},
-  {"scheme", VAR_SCHEME},
-  {"stdout", VAR_STDOUT},
-  {"stderr", VAR_STDERR},
-  {NULL, VAR_NONE}
 };
 
 void ourWriteOut(CURL *curl, struct OutStruct *outs, const char *writeinfo)
@@ -138,10 +132,10 @@ void ourWriteOut(CURL *curl, struct OutStruct *outs, const char *writeinfo)
           }
           keepit = *end;
           *end = 0; /* zero terminate */
-          for(i = 0; replacements[i].name; i++) {
+          for(i = 0; variables[i].name; i++) {
-            if(curl_strequal(ptr, replacements[i].name)) {
+            if(curl_strequal(ptr, variables[i].name)) {
               match = TRUE;
-              switch(replacements[i].id) {
+              switch(variables[i].id) {
               case VAR_EFFECTIVE_URL:
                 if((CURLE_OK ==
                     curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &stringp))
@@ -334,6 +328,8 @@ void ourWriteOut(CURL *curl, struct OutStruct *outs, const char *writeinfo)
               case VAR_STDERR:
                 stream = stderr;
                 break;
+              case VAR_JSON:
+                ourWriteOutJSON(variables, curl, outs, stream);
               default:
                 break;
               }

src/tool_writeout.h

@@ -7,7 +7,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -23,6 +23,61 @@
  ***************************************************************************/
 #include "tool_setup.h"
 
+typedef enum {
+  VAR_NONE,       /* must be the first */
+  VAR_TOTAL_TIME,
+  VAR_NAMELOOKUP_TIME,
+  VAR_CONNECT_TIME,
+  VAR_APPCONNECT_TIME,
+  VAR_PRETRANSFER_TIME,
+  VAR_STARTTRANSFER_TIME,
+  VAR_SIZE_DOWNLOAD,
+  VAR_SIZE_UPLOAD,
+  VAR_SPEED_DOWNLOAD,
+  VAR_SPEED_UPLOAD,
+  VAR_HTTP_CODE,
+  VAR_HTTP_CODE_PROXY,
+  VAR_HEADER_SIZE,
+  VAR_REQUEST_SIZE,
+  VAR_EFFECTIVE_URL,
+  VAR_CONTENT_TYPE,
+  VAR_NUM_CONNECTS,
+  VAR_REDIRECT_TIME,
+  VAR_REDIRECT_COUNT,
+  VAR_FTP_ENTRY_PATH,
+  VAR_REDIRECT_URL,
+  VAR_SSL_VERIFY_RESULT,
+  VAR_PROXY_SSL_VERIFY_RESULT,
+  VAR_EFFECTIVE_FILENAME,
+  VAR_PRIMARY_IP,
+  VAR_PRIMARY_PORT,
+  VAR_LOCAL_IP,
+  VAR_LOCAL_PORT,
+  VAR_HTTP_VERSION,
+  VAR_SCHEME,
+  VAR_STDOUT,
+  VAR_STDERR,
+  VAR_JSON,
+  VAR_NUM_OF_VARS /* must be the last */
+} writeoutid;
+
+typedef enum {
+  JSON_NONE,
+  JSON_STRING,
+  JSON_LONG,
+  JSON_TIME,
+  JSON_VERSION,
+  JSON_FILENAME
+} jsontype;
+
+struct writeoutvar {
+  const char *name;
+  writeoutid id;
+  int is_ctrl;
+  CURLINFO cinfo;
+  jsontype jsontype;
+};
+
 void ourWriteOut(CURL *curl, struct OutStruct *outs, const char *writeinfo);
 
 #endif /* HEADER_CURL_TOOL_WRITEOUT_H */

src/tool_writeout_json.c

@@ -0,0 +1,181 @@
+/***************************************************************************
+ *                                  _   _ ____  _
+ *  Project                     ___| | | |  _ \| |
+ *                             / __| | | | |_) | |
+ *                            | (__| |_| |  _ <| |___
+ *                             \___|\___/|_| \_\_____|
+ *
+ * Copyright (C) 1998 - 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
+ *
+ * This software is licensed as described in the file COPYING, which
+ * you should have received as part of this distribution. The terms
+ * are also available at https://curl.haxx.se/docs/copyright.html.
+ *
+ * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+ * copies of the Software, and permit persons to whom the Software is
+ * furnished to do so, under the terms of the COPYING file.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ ***************************************************************************/
+#include "tool_setup.h"
+
+#define ENABLE_CURLX_PRINTF
+
+/* use our own printf() functions */
+#include "curlx.h"
+#include "tool_cfgable.h"
+#include "tool_writeout_json.h"
+#include "tool_writeout.h"
+
+
+static const char *http_version[] = {
+  "0",   /* CURL_HTTP_VERSION_NONE */
+  "1",   /* CURL_HTTP_VERSION_1_0 */
+  "1.1", /* CURL_HTTP_VERSION_1_1 */
+  "2"    /* CURL_HTTP_VERSION_2 */
+  "3"    /* CURL_HTTP_VERSION_3 */
+};
+
+static void jsonEscape(FILE *stream, const char *in)
+{
+  const char *i = in;
+  const char *in_end = in + strlen(in);
+
+  for(; i < in_end; i++) {
+    switch(*i) {
+    case '\\':
+      fputs("\\\\", stream);
+      break;
+    case '\"':
+      fputs("\\\"", stream);
+      break;
+    case '\b':
+      fputs("\\b", stream);
+      break;
+    case '\f':
+      fputs("\\f", stream);
+      break;
+    case '\n':
+      fputs("\\n", stream);
+      break;
+    case '\r':
+      fputs("\\r", stream);
+      break;
+    case '\t':
+      fputs("\\t", stream);
+      break;
+    default:
+      if (*i < 32) {
+        fprintf(stream, "u%04x", *i);
+      }
+      else {
+        fputc(*i, stream);
+      }
+      break;
+    };
+  }
+}
+
+static int writeTime(FILE *str, CURL *curl, const char *key, CURLINFO ci)
+{
+  curl_off_t val = 0;
+  if(CURLE_OK == curl_easy_getinfo(curl, ci, &val)) {
+    curl_off_t s = val / 1000000l;
+    curl_off_t ms = val % 1000000l;
+    fprintf(str, "\"%s\":%ld.%06ld", key, s, ms);
+    return 1;
+  }
+  return 0;
+}
+
+static int writeString(FILE *str, CURL *curl, const char *key, CURLINFO ci)
+{
+  char *valp = NULL;
+  if((CURLE_OK == curl_easy_getinfo(curl, ci, &valp)) && valp) {
+    fprintf(str, "\"%s\":\"", key);
+    jsonEscape(str, valp);
+    fprintf(str, "\"");
+    return 1;
+  }
+  return 0;
+}
+
+static int writeLong(FILE *str, CURL *curl, const char *key, CURLINFO ci)
+{
+  curl_off_t val = 0;
+  if(CURLE_OK == curl_easy_getinfo(curl, ci, &val)) {
+    fprintf(str, "\"%s\":%ld", key, val);
+    return 1;
+  }
+  return 0;
+}
+
+static int writeFilename(FILE *str, const char *key, const char *filename)
+{
+  if(filename) {
+    fprintf(str, "\"%s\":\"", key);
+    jsonEscape(str, filename);
+    fprintf(str, "\"");
+  }
+  else {
+    fprintf(str, "\"%s\":null", key);
+  }
+  return 1;
+}
+
+static int writeVersion(FILE *str, CURL *curl, const char *key, CURLINFO ci)
+{
+  long version = 0;
+  if(CURLE_OK == curl_easy_getinfo(curl, ci, &version) &&
+     (version >= 0) &&
+     (version < (long)(sizeof(http_version)/sizeof(char *)))) {
+    fprintf(str, "\"%s\":\"%s\"", key, http_version[version]);
+    return 1;
+  }
+  return 0;
+}
+
+void ourWriteOutJSON(const struct writeoutvar mappings[], CURL *curl,
+        struct OutStruct *outs, FILE *stream)
+{
+  int i;
+
+  fputs("{", stream);
+  for(i = 0; mappings[i].name != NULL; i++) {
+    const char *name = mappings[i].name;
+    CURLINFO cinfo = mappings[i].cinfo;
+    int ok = 0;
+
+    if(mappings[i].is_ctrl == 1) {
+      continue;
+    }
+
+    switch(mappings[i].jsontype) {
+    case JSON_STRING:
+      ok = writeString(stream, curl, name, cinfo);
+      break;
+    case JSON_LONG:
+      ok = writeLong(stream, curl, name, cinfo);
+      break;
+    case JSON_TIME:
+      ok = writeTime(stream, curl, name, cinfo);
+      break;
+    case JSON_FILENAME:
+      ok = writeFilename(stream, name, outs->filename);
+      break;
+    case JSON_VERSION:
+      ok = writeVersion(stream, curl, name, cinfo);
+      break;
+    default:
+      break;
+    }
+
+    if(ok) {
+      fputs(",", stream);
+    }
+  }
+
+  fprintf(stream, "\"curl_version\":\"%s\"}", curl_version());
+}

src/tool_writeout_json.h

@@ -0,0 +1,30 @@
+#ifndef HEADER_CURL_TOOL_WRITEOUT_JSON_H
+#define HEADER_CURL_TOOL_WRITEOUT_JSON_H
+/***************************************************************************
+ *                                  _   _ ____  _
+ *  Project                     ___| | | |  _ \| |
+ *                             / __| | | | |_) | |
+ *                            | (__| |_| |  _ <| |___
+ *                             \___|\___/|_| \_\_____|
+ *
+ * Copyright (C) 1998 - 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
+ *
+ * This software is licensed as described in the file COPYING, which
+ * you should have received as part of this distribution. The terms
+ * are also available at https://curl.haxx.se/docs/copyright.html.
+ *
+ * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+ * copies of the Software, and permit persons to whom the Software is
+ * furnished to do so, under the terms of the COPYING file.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ ***************************************************************************/
+#include "tool_setup.h"
+#include "tool_writeout.h"
+
+void ourWriteOutJSON(const struct writeoutvar mappings[],
+        CURL *curl, struct OutStruct *outs, FILE *stream);
+
+#endif /* HEADER_CURL_TOOL_WRITEOUT_H */