From 04c03416e68fd635a15cae8201872f5c29fdcca8 Mon Sep 17 00:00:00 2001 From: Mathias Gumz Date: Sat, 1 Feb 2020 18:55:24 +0100 Subject: [PATCH] writeout: support to generate JSON output This commit adds support to generate JSON via the writeout feature: -w "%{json}" It leverages the existing infrastructure as much as possible. Thus, generating the JSON on STDERR is possible by: -w "%{stderr}%{json}" This implements a variant of https://github.com/curl/curl/wiki/JSON#--write-out-json. Closes #4870 --- docs/cmdline-opts/write-out.d | 3 + lib/version.c | 10 ++ packages/Symbian/group/curl.mmp | 1 + src/Makefile.inc | 2 + src/tool_writeout.c | 160 ++++++++++++++-------------- src/tool_writeout.h | 57 +++++++++- src/tool_writeout_json.c | 181 ++++++++++++++++++++++++++++++++ src/tool_writeout_json.h | 30 ++++++ 8 files changed, 361 insertions(+), 83 deletions(-) create mode 100644 src/tool_writeout_json.c create mode 100644 src/tool_writeout_json.h diff --git a/docs/cmdline-opts/write-out.d b/docs/cmdline-opts/write-out.d index 2fc0ff21a..9024d32e0 100644 --- a/docs/cmdline-opts/write-out.d +++ b/docs/cmdline-opts/write-out.d @@ -50,6 +50,9 @@ curl CONNECT request. (Added in 7.12.4) .B http_version The http version that was effectively used. (Added in 7.50.0) .TP +.B json +A JSON object with all available keys. +.TP .B local_ip The IP address of the local end of the most recently done connection - can be either IPv4 or IPv6 (Added in 7.29.0) diff --git a/lib/version.c b/lib/version.c index 8170106ee..c7fe42a07 100644 --- a/lib/version.c +++ b/lib/version.c @@ -127,6 +127,16 @@ char *curl_version(void) int i = 0; int j; +#ifdef DEBUGBUILD + /* Override version string when environment variable CURL_VERSION is set */ + const char *debugversion = getenv("CURL_VERSION"); + if(debugversion) { + strncpy(out, debugversion, sizeof(out)-1); + out[sizeof(out)-1] = '\0'; + return out; + } +#endif + src[i++] = LIBCURL_NAME "/" LIBCURL_VERSION; #ifdef USE_SSL Curl_ssl_version(ssl_version, sizeof(ssl_version)); diff --git a/packages/Symbian/group/curl.mmp b/packages/Symbian/group/curl.mmp index 14ccf1419..361e0a620 100644 --- a/packages/Symbian/group/curl.mmp +++ b/packages/Symbian/group/curl.mmp @@ -45,6 +45,7 @@ SOURCE \ tool_vms.c \ tool_writeenv.c \ tool_writeout.c \ + tool_writeout_json.c \ tool_xattr.c SOURCEPATH ../../../lib diff --git a/src/Makefile.inc b/src/Makefile.inc index dd6b9d336..ac96f6514 100644 --- a/src/Makefile.inc +++ b/src/Makefile.inc @@ -62,6 +62,7 @@ CURL_CFILES = \ tool_util.c \ tool_vms.c \ tool_writeout.c \ + tool_writeout_json.c \ tool_xattr.c CURL_HFILES = \ @@ -107,6 +108,7 @@ CURL_HFILES = \ tool_version.h \ tool_vms.h \ tool_writeout.h \ + tool_writeout_json.h \ tool_xattr.h CURL_RCFILES = curl.rc diff --git a/src/tool_writeout.c b/src/tool_writeout.c index 27b2ac50d..9fbc8665d 100644 --- a/src/tool_writeout.c +++ b/src/tool_writeout.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2019, Daniel Stenberg, , et al. + * Copyright (C) 1998 - 2020, Daniel Stenberg, , et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -25,87 +25,81 @@ #include "curlx.h" #include "tool_cfgable.h" #include "tool_writeout.h" +#include "tool_writeout_json.h" #include "memdebug.h" /* keep this as LAST include */ -typedef enum { - VAR_NONE, /* must be the first */ - VAR_TOTAL_TIME, - VAR_NAMELOOKUP_TIME, - VAR_CONNECT_TIME, - VAR_APPCONNECT_TIME, - VAR_PRETRANSFER_TIME, - VAR_STARTTRANSFER_TIME, - VAR_SIZE_DOWNLOAD, - VAR_SIZE_UPLOAD, - VAR_SPEED_DOWNLOAD, - VAR_SPEED_UPLOAD, - VAR_HTTP_CODE, - VAR_HTTP_CODE_PROXY, - VAR_HEADER_SIZE, - VAR_REQUEST_SIZE, - VAR_EFFECTIVE_URL, - VAR_CONTENT_TYPE, - VAR_NUM_CONNECTS, - VAR_REDIRECT_TIME, - VAR_REDIRECT_COUNT, - VAR_FTP_ENTRY_PATH, - VAR_REDIRECT_URL, - VAR_SSL_VERIFY_RESULT, - VAR_PROXY_SSL_VERIFY_RESULT, - VAR_EFFECTIVE_FILENAME, - VAR_PRIMARY_IP, - VAR_PRIMARY_PORT, - VAR_LOCAL_IP, - VAR_LOCAL_PORT, - VAR_HTTP_VERSION, - VAR_SCHEME, - VAR_STDOUT, - VAR_STDERR, - VAR_NUM_OF_VARS /* must be the last */ -} replaceid; - -struct variable { - const char *name; - replaceid id; -}; - - -static const struct variable replacements[]={ - {"url_effective", VAR_EFFECTIVE_URL}, - {"http_code", VAR_HTTP_CODE}, - {"response_code", VAR_HTTP_CODE}, - {"http_connect", VAR_HTTP_CODE_PROXY}, - {"time_total", VAR_TOTAL_TIME}, - {"time_namelookup", VAR_NAMELOOKUP_TIME}, - {"time_connect", VAR_CONNECT_TIME}, - {"time_appconnect", VAR_APPCONNECT_TIME}, - {"time_pretransfer", VAR_PRETRANSFER_TIME}, - {"time_starttransfer", VAR_STARTTRANSFER_TIME}, - {"size_header", VAR_HEADER_SIZE}, - {"size_request", VAR_REQUEST_SIZE}, - {"size_download", VAR_SIZE_DOWNLOAD}, - {"size_upload", VAR_SIZE_UPLOAD}, - {"speed_download", VAR_SPEED_DOWNLOAD}, - {"speed_upload", VAR_SPEED_UPLOAD}, - {"content_type", VAR_CONTENT_TYPE}, - {"num_connects", VAR_NUM_CONNECTS}, - {"time_redirect", VAR_REDIRECT_TIME}, - {"num_redirects", VAR_REDIRECT_COUNT}, - {"ftp_entry_path", VAR_FTP_ENTRY_PATH}, - {"redirect_url", VAR_REDIRECT_URL}, - {"ssl_verify_result", VAR_SSL_VERIFY_RESULT}, - {"proxy_ssl_verify_result", VAR_PROXY_SSL_VERIFY_RESULT}, - {"filename_effective", VAR_EFFECTIVE_FILENAME}, - {"remote_ip", VAR_PRIMARY_IP}, - {"remote_port", VAR_PRIMARY_PORT}, - {"local_ip", VAR_LOCAL_IP}, - {"local_port", VAR_LOCAL_PORT}, - {"http_version", VAR_HTTP_VERSION}, - {"scheme", VAR_SCHEME}, - {"stdout", VAR_STDOUT}, - {"stderr", VAR_STDERR}, - {NULL, VAR_NONE} +static const struct writeoutvar variables[] = { + {"url_effective", VAR_EFFECTIVE_URL, 0, + CURLINFO_EFFECTIVE_URL, JSON_STRING}, + {"http_code", VAR_HTTP_CODE, 0, + CURLINFO_RESPONSE_CODE, JSON_LONG}, + {"response_code", VAR_HTTP_CODE, 0, + CURLINFO_RESPONSE_CODE, JSON_LONG}, + {"http_connect", VAR_HTTP_CODE_PROXY, 0, + CURLINFO_HTTP_CONNECTCODE, JSON_LONG}, + {"time_total", VAR_TOTAL_TIME, 0, + CURLINFO_TOTAL_TIME_T, JSON_TIME}, + {"time_namelookup", VAR_NAMELOOKUP_TIME, 0, + CURLINFO_NAMELOOKUP_TIME_T, JSON_TIME}, + {"time_connect", VAR_CONNECT_TIME, 0, + CURLINFO_CONNECT_TIME_T, JSON_TIME}, + {"time_appconnect", VAR_APPCONNECT_TIME, 0, + CURLINFO_APPCONNECT_TIME_T, JSON_TIME}, + {"time_pretransfer", VAR_PRETRANSFER_TIME, 0, + CURLINFO_PRETRANSFER_TIME_T, JSON_TIME}, + {"time_starttransfer", VAR_STARTTRANSFER_TIME, 0, + CURLINFO_STARTTRANSFER_TIME_T, JSON_TIME}, + {"size_header", VAR_HEADER_SIZE, 0, + CURLINFO_HEADER_SIZE, JSON_LONG}, + {"size_request", VAR_REQUEST_SIZE, 0, + CURLINFO_REQUEST_SIZE, JSON_LONG}, + {"size_download", VAR_SIZE_DOWNLOAD, 0, + CURLINFO_SIZE_DOWNLOAD_T, JSON_LONG}, + {"size_upload", VAR_SIZE_UPLOAD, 0, + CURLINFO_SIZE_UPLOAD_T, JSON_LONG}, + {"speed_download", VAR_SPEED_DOWNLOAD, 0, + CURLINFO_SPEED_DOWNLOAD_T, JSON_TIME}, + {"speed_upload", VAR_SPEED_UPLOAD, 0, + CURLINFO_SPEED_UPLOAD_T, JSON_TIME}, + {"content_type", VAR_CONTENT_TYPE, 0, + CURLINFO_CONTENT_TYPE, JSON_STRING}, + {"num_connects", VAR_NUM_CONNECTS, 0, + CURLINFO_NUM_CONNECTS, JSON_LONG}, + {"time_redirect", VAR_REDIRECT_TIME, 0, + CURLINFO_REDIRECT_TIME_T, JSON_TIME}, + {"num_redirects", VAR_REDIRECT_COUNT, 0, + CURLINFO_REDIRECT_COUNT, JSON_LONG}, + {"ftp_entry_path", VAR_FTP_ENTRY_PATH, 0, + CURLINFO_FTP_ENTRY_PATH, JSON_STRING}, + {"redirect_url", VAR_REDIRECT_URL, 0, + CURLINFO_REDIRECT_URL, JSON_STRING}, + {"ssl_verify_result", VAR_SSL_VERIFY_RESULT, 0, + CURLINFO_SSL_VERIFYRESULT, JSON_LONG}, + {"proxy_ssl_verify_result", VAR_PROXY_SSL_VERIFY_RESULT, 0, + CURLINFO_PROXY_SSL_VERIFYRESULT, JSON_LONG}, + {"filename_effective", VAR_EFFECTIVE_FILENAME, 0, + 0, JSON_FILENAME}, + {"remote_ip", VAR_PRIMARY_IP, 0, + CURLINFO_PRIMARY_IP, JSON_STRING}, + {"remote_port", VAR_PRIMARY_PORT, 0, + CURLINFO_PRIMARY_PORT, JSON_LONG}, + {"local_ip", VAR_LOCAL_IP, 0, + CURLINFO_LOCAL_IP, JSON_STRING}, + {"local_port", VAR_LOCAL_PORT, 0, + CURLINFO_LOCAL_PORT, JSON_LONG}, + {"http_version", VAR_HTTP_VERSION, 0, + CURLINFO_HTTP_VERSION, JSON_VERSION}, + {"scheme", VAR_SCHEME, 0, + CURLINFO_SCHEME, JSON_STRING}, + {"stdout", VAR_STDOUT, 1, + 0, JSON_NONE}, + {"stderr", VAR_STDERR, 1, + 0, JSON_NONE}, + {"json", VAR_JSON, 1, + 0, JSON_NONE}, + {NULL, VAR_NONE, 1, + 0, JSON_NONE} }; void ourWriteOut(CURL *curl, struct OutStruct *outs, const char *writeinfo) @@ -138,10 +132,10 @@ void ourWriteOut(CURL *curl, struct OutStruct *outs, const char *writeinfo) } keepit = *end; *end = 0; /* zero terminate */ - for(i = 0; replacements[i].name; i++) { - if(curl_strequal(ptr, replacements[i].name)) { + for(i = 0; variables[i].name; i++) { + if(curl_strequal(ptr, variables[i].name)) { match = TRUE; - switch(replacements[i].id) { + switch(variables[i].id) { case VAR_EFFECTIVE_URL: if((CURLE_OK == curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &stringp)) @@ -334,6 +328,8 @@ void ourWriteOut(CURL *curl, struct OutStruct *outs, const char *writeinfo) case VAR_STDERR: stream = stderr; break; + case VAR_JSON: + ourWriteOutJSON(variables, curl, outs, stream); default: break; } diff --git a/src/tool_writeout.h b/src/tool_writeout.h index ee8990f77..64d759575 100644 --- a/src/tool_writeout.h +++ b/src/tool_writeout.h @@ -7,7 +7,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2019, Daniel Stenberg, , et al. + * Copyright (C) 1998 - 2020, Daniel Stenberg, , et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -23,6 +23,61 @@ ***************************************************************************/ #include "tool_setup.h" +typedef enum { + VAR_NONE, /* must be the first */ + VAR_TOTAL_TIME, + VAR_NAMELOOKUP_TIME, + VAR_CONNECT_TIME, + VAR_APPCONNECT_TIME, + VAR_PRETRANSFER_TIME, + VAR_STARTTRANSFER_TIME, + VAR_SIZE_DOWNLOAD, + VAR_SIZE_UPLOAD, + VAR_SPEED_DOWNLOAD, + VAR_SPEED_UPLOAD, + VAR_HTTP_CODE, + VAR_HTTP_CODE_PROXY, + VAR_HEADER_SIZE, + VAR_REQUEST_SIZE, + VAR_EFFECTIVE_URL, + VAR_CONTENT_TYPE, + VAR_NUM_CONNECTS, + VAR_REDIRECT_TIME, + VAR_REDIRECT_COUNT, + VAR_FTP_ENTRY_PATH, + VAR_REDIRECT_URL, + VAR_SSL_VERIFY_RESULT, + VAR_PROXY_SSL_VERIFY_RESULT, + VAR_EFFECTIVE_FILENAME, + VAR_PRIMARY_IP, + VAR_PRIMARY_PORT, + VAR_LOCAL_IP, + VAR_LOCAL_PORT, + VAR_HTTP_VERSION, + VAR_SCHEME, + VAR_STDOUT, + VAR_STDERR, + VAR_JSON, + VAR_NUM_OF_VARS /* must be the last */ +} writeoutid; + +typedef enum { + JSON_NONE, + JSON_STRING, + JSON_LONG, + JSON_TIME, + JSON_VERSION, + JSON_FILENAME +} jsontype; + +struct writeoutvar { + const char *name; + writeoutid id; + int is_ctrl; + CURLINFO cinfo; + jsontype jsontype; +}; + void ourWriteOut(CURL *curl, struct OutStruct *outs, const char *writeinfo); #endif /* HEADER_CURL_TOOL_WRITEOUT_H */ diff --git a/src/tool_writeout_json.c b/src/tool_writeout_json.c new file mode 100644 index 000000000..99252ecb8 --- /dev/null +++ b/src/tool_writeout_json.c @@ -0,0 +1,181 @@ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2020, Daniel Stenberg, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ +#include "tool_setup.h" + +#define ENABLE_CURLX_PRINTF + +/* use our own printf() functions */ +#include "curlx.h" +#include "tool_cfgable.h" +#include "tool_writeout_json.h" +#include "tool_writeout.h" + + +static const char *http_version[] = { + "0", /* CURL_HTTP_VERSION_NONE */ + "1", /* CURL_HTTP_VERSION_1_0 */ + "1.1", /* CURL_HTTP_VERSION_1_1 */ + "2" /* CURL_HTTP_VERSION_2 */ + "3" /* CURL_HTTP_VERSION_3 */ +}; + +static void jsonEscape(FILE *stream, const char *in) +{ + const char *i = in; + const char *in_end = in + strlen(in); + + for(; i < in_end; i++) { + switch(*i) { + case '\\': + fputs("\\\\", stream); + break; + case '\"': + fputs("\\\"", stream); + break; + case '\b': + fputs("\\b", stream); + break; + case '\f': + fputs("\\f", stream); + break; + case '\n': + fputs("\\n", stream); + break; + case '\r': + fputs("\\r", stream); + break; + case '\t': + fputs("\\t", stream); + break; + default: + if (*i < 32) { + fprintf(stream, "u%04x", *i); + } + else { + fputc(*i, stream); + } + break; + }; + } +} + +static int writeTime(FILE *str, CURL *curl, const char *key, CURLINFO ci) +{ + curl_off_t val = 0; + if(CURLE_OK == curl_easy_getinfo(curl, ci, &val)) { + curl_off_t s = val / 1000000l; + curl_off_t ms = val % 1000000l; + fprintf(str, "\"%s\":%ld.%06ld", key, s, ms); + return 1; + } + return 0; +} + +static int writeString(FILE *str, CURL *curl, const char *key, CURLINFO ci) +{ + char *valp = NULL; + if((CURLE_OK == curl_easy_getinfo(curl, ci, &valp)) && valp) { + fprintf(str, "\"%s\":\"", key); + jsonEscape(str, valp); + fprintf(str, "\""); + return 1; + } + return 0; +} + +static int writeLong(FILE *str, CURL *curl, const char *key, CURLINFO ci) +{ + curl_off_t val = 0; + if(CURLE_OK == curl_easy_getinfo(curl, ci, &val)) { + fprintf(str, "\"%s\":%ld", key, val); + return 1; + } + return 0; +} + +static int writeFilename(FILE *str, const char *key, const char *filename) +{ + if(filename) { + fprintf(str, "\"%s\":\"", key); + jsonEscape(str, filename); + fprintf(str, "\""); + } + else { + fprintf(str, "\"%s\":null", key); + } + return 1; +} + +static int writeVersion(FILE *str, CURL *curl, const char *key, CURLINFO ci) +{ + long version = 0; + if(CURLE_OK == curl_easy_getinfo(curl, ci, &version) && + (version >= 0) && + (version < (long)(sizeof(http_version)/sizeof(char *)))) { + fprintf(str, "\"%s\":\"%s\"", key, http_version[version]); + return 1; + } + return 0; +} + +void ourWriteOutJSON(const struct writeoutvar mappings[], CURL *curl, + struct OutStruct *outs, FILE *stream) +{ + int i; + + fputs("{", stream); + for(i = 0; mappings[i].name != NULL; i++) { + const char *name = mappings[i].name; + CURLINFO cinfo = mappings[i].cinfo; + int ok = 0; + + if(mappings[i].is_ctrl == 1) { + continue; + } + + switch(mappings[i].jsontype) { + case JSON_STRING: + ok = writeString(stream, curl, name, cinfo); + break; + case JSON_LONG: + ok = writeLong(stream, curl, name, cinfo); + break; + case JSON_TIME: + ok = writeTime(stream, curl, name, cinfo); + break; + case JSON_FILENAME: + ok = writeFilename(stream, name, outs->filename); + break; + case JSON_VERSION: + ok = writeVersion(stream, curl, name, cinfo); + break; + default: + break; + } + + if(ok) { + fputs(",", stream); + } + } + + fprintf(stream, "\"curl_version\":\"%s\"}", curl_version()); +} diff --git a/src/tool_writeout_json.h b/src/tool_writeout_json.h new file mode 100644 index 000000000..d3988d521 --- /dev/null +++ b/src/tool_writeout_json.h @@ -0,0 +1,30 @@ +#ifndef HEADER_CURL_TOOL_WRITEOUT_JSON_H +#define HEADER_CURL_TOOL_WRITEOUT_JSON_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2020, Daniel Stenberg, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ +#include "tool_setup.h" +#include "tool_writeout.h" + +void ourWriteOutJSON(const struct writeoutvar mappings[], + CURL *curl, struct OutStruct *outs, FILE *stream); + +#endif /* HEADER_CURL_TOOL_WRITEOUT_H */