From b7b4dc0d49543175ab0d9bb1cdc257a2d7f7cf0a Mon Sep 17 00:00:00 2001 From: Viktor Szakats Date: Sat, 5 Aug 2017 09:26:04 +0000 Subject: [PATCH] ssh: add the ability to enable compression (for SCP/SFTP) The required low-level logic was already available as part of `libssh2` (via `LIBSSH2_FLAG_COMPRESS` `libssh2_session_flag()`[1] option.) This patch adds the new `libcurl` option `CURLOPT_SSH_COMPRESSION` (boolean) and the new `curl` command-line option `--compressed-ssh` to request this `libssh2` feature. To have compression enabled, it is required that the SSH server supports a (zlib) compatible compression method and that `libssh2` was built with `zlib` support enabled. [1] https://www.libssh2.org/libssh2_session_flag.html Ref: https://github.com/curl/curl/issues/1732 Closes https://github.com/curl/curl/pull/1735 --- docs/cmdline-opts/Makefile.inc | 3 +- docs/cmdline-opts/compressed-ssh.d | 6 +++ docs/libcurl/curl_easy_setopt.3 | 2 + docs/libcurl/opts/CURLOPT_SSH_COMPRESSION.3 | 58 +++++++++++++++++++++ docs/libcurl/opts/Makefile.inc | 1 + docs/libcurl/symbols-in-versions | 1 + include/curl/curl.h | 3 ++ lib/ssh.c | 7 +++ lib/url.c | 3 ++ lib/urldata.h | 1 + packages/OS400/curl.inc.in | 2 + src/tool_cfgable.h | 1 + src/tool_getparam.c | 4 ++ src/tool_help.c | 2 + src/tool_operate.c | 4 ++ tests/data/Makefile.inc | 2 +- tests/data/test642 | 42 +++++++++++++++ 17 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 docs/cmdline-opts/compressed-ssh.d create mode 100644 docs/libcurl/opts/CURLOPT_SSH_COMPRESSION.3 create mode 100644 tests/data/test642 diff --git a/docs/cmdline-opts/Makefile.inc b/docs/cmdline-opts/Makefile.inc index 7eea5c6c5..e8f46410b 100644 --- a/docs/cmdline-opts/Makefile.inc +++ b/docs/cmdline-opts/Makefile.inc @@ -1,7 +1,8 @@ # Shared between Makefile.am and CMakeLists.txt DPAGES = abstract-unix-socket.d anyauth.d append.d basic.d cacert.d capath.d cert.d \ - cert-status.d cert-type.d ciphers.d compressed.d config.d \ + cert-status.d cert-type.d ciphers.d compressed.d compressed-ssh.d \ + config.d \ connect-timeout.d connect-to.d continue-at.d cookie.d cookie-jar.d \ create-dirs.d crlf.d crlfile.d data-ascii.d data-binary.d data.d \ data-raw.d data-urlencode.d delegation.d digest.d disable.d \ diff --git a/docs/cmdline-opts/compressed-ssh.d b/docs/cmdline-opts/compressed-ssh.d new file mode 100644 index 000000000..beb5cb7f4 --- /dev/null +++ b/docs/cmdline-opts/compressed-ssh.d @@ -0,0 +1,6 @@ +Long: compressed-ssh +Help: Enable SSH compression +Protocols: SCP SFTP +--- +Enables built-in SSH compression. +This is a request, not an order; the server may or may not do it. diff --git a/docs/libcurl/curl_easy_setopt.3 b/docs/libcurl/curl_easy_setopt.3 index 26f73ee14..01dfa85b6 100644 --- a/docs/libcurl/curl_easy_setopt.3 +++ b/docs/libcurl/curl_easy_setopt.3 @@ -547,6 +547,8 @@ Disable GSS-API delegation. See \fICURLOPT_GSSAPI_DELEGATION(3)\fP .SH SSH OPTIONS .IP CURLOPT_SSH_AUTH_TYPES SSH authentication types. See \fICURLOPT_SSH_AUTH_TYPES(3)\fP +.IP CURLOPT_SSH_COMPRESSION +Enable SSH compression. See \fICURLOPT_SSH_COMPRESSION(3)\fP .IP CURLOPT_SSH_HOST_PUBLIC_KEY_MD5 MD5 of host's public key. See \fICURLOPT_SSH_HOST_PUBLIC_KEY_MD5(3)\fP .IP CURLOPT_SSH_PUBLIC_KEYFILE diff --git a/docs/libcurl/opts/CURLOPT_SSH_COMPRESSION.3 b/docs/libcurl/opts/CURLOPT_SSH_COMPRESSION.3 new file mode 100644 index 000000000..193320512 --- /dev/null +++ b/docs/libcurl/opts/CURLOPT_SSH_COMPRESSION.3 @@ -0,0 +1,58 @@ +.\" ************************************************************************** +.\" * _ _ ____ _ +.\" * Project ___| | | | _ \| | +.\" * / __| | | | |_) | | +.\" * | (__| |_| | _ <| |___ +.\" * \___|\___/|_| \_\_____| +.\" * +.\" * Copyright (C) 1998 - 2017, Daniel Stenberg, , et al. +.\" * +.\" * This software is licensed as described in the file COPYING, which +.\" * you should have received as part of this distribution. The terms +.\" * are also available at https://curl.haxx.se/docs/copyright.html. +.\" * +.\" * You may opt to use, copy, modify, merge, publish, distribute and/or sell +.\" * copies of the Software, and permit persons to whom the Software is +.\" * furnished to do so, under the terms of the COPYING file. +.\" * +.\" * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY +.\" * KIND, either express or implied. +.\" * +.\" ************************************************************************** +.\" +.TH CURLOPT_SSH_COMPRESSION 3 "05 Aug 2017" "libcurl 7.56.0" "curl_easy_setopt options" +.SH NAME +CURLOPT_SSH_COMPRESSION \- enables automatic decompression of HTTP downloads +.SH SYNOPSIS +#include + +CURLcode curl_easy_setopt(CURL *handle, CURLOPT_SSH_COMPRESSION, long enable); +.SH DESCRIPTION +Pass a long as parameter to enable or disable. + +Enables built-in SSH compression. +This is a request, not an order; the server may or may not do it. +.SH DEFAULT + +.SH PROTOCOLS +All SSH based protocols: SCP, SFTP +.SH EXAMPLE +.nf +CURL *curl = curl_easy_init(); +if(curl) { + curl_easy_setopt(curl, CURLOPT_URL, "sftp://example.com"); + + /* enable built-in compression */ + curl_easy_setopt(curl, CURLOPT_SSH_COMPRESSION, 1L); + + /* Perform the request */ + curl_easy_perform(curl); +} +.fi +.SH AVAILABILITY +Added in 7.56.0 +.SH RETURN VALUE +Returns CURLE_OK if the option is supported, CURLE_UNKNOWN_OPTION if not, or +CURLE_OUT_OF_MEMORY if there was insufficient heap space. +.SH "SEE ALSO" +.BR CURLOPT_ACCEPT_ENCODING "(3), " CURLOPT_TRANSFER_ENCODING "(3), " diff --git a/docs/libcurl/opts/Makefile.inc b/docs/libcurl/opts/Makefile.inc index c3db5d225..ad09dce2d 100644 --- a/docs/libcurl/opts/Makefile.inc +++ b/docs/libcurl/opts/Makefile.inc @@ -258,6 +258,7 @@ man_MANS = \ CURLOPT_SOCKS5_GSSAPI_NEC.3 \ CURLOPT_SOCKS5_GSSAPI_SERVICE.3 \ CURLOPT_SSH_AUTH_TYPES.3 \ + CURLOPT_SSH_COMPRESSION.3 \ CURLOPT_SSH_HOST_PUBLIC_KEY_MD5.3 \ CURLOPT_SSH_KEYDATA.3 \ CURLOPT_SSH_KEYFUNCTION.3 \ diff --git a/docs/libcurl/symbols-in-versions b/docs/libcurl/symbols-in-versions index e141b6a57..d3a2ff886 100644 --- a/docs/libcurl/symbols-in-versions +++ b/docs/libcurl/symbols-in-versions @@ -540,6 +540,7 @@ CURLOPT_SOURCE_QUOTE 7.13.0 - 7.15.5 CURLOPT_SOURCE_URL 7.13.0 - 7.15.5 CURLOPT_SOURCE_USERPWD 7.12.1 - 7.15.5 CURLOPT_SSH_AUTH_TYPES 7.16.1 +CURLOPT_SSH_COMPRESSION 7.56.0 CURLOPT_SSH_HOST_PUBLIC_KEY_MD5 7.17.1 CURLOPT_SSH_KEYDATA 7.19.6 CURLOPT_SSH_KEYFUNCTION 7.19.6 diff --git a/include/curl/curl.h b/include/curl/curl.h index 82fb8aeb8..3936d1e43 100644 --- a/include/curl/curl.h +++ b/include/curl/curl.h @@ -1788,6 +1788,9 @@ typedef enum { /* bitmask of allowed auth methods for connections to SOCKS5 proxies */ CINIT(SOCKS5_AUTH, LONG, 267), + /* Enable/disable SSH compression */ + CINIT(SSH_COMPRESSION, LONG, 268), + CURLOPT_LASTENTRY /* the last unused */ } CURLoption; diff --git a/lib/ssh.c b/lib/ssh.c index cc5178fe7..28f709ada 100644 --- a/lib/ssh.c +++ b/lib/ssh.c @@ -2942,6 +2942,13 @@ static CURLcode ssh_connect(struct connectdata *conn, bool *done) return CURLE_FAILED_INIT; } + if(data->set.ssh_compression) { +#if LIBSSH2_VERSION_NUM >= 0x010208 + if(libssh2_session_flag(ssh->ssh_session, LIBSSH2_FLAG_COMPRESS, 1) < 0) +#endif + infof(data, "Failed to enable compression for ssh session\n"); + } + #ifdef HAVE_LIBSSH2_KNOWNHOST_API if(data->set.str[STRING_SSH_KNOWNHOSTS]) { int rc; diff --git a/lib/url.c b/lib/url.c index 71d4d8bb7..32623abe5 100644 --- a/lib/url.c +++ b/lib/url.c @@ -2936,6 +2936,9 @@ CURLcode Curl_setopt(struct Curl_easy *data, CURLoption option, case CURLOPT_SUPPRESS_CONNECT_HEADERS: data->set.suppress_connect_headers = (0 != va_arg(param, long))?TRUE:FALSE; break; + case CURLOPT_SSH_COMPRESSION: + data->set.ssh_compression = (0 != va_arg(param, long))?TRUE:FALSE; + break; default: /* unknown tag and its companion, just ignore: */ result = CURLE_UNKNOWN_OPTION; diff --git a/lib/urldata.h b/lib/urldata.h index e45baff17..b0faa9a35 100644 --- a/lib/urldata.h +++ b/lib/urldata.h @@ -1742,6 +1742,7 @@ struct UserDefined { curl_sshkeycallback ssh_keyfunc; /* key matching callback */ void *ssh_keyfunc_userp; /* custom pointer to callback */ + bool ssh_compression; /* enable SSH compression */ /* Here follows boolean settings that define how to behave during this session. They are STATIC, set by libcurl users or at least initially diff --git a/packages/OS400/curl.inc.in b/packages/OS400/curl.inc.in index e4d2d30ca..41b6be190 100644 --- a/packages/OS400/curl.inc.in +++ b/packages/OS400/curl.inc.in @@ -1312,6 +1312,8 @@ d c 10264 d CURLOPT_SUPPRESS_CONNECT_HEADERS... d c 00265 + d CURLOPT_SSH_COMPRESSION... + d c 00268 * /if not defined(CURL_NO_OLDIES) d CURLOPT_FILE c 10001 diff --git a/src/tool_cfgable.h b/src/tool_cfgable.h index 254805c8c..2b436657f 100644 --- a/src/tool_cfgable.h +++ b/src/tool_cfgable.h @@ -247,6 +247,7 @@ struct OperationConfig { from user callbacks */ curl_error synthetic_error; /* if non-zero, it overrides any libcurl error */ + bool ssh_compression; /* enable/disable SSH compression */ struct GlobalConfig *global; struct OperationConfig *prev; struct OperationConfig *next; /* Always last in the struct */ diff --git a/src/tool_getparam.c b/src/tool_getparam.c index 40b39a8aa..3dd1dec3f 100644 --- a/src/tool_getparam.c +++ b/src/tool_getparam.c @@ -188,6 +188,7 @@ static const struct LongShort aliases[]= { {"$W", "abstract-unix-socket", ARG_STRING}, {"$X", "tls-max", ARG_STRING}, {"$Y", "suppress-connect-headers", ARG_BOOL}, + {"$Z", "compressed-ssh", ARG_BOOL}, {"0", "http1.0", ARG_NONE}, {"01", "http1.1", ARG_NONE}, {"02", "http2", ARG_NONE}, @@ -1076,6 +1077,9 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */ case 'Y': /* --suppress-connect-headers */ config->suppress_connect_headers = toggle; break; + case 'Z': /* --compressed-ssh */ + config->ssh_compression = toggle; + break; } break; case '#': /* --progress-bar */ diff --git a/src/tool_help.c b/src/tool_help.c index a5bfaba24..04d84dd7a 100644 --- a/src/tool_help.c +++ b/src/tool_help.c @@ -68,6 +68,8 @@ static const struct helptxt helptext[] = { "SSL ciphers to use"}, {" --compressed", "Request compressed response"}, + {" --compressed-ssh", + "Enable SSH compression"}, {"-K, --config ", "Read config from a file"}, {" --connect-timeout ", diff --git a/src/tool_operate.c b/src/tool_operate.c index 1e8d0073c..3c8c6ed2f 100644 --- a/src/tool_operate.c +++ b/src/tool_operate.c @@ -1091,6 +1091,10 @@ static CURLcode operate_do(struct GlobalConfig *global, to fail if we are not talking to who we think we should */ my_setopt_str(curl, CURLOPT_SSH_HOST_PUBLIC_KEY_MD5, config->hostpubmd5); + + /* new in libcurl 7.56.0 */ + if(config->ssh_compression) + my_setopt(curl, CURLOPT_SSH_COMPRESSION, 1L); } if(config->cacert) diff --git a/tests/data/Makefile.inc b/tests/data/Makefile.inc index b92f7f92f..88e5e2a9a 100644 --- a/tests/data/Makefile.inc +++ b/tests/data/Makefile.inc @@ -77,7 +77,7 @@ test599 test600 test601 test602 test603 test604 test605 test606 test607 \ test608 test609 test610 test611 test612 test613 test614 test615 test616 \ test617 test618 test619 test620 test621 test622 test623 test624 test625 \ test626 test627 test628 test629 test630 test631 test632 test633 test634 \ -test635 test636 test637 test638 test639 test640 test641 \ +test635 test636 test637 test638 test639 test640 test641 test642 \ \ test700 test701 test702 test703 test704 test705 test706 test707 test708 \ test709 test710 test711 test712 test713 test714 test715 \ diff --git a/tests/data/test642 b/tests/data/test642 new file mode 100644 index 000000000..41fd444fc --- /dev/null +++ b/tests/data/test642 @@ -0,0 +1,42 @@ + + + +SFTP + + + +# +# Server-side + + +Test data +for ssh test + + + +# +# Client-side + + +sftp + + +SFTP retrieval + + +--key curl_client_key --pubkey curl_client_key.pub -u %USER: --compressed-ssh sftp://%HOSTIP:%SSHPORT%PWD/log/file642.txt --insecure + + +Test data +for ssh test + + + +# +# Verify data after the test has been "shot" + + +disable + + +