From c487d152b2b368e480b35bc21291131bd1e3a774 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 22 Feb 2013 21:54:07 +0900 Subject: [PATCH] shrpx: Add non-TLS SPDY backend connection support Use --backend-spdy-no-tls to disable TLS on backend SPDY connection. The SPDY protocol used there must be configured by --backend-spdy-proto option. --- src/shrpx.cc | 46 ++++++++++++++- src/shrpx_config.cc | 16 ++++++ src/shrpx_config.h | 11 ++++ src/shrpx_spdy_session.cc | 116 +++++++++++++++++++++++++------------- src/shrpx_spdy_session.h | 5 ++ src/shrpx_worker.cc | 2 +- 6 files changed, 153 insertions(+), 43 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index e0e15ec..8d0ae21 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -240,10 +240,12 @@ int event_loop() if(get_config()->client_mode) { sv_ssl_ctx = 0; - cl_ssl_ctx = ssl::create_ssl_client_context(); + cl_ssl_ctx = get_config()->spdy_downstream_no_tls ? + 0 : ssl::create_ssl_client_context(); } else { sv_ssl_ctx = get_config()->default_ssl_ctx; - cl_ssl_ctx = get_config()->spdy_bridge ? + cl_ssl_ctx = get_config()->spdy_bridge && + !get_config()->spdy_downstream_no_tls ? ssl::create_ssl_client_context() : 0; } @@ -271,7 +273,7 @@ int event_loop() if(get_config()->num_worker > 1) { listener_handler->create_worker_thread(get_config()->num_worker); - } else if(cl_ssl_ctx) { + } else if(get_config()->downstream_proto == PROTO_SPDY) { listener_handler->create_spdy_session(); } @@ -354,6 +356,9 @@ void fill_default_config() mod_config()->spdy_upstream_window_bits = 16; mod_config()->spdy_downstream_window_bits = 16; + mod_config()->spdy_downstream_no_tls = false; + mod_config()->spdy_downstream_version = 3; + set_config_str(&mod_config()->downstream_host, "127.0.0.1"); mod_config()->downstream_port = 80; mod_config()->downstream_hostport = 0; @@ -530,6 +535,15 @@ void print_help(std::ostream& out) << " backend connection to 2**.\n" << " Default: " << get_config()->spdy_downstream_window_bits << "\n" + << " --backend-spdy-no-tls\n" + << " Disable SSL/TLS on backend SPDY connections.\n" + << " SPDY protocol must be specified using\n" + << " --backend-spdy-proto\n" + << " --backend-spdy-proto\n" + << " Specify SPDY protocol used in backend\n" + << " connection if --backend-spdy-no-tls is used.\n" + << " Default: spdy/" + << get_config()->spdy_downstream_version << "\n" << "\n" << " Mode:\n" << " -s, --spdy-proxy Enable secure SPDY proxy mode.\n" @@ -628,6 +642,8 @@ int main(int argc, char **argv) {"subcert", required_argument, &flag, 24}, {"spdy-bridge", no_argument, &flag, 25}, {"backend-http-proxy-uri", required_argument, &flag, 26}, + {"backend-spdy-no-tls", no_argument, &flag, 27}, + {"backend-spdy-proto", required_argument, &flag, 28}, {0, 0, 0, 0 } }; int option_index = 0; @@ -787,6 +803,16 @@ int main(int argc, char **argv) cmdcfgs.push_back(std::make_pair(SHRPX_OPT_BACKEND_HTTP_PROXY_URI, optarg)); break; + case 27: + // --backend-spdy-no-tls + cmdcfgs.push_back(std::make_pair(SHRPX_OPT_BACKEND_SPDY_NO_TLS, + "yes")); + break; + case 28: + // --backend-spdy-proto + cmdcfgs.push_back(std::make_pair(SHRPX_OPT_BACKEND_SPDY_PROTO, + optarg)); + break; default: break; } @@ -842,6 +868,20 @@ int main(int argc, char **argv) mod_config()->client_mode = true; } + if(get_config()->client_mode || get_config()->spdy_bridge) { + mod_config()->downstream_proto = PROTO_SPDY; + } else { + mod_config()->downstream_proto = PROTO_HTTP; + } + + if(mod_config()->downstream_proto == PROTO_SPDY && + mod_config()->spdy_downstream_no_tls && + mod_config()->spdy_downstream_version == 0) { + LOG(FATAL) << "--backend-spdy-no-tls: Specify backend SPDY protocol using" + << " --backend-spdy-proto"; + exit(EXIT_FAILURE); + } + if(!get_config()->client_mode) { if(!get_config()->private_key_file || !get_config()->cert_file) { print_usage(std::cerr); diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index b22f8a2..9061ef1 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -36,6 +36,8 @@ #include #include +#include + #include "shrpx_log.h" #include "shrpx_ssl.h" #include "shrpx_http.h" @@ -73,6 +75,8 @@ const char SHRPX_OPT_BACKEND_KEEP_ALIVE_TIMEOUT[] = "backend-keep-alive-timeout"; const char SHRPX_OPT_FRONTEND_SPDY_WINDOW_BITS[] = "frontend-spdy-window-bits"; const char SHRPX_OPT_BACKEND_SPDY_WINDOW_BITS[] = "backend-spdy-window-bits"; +const char SHRPX_OPT_BACKEND_SPDY_NO_TLS[] = "backend-spdy-no-tls"; +const char SHRPX_OPT_BACKEND_SPDY_PROTO[] = "backend-spdy-proto"; const char SHRPX_OPT_PID_FILE[] = "pid-file"; const char SHRPX_OPT_USER[] = "user"; const char SHRPX_OPT_SYSLOG[] = "syslog"; @@ -259,6 +263,18 @@ int parse_config(const char *opt, const char *optarg) << " specify the integer in the range [0, 30], inclusive"; return -1; } + } else if(util::strieq(opt, SHRPX_OPT_BACKEND_SPDY_NO_TLS)) { + mod_config()->spdy_downstream_no_tls = util::strieq(optarg, "yes"); + } else if(util::strieq(opt, SHRPX_OPT_BACKEND_SPDY_PROTO)) { + size_t len = strlen(optarg); + const unsigned char *proto; + proto = reinterpret_cast(optarg); + uint16_t version = spdylay_npn_get_version(proto, len); + if(!version) { + LOG(ERROR) << "Unsupported SPDY version: " << optarg; + return -1; + } + mod_config()->spdy_downstream_version = version; } else if(util::strieq(opt, SHRPX_OPT_PID_FILE)) { set_config_str(&mod_config()->pid_file, optarg); } else if(util::strieq(opt, SHRPX_OPT_USER)) { diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 2b13320..7fd56da 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -68,6 +68,8 @@ extern const char SHRPX_OPT_ACCESSLOG[]; extern const char SHRPX_OPT_BACKEND_KEEP_ALIVE_TIMEOUT[]; extern const char SHRPX_OPT_FRONTEND_SPDY_WINDOW_BITS[]; extern const char SHRPX_OPT_BACKEND_SPDY_WINDOW_BITS[]; +extern const char SHRPX_OPT_BACKEND_SPDY_NO_TLS[]; +extern const char SHRPX_OPT_BACKEND_SPDY_PROTO[]; extern const char SHRPX_OPT_PID_FILE[]; extern const char SHRPX_OPT_USER[]; extern const char SHRPX_OPT_SYSLOG[]; @@ -88,6 +90,11 @@ union sockaddr_union { sockaddr_in in; }; +enum shrpx_proto { + PROTO_SPDY, + PROTO_HTTP +}; + struct Config { bool verbose; bool daemon; @@ -121,6 +128,8 @@ struct Config { bool accesslog; size_t spdy_upstream_window_bits; size_t spdy_downstream_window_bits; + bool spdy_downstream_no_tls; + uint16_t spdy_downstream_version; char *pid_file; uid_t uid; gid_t gid; @@ -134,6 +143,8 @@ struct Config { bool client; // true if --client or --client-proxy are enabled. bool client_mode; + // downstream protocol; this will be determined by given options. + shrpx_proto downstream_proto; bool insecure; char *cacert; bool backend_ipv4; diff --git a/src/shrpx_spdy_session.cc b/src/shrpx_spdy_session.cc index ce9b8d2..e702c10 100644 --- a/src/shrpx_spdy_session.cc +++ b/src/shrpx_spdy_session.cc @@ -79,14 +79,18 @@ int SpdySession::disconnect() } if(bev_) { int fd = bufferevent_getfd(bev_); - if(fd != -1 && fd_ == -1) { - fd_ = fd; - } else if(fd != -1 && fd_ != -1) { - assert(fd == fd_); - } bufferevent_disable(bev_, EV_READ | EV_WRITE); bufferevent_free(bev_); bev_ = 0; + if(fd != -1) { + if(fd_ == -1) { + fd_ = fd; + } else if(fd != fd_) { + SSLOG(WARNING, this) << "fd in bev_ != fd_"; + shutdown(fd, SHUT_WR); + close(fd); + } + } } if(ssl_) { SSL_free(ssl_); @@ -94,6 +98,9 @@ int SpdySession::disconnect() ssl_ = 0; if(fd_ != -1) { + if(LOG_ENABLED(INFO)) { + SSLOG(INFO, this) << "Closing fd=" << fd_; + } shutdown(fd_, SHUT_WR); close(fd_); fd_ = -1; @@ -241,7 +248,8 @@ void eventcb(bufferevent *bev, short events, void *ptr) SSLOG(INFO, spdy) << "Connection established"; } spdy->set_state(SpdySession::CONNECTED); - if((!get_config()->insecure && spdy->check_cert() != 0) || + if((!get_config()->spdy_downstream_no_tls && + !get_config()->insecure && spdy->check_cert() != 0) || spdy->on_connect() != 0) { spdy->disconnect(); return; @@ -282,7 +290,9 @@ void proxy_readcb(bufferevent *bev, void *ptr) // here. But we keep fd_ inside it. spdy->unwrap_free_bev(); // Initiate SSL/TLS handshake through established tunnel. - spdy->initiate_connection(); + if(spdy->initiate_connection() != 0) { + spdy->disconnect(); + } break; case SpdySession::PROXY_FAILED: spdy->disconnect(); @@ -347,7 +357,7 @@ int SpdySession::check_cert() int SpdySession::initiate_connection() { - int rv; + int rv = 0; if(get_config()->downstream_http_proxy_host && state_ == DISCONNECTED) { if(LOG_ENABLED(INFO)) { SSLOG(INFO, this) << "Connecting to the proxy " @@ -383,30 +393,50 @@ int SpdySession::initiate_connection() if(LOG_ENABLED(INFO)) { SSLOG(INFO, this) << "Connecting to downstream server"; } + if(ssl_ctx_) { + // We are establishing TLS connection. + ssl_ = SSL_new(ssl_ctx_); + if(!ssl_) { + SSLOG(ERROR, this) << "SSL_new() failed: " + << ERR_error_string(ERR_get_error(), NULL); + return -1; + } - ssl_ = SSL_new(ssl_ctx_); - if(!ssl_) { - SSLOG(ERROR, this) << "SSL_new() failed: " - << ERR_error_string(ERR_get_error(), NULL); - return -1; + if(!ssl::numeric_host(get_config()->downstream_host)) { + // TLS extensions: SNI. There is no documentation about the return + // code for this function (actually this is macro wrapping SSL_ctrl + // at the time of this writing). + SSL_set_tlsext_host_name(ssl_, get_config()->downstream_host); + } + // If state_ == PROXY_CONNECTED, we has connected to the proxy + // using fd_ and tunnel has been established. + bev_ = bufferevent_openssl_socket_new(evbase_, fd_, ssl_, + BUFFEREVENT_SSL_CONNECTING, + BEV_OPT_DEFER_CALLBACKS); + rv = bufferevent_socket_connect + (bev_, + // TODO maybe not thread-safe? + const_cast(&get_config()->downstream_addr.sa), + get_config()->downstream_addrlen); + } else if(state_ == DISCONNECTED) { + // Without TLS and proxy. + bev_ = bufferevent_socket_new(evbase_, -1, BEV_OPT_DEFER_CALLBACKS); + rv = bufferevent_socket_connect + (bev_, + const_cast(&get_config()->downstream_addr.sa), + get_config()->downstream_addrlen); + } else { + assert(state_ == PROXY_CONNECTED); + // Without TLS but with proxy. + bev_ = bufferevent_socket_new(evbase_, fd_, BEV_OPT_DEFER_CALLBACKS); + // Connection already established. + eventcb(bev_, BEV_EVENT_CONNECTED, this); + // eventcb() has no return value. Check state_ to whether it was + // failed or not. + if(state_ == DISCONNECTED) { + return -1; + } } - - if(!ssl::numeric_host(get_config()->downstream_host)) { - // TLS extensions: SNI. There is no documentation about the return - // code for this function (actually this is macro wrapping SSL_ctrl - // at the time of this writing). - SSL_set_tlsext_host_name(ssl_, get_config()->downstream_host); - } - // If state_ == PROXY_CONNECTED, we has connected to the proxy - // using fd_ and tunnel has been established. - bev_ = bufferevent_openssl_socket_new(evbase_, fd_, ssl_, - BUFFEREVENT_SSL_CONNECTING, - BEV_OPT_DEFER_CALLBACKS); - rv = bufferevent_socket_connect - (bev_, - // TODO maybe not thread-safe? - const_cast(&get_config()->downstream_addr.sa), - get_config()->downstream_addrlen); if(rv != 0) { bufferevent_free(bev_); bev_ = 0; @@ -418,7 +448,10 @@ int SpdySession::initiate_connection() bufferevent_setcb(bev_, readcb, writecb, eventcb, this); // No timeout for SPDY session - state_ = CONNECTING; + // We have been already connected when no TLS and proxy is used. + if(state_ != CONNECTED) { + state_ = CONNECTING; + } } else { // Unreachable DIE(); @@ -972,17 +1005,22 @@ void on_unknown_ctrl_recv_callback(spdylay_session *session, int SpdySession::on_connect() { int rv; + uint16_t version; const unsigned char *next_proto = 0; unsigned int next_proto_len; - SSL_get0_next_proto_negotiated(ssl_, &next_proto, &next_proto_len); + if(ssl_ctx_) { + SSL_get0_next_proto_negotiated(ssl_, &next_proto, &next_proto_len); - if(LOG_ENABLED(INFO)) { - std::string proto(next_proto, next_proto+next_proto_len); - SSLOG(INFO, this) << "Negotiated next protocol: " << proto; - } - uint16_t version = spdylay_npn_get_version(next_proto, next_proto_len); - if(!version) { - return -1; + if(LOG_ENABLED(INFO)) { + std::string proto(next_proto, next_proto+next_proto_len); + SSLOG(INFO, this) << "Negotiated next protocol: " << proto; + } + version = spdylay_npn_get_version(next_proto, next_proto_len); + if(!version) { + return -1; + } + } else { + version = get_config()->spdy_downstream_version; } spdylay_session_callbacks callbacks; memset(&callbacks, 0, sizeof(callbacks)); diff --git a/src/shrpx_spdy_session.h b/src/shrpx_spdy_session.h index 8c7c24c..c728f35 100644 --- a/src/shrpx_spdy_session.h +++ b/src/shrpx_spdy_session.h @@ -111,8 +111,13 @@ public: }; private: event_base *evbase_; + // NULL if no TLS is configured SSL_CTX *ssl_ctx_; SSL *ssl_; + // fd_ is used for proxy connection and no TLS connection. For + // direct or TLS connection, it may be -1 even after connection is + // established. Use bufferevent_getfd(bev_) to get file descriptor + // in these cases. int fd_; spdylay_session *session_; bufferevent *bev_; diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 43d2b16..a0b4587 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -75,7 +75,7 @@ void Worker::run() bufferevent *bev = bufferevent_socket_new(evbase, fd_, BEV_OPT_DEFER_CALLBACKS); SpdySession *spdy = 0; - if(cl_ssl_ctx_) { + if(get_config()->downstream_proto == PROTO_SPDY) { spdy = new SpdySession(evbase, cl_ssl_ctx_); if(spdy->init_notification() == -1) { DIE();