From 12ab6863c4b3bc06b669fa58d947338524f7fd70 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 11 Jul 2012 18:32:04 +0900 Subject: [PATCH] Use http-parser instead of htparse --- examples/Makefile.am | 2 +- examples/shrpx_downstream.cc | 144 +++++++++++++++++++++---------- examples/shrpx_downstream.h | 17 +++- examples/shrpx_https_upstream.cc | 125 ++++++++++++--------------- examples/shrpx_https_upstream.h | 4 +- examples/shrpx_spdy_upstream.cc | 6 -- 6 files changed, 173 insertions(+), 125 deletions(-) diff --git a/examples/Makefile.am b/examples/Makefile.am index 1f1c061..e5b1f44 100644 --- a/examples/Makefile.am +++ b/examples/Makefile.am @@ -88,7 +88,7 @@ shrpx_SOURCES = ${HELPER_OBJECTS} ${HELPER_HFILES} \ shrpx_ssl.cc shrpx_ssl.h \ shrpx_thread_event_receiver.cc shrpx_thread_event_receiver.h \ shrpx_worker.cc shrpx_worker.h \ - htparse/htparse.c htparse/htparse.h + http-parser/http_parser.c http-parser/http_parser.h endif # HAVE_LIBEVENT_OPENSSL noinst_PROGRAMS = spdycli diff --git a/examples/shrpx_downstream.cc b/examples/shrpx_downstream.cc index 512a496..decc15d 100644 --- a/examples/shrpx_downstream.cc +++ b/examples/shrpx_downstream.cc @@ -50,18 +50,20 @@ Downstream::Downstream(Upstream *upstream, int stream_id, int priority) chunked_request_(false), request_connection_close_(false), request_expect_100_continue_(false), + request_header_key_prev_(false), response_state_(INITIAL), response_http_status_(0), response_major_(1), response_minor_(1), chunked_response_(false), response_connection_close_(false), - response_htp_(htparser_new()), + response_header_key_prev_(false), + response_htp_(new http_parser()), response_body_buf_(0), recv_window_size_(0) { - htparser_init(response_htp_, htp_type_response); - htparser_set_userdata(response_htp_, this); + http_parser_init(response_htp_, HTTP_RESPONSE); + response_htp_->data = this; } Downstream::~Downstream() @@ -157,11 +159,13 @@ void check_connection_close(bool *connection_close, void Downstream::add_request_header(const std::string& name, const std::string& value) { + request_header_key_prev_ = true; request_headers_.push_back(std::make_pair(name, value)); } void Downstream::set_last_request_header_value(const std::string& value) { + request_header_key_prev_ = false; Headers::value_type &item = request_headers_.back(); item.second = value; check_transfer_encoding_chunked(&chunked_request_, item); @@ -169,6 +173,25 @@ void Downstream::set_last_request_header_value(const std::string& value) //check_connection_close(&request_connection_close_, item); } +bool Downstream::get_request_header_key_prev() const +{ + return request_header_key_prev_; +} + +void Downstream::append_last_request_header_key(const char *data, size_t len) +{ + assert(request_header_key_prev_); + Headers::value_type &item = request_headers_.back(); + item.first.append(data, len); +} + +void Downstream::append_last_request_header_value(const char *data, size_t len) +{ + assert(!request_header_key_prev_); + Headers::value_type &item = request_headers_.back(); + item.second.append(data, len); +} + void Downstream::set_request_method(const std::string& method) { request_method_ = method; @@ -184,6 +207,11 @@ void Downstream::set_request_path(const std::string& path) request_path_ = path; } +void Downstream::append_request_path(const char *data, size_t len) +{ + request_path_.append(data, len); +} + const std::string& Downstream::get_request_path() const { return request_path_; @@ -393,17 +421,39 @@ const Headers& Downstream::get_response_headers() const void Downstream::add_response_header(const std::string& name, const std::string& value) { + response_header_key_prev_ = true; response_headers_.push_back(std::make_pair(name, value)); } void Downstream::set_last_response_header_value(const std::string& value) { + response_header_key_prev_ = false; Headers::value_type &item = response_headers_.back(); item.second = value; check_transfer_encoding_chunked(&chunked_response_, item); //check_connection_close(&response_connection_close_, item); } +bool Downstream::get_response_header_key_prev() const +{ + return response_header_key_prev_; +} + +void Downstream::append_last_response_header_key(const char *data, size_t len) +{ + assert(response_header_key_prev_); + Headers::value_type &item = response_headers_.back(); + item.first.append(data, len); +} + +void Downstream::append_last_response_header_value(const char *data, + size_t len) +{ + assert(!response_header_key_prev_); + Headers::value_type &item = response_headers_.back(); + item.second.append(data, len); +} + unsigned int Downstream::get_response_http_status() const { return response_http_status_; @@ -455,14 +505,14 @@ void Downstream::set_response_connection_close(bool f) } namespace { -int htp_hdrs_completecb(htparser *htp) +int htp_hdrs_completecb(http_parser *htp) { Downstream *downstream; - downstream = reinterpret_cast(htparser_get_userdata(htp)); - downstream->set_response_http_status(htparser_get_status(htp)); - downstream->set_response_major(htparser_get_major(htp)); - downstream->set_response_minor(htparser_get_minor(htp)); - downstream->set_response_connection_close(!htparser_should_keep_alive(htp)); + downstream = reinterpret_cast(htp->data); + downstream->set_response_http_status(htp->status_code); + downstream->set_response_major(htp->http_major); + downstream->set_response_minor(htp->http_minor); + downstream->set_response_connection_close(!http_should_keep_alive(htp)); downstream->set_response_state(Downstream::HEADER_COMPLETE); downstream->get_upstream()->on_downstream_header_complete(downstream); return 0; @@ -470,30 +520,38 @@ int htp_hdrs_completecb(htparser *htp) } // namespace namespace { -int htp_hdr_keycb(htparser *htp, const char *data, size_t len) +int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { Downstream *downstream; - downstream = reinterpret_cast(htparser_get_userdata(htp)); - downstream->add_response_header(std::string(data, len), ""); + downstream = reinterpret_cast(htp->data); + if(downstream->get_response_header_key_prev()) { + downstream->append_last_response_header_key(data, len); + } else { + downstream->add_response_header(std::string(data, len), ""); + } return 0; } } // namespace namespace { -int htp_hdr_valcb(htparser *htp, const char *data, size_t len) +int htp_hdr_valcb(http_parser *htp, const char *data, size_t len) { Downstream *downstream; - downstream = reinterpret_cast(htparser_get_userdata(htp)); - downstream->set_last_response_header_value(std::string(data, len)); + downstream = reinterpret_cast(htp->data); + if(downstream->get_response_header_key_prev()) { + downstream->set_last_response_header_value(std::string(data, len)); + } else { + downstream->append_last_response_header_value(data, len); + } return 0; } } // namespace namespace { -int htp_bodycb(htparser *htp, const char *data, size_t len) +int htp_bodycb(http_parser *htp, const char *data, size_t len) { Downstream *downstream; - downstream = reinterpret_cast(htparser_get_userdata(htp)); + downstream = reinterpret_cast(htp->data); downstream->get_upstream()->on_downstream_body (downstream, reinterpret_cast(data), len); @@ -502,10 +560,10 @@ int htp_bodycb(htparser *htp, const char *data, size_t len) } // namespace namespace { -int htp_body_completecb(htparser *htp) +int htp_msg_completecb(http_parser *htp) { Downstream *downstream; - downstream = reinterpret_cast(htparser_get_userdata(htp)); + downstream = reinterpret_cast(htp->data); if(downstream->tunnel_established()) { // For tunneling, we remove timeouts. @@ -514,30 +572,26 @@ int htp_body_completecb(htparser *htp) downstream->set_response_state(Downstream::MSG_COMPLETE); downstream->get_upstream()->on_downstream_body_complete(downstream); - return 0; + + if(downstream->get_request_method() == "HEAD") { + // Ignore the response body. HEAD response may contain + // Content-Length or Transfer-Encoding: chunked. + return 1; + } else { + return 0; + } } } // namespace namespace { -htparse_hooks htp_hooks = { - 0, /*htparse_hook on_msg_begin;*/ - 0, /*htparse_data_hook method;*/ - 0, /* htparse_data_hook scheme;*/ - 0, /* htparse_data_hook host; */ - 0, /* htparse_data_hook port; */ - 0, /* htparse_data_hook path; */ - 0, /* htparse_data_hook args; */ - 0, /* htparse_data_hook uri; */ - 0, /* htparse_hook on_hdrs_begin; */ - htp_hdr_keycb, /* htparse_data_hook hdr_key; */ - htp_hdr_valcb, /* htparse_data_hook hdr_val; */ - 0, /* htparse_data_hook hostname; */ - htp_hdrs_completecb, /* htparse_hook on_hdrs_complete; */ - 0, /*htparse_hook on_new_chunk;*/ - 0, /*htparse_hook on_chunk_complete;*/ - 0, /*htparse_hook on_chunks_complete;*/ - htp_bodycb, /* htparse_data_hook body; */ - htp_body_completecb /* htparse_hook on_msg_complete;*/ +http_parser_settings htp_hooks = { + 0, /*http_cb on_message_begin;*/ + 0, /*http_data_cb on_url;*/ + htp_hdr_keycb, /*http_data_cb on_header_field;*/ + htp_hdr_valcb, /*http_data_cb on_header_value;*/ + htp_hdrs_completecb, /*http_cb on_headers_complete;*/ + htp_bodycb, /*http_data_cb on_body;*/ + htp_msg_completecb /*http_cb on_message_complete;*/ }; } // namespace @@ -547,17 +601,19 @@ int Downstream::parse_http_response() evbuffer *input = bufferevent_get_input(bev); unsigned char *mem = evbuffer_pullup(input, -1); - size_t nread = htparser_run(response_htp_, &htp_hooks, - reinterpret_cast(mem), - evbuffer_get_length(input)); + size_t nread = http_parser_execute(response_htp_, &htp_hooks, + reinterpret_cast(mem), + evbuffer_get_length(input)); evbuffer_drain(input, nread); - if(htparser_get_error(response_htp_) == htparse_error_none) { + http_errno htperr = HTTP_PARSER_ERRNO(response_htp_); + if(htperr == HPE_OK) { return 0; } else { if(ENABLE_LOG) { LOG(INFO) << "Downstream HTTP parser failure: " - << htparser_get_strerror(response_htp_); + << "(" << http_errno_name(htperr) << ") " + << http_errno_description(htperr); } return SHRPX_ERR_HTTP_PARSE; } diff --git a/examples/shrpx_downstream.h b/examples/shrpx_downstream.h index 436f50a..8e36cba 100644 --- a/examples/shrpx_downstream.h +++ b/examples/shrpx_downstream.h @@ -36,7 +36,7 @@ #include extern "C" { -#include "htparse/htparse.h" +#include "http-parser/http_parser.h" } #include "shrpx_io_control.h" @@ -72,9 +72,15 @@ public: const Headers& get_request_headers() const; void add_request_header(const std::string& name, const std::string& value); void set_last_request_header_value(const std::string& value); + + bool get_request_header_key_prev() const; + void append_last_request_header_key(const char *data, size_t len); + void append_last_request_header_value(const char *data, size_t len); + void set_request_method(const std::string& method); const std::string& get_request_method() const; void set_request_path(const std::string& path); + void append_request_path(const char *data, size_t len); const std::string& get_request_path() const; void set_request_major(int major); void set_request_minor(int minor); @@ -101,6 +107,11 @@ public: const Headers& get_response_headers() const; void add_response_header(const std::string& name, const std::string& value); void set_last_response_header_value(const std::string& value); + + bool get_response_header_key_prev() const; + void append_last_response_header_key(const char *data, size_t len); + void append_last_response_header_value(const char *data, size_t len); + unsigned int get_response_http_status() const; void set_response_http_status(unsigned int status); void set_response_major(int major); @@ -131,6 +142,7 @@ private: bool request_connection_close_; bool request_expect_100_continue_; Headers request_headers_; + bool request_header_key_prev_; int response_state_; unsigned int response_http_status_; @@ -139,7 +151,8 @@ private: bool chunked_response_; bool response_connection_close_; Headers response_headers_; - htparser *response_htp_; + bool response_header_key_prev_; + http_parser *response_htp_; // This buffer is used to temporarily store downstream response // body. Spdylay reads data from this in the callback. evbuffer *response_body_buf_; diff --git a/examples/shrpx_https_upstream.cc b/examples/shrpx_https_upstream.cc index b982027..040fea8 100644 --- a/examples/shrpx_https_upstream.cc +++ b/examples/shrpx_https_upstream.cc @@ -46,12 +46,12 @@ const size_t SHRPX_HTTPS_MAX_HEADER_LENGTH = 64*1024; HttpsUpstream::HttpsUpstream(ClientHandler *handler) : handler_(handler), - htp_(htparser_new()), + htp_(new http_parser()), current_header_length_(0), ioctrl_(handler->get_bev()) { - htparser_init(htp_, htp_type_request); - htparser_set_userdata(htp_, this); + http_parser_init(htp_, HTTP_REQUEST); + htp_->data = this; } HttpsUpstream::~HttpsUpstream() @@ -69,10 +69,10 @@ void HttpsUpstream::reset_current_header_length() } namespace { -int htp_msg_begin(htparser *htp) +int htp_msg_begin(http_parser *htp) { HttpsUpstream *upstream; - upstream = reinterpret_cast(htparser_get_userdata(htp)); + upstream = reinterpret_cast(htp->data); if(ENABLE_LOG) { LOG(INFO) << "Upstream https request start " << upstream; } @@ -84,73 +84,61 @@ int htp_msg_begin(htparser *htp) } // namespace namespace { -int htp_methodcb(htparser *htp, const char *data, size_t len) +int htp_uricb(http_parser *htp, const char *data, size_t len) { HttpsUpstream *upstream; - upstream = reinterpret_cast(htparser_get_userdata(htp)); + upstream = reinterpret_cast(htp->data); Downstream *downstream = upstream->get_last_downstream(); - downstream->set_request_method(std::string(data, len)); + downstream->append_request_path(data, len); return 0; } } // namespace namespace { -int htp_uricb(htparser *htp, const char *data, size_t len) +int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { HttpsUpstream *upstream; - upstream = reinterpret_cast(htparser_get_userdata(htp)); + upstream = reinterpret_cast(htp->data); Downstream *downstream = upstream->get_last_downstream(); - downstream->set_request_path(std::string(data, len)); - return 0; -} -} // namespace - -namespace { -int htp_hdrs_begincb(htparser *htp) -{ - if(ENABLE_LOG) { - LOG(INFO) << "Upstream https request headers start"; + if(downstream->get_request_header_key_prev()) { + downstream->append_last_request_header_key(data, len); + } else { + downstream->add_request_header(std::string(data, len), ""); } return 0; } } // namespace namespace { -int htp_hdr_keycb(htparser *htp, const char *data, size_t len) +int htp_hdr_valcb(http_parser *htp, const char *data, size_t len) { HttpsUpstream *upstream; - upstream = reinterpret_cast(htparser_get_userdata(htp)); + upstream = reinterpret_cast(htp->data); Downstream *downstream = upstream->get_last_downstream(); - downstream->add_request_header(std::string(data, len), ""); + if(downstream->get_request_header_key_prev()) { + downstream->set_last_request_header_value(std::string(data, len)); + } else { + downstream->append_last_request_header_value(data, len); + } return 0; } } // namespace namespace { -int htp_hdr_valcb(htparser *htp, const char *data, size_t len) +int htp_hdrs_completecb(http_parser *htp) { HttpsUpstream *upstream; - upstream = reinterpret_cast(htparser_get_userdata(htp)); - Downstream *downstream = upstream->get_last_downstream(); - downstream->set_last_request_header_value(std::string(data, len)); - return 0; -} -} // namespace - -namespace { -int htp_hdrs_completecb(htparser *htp) -{ - HttpsUpstream *upstream; - upstream = reinterpret_cast(htparser_get_userdata(htp)); + upstream = reinterpret_cast(htp->data); if(ENABLE_LOG) { LOG(INFO) << "Upstream https request headers complete " << upstream; } Downstream *downstream = upstream->get_last_downstream(); - downstream->set_request_major(htparser_get_major(htp)); - downstream->set_request_minor(htparser_get_minor(htp)); + downstream->set_request_method(http_method_str((enum http_method)htp->method)); + downstream->set_request_major(htp->http_major); + downstream->set_request_minor(htp->http_minor); - downstream->set_request_connection_close(!htparser_should_keep_alive(htp)); + downstream->set_request_connection_close(!http_should_keep_alive(htp)); DownstreamConnection *dconn; dconn = upstream->get_client_handler()->get_downstream_connection(); @@ -176,10 +164,10 @@ int htp_hdrs_completecb(htparser *htp) } // namespace namespace { -int htp_bodycb(htparser *htp, const char *data, size_t len) +int htp_bodycb(http_parser *htp, const char *data, size_t len) { HttpsUpstream *upstream; - upstream = reinterpret_cast(htparser_get_userdata(htp)); + upstream = reinterpret_cast(htp->data); Downstream *downstream = upstream->get_last_downstream(); downstream->push_upload_data_chunk(reinterpret_cast(data), len); @@ -188,44 +176,35 @@ int htp_bodycb(htparser *htp, const char *data, size_t len) } // namespace namespace { -int htp_msg_completecb(htparser *htp) +int htp_msg_completecb(http_parser *htp) { if(ENABLE_LOG) { LOG(INFO) << "Upstream https request complete"; } HttpsUpstream *upstream; - upstream = reinterpret_cast(htparser_get_userdata(htp)); + upstream = reinterpret_cast(htp->data); Downstream *downstream = upstream->get_last_downstream(); downstream->end_upload_data(); downstream->set_request_state(Downstream::MSG_COMPLETE); // Stop further processing to complete this request - return 1; + http_parser_pause(htp, 1); + return 0; } } // namespace namespace { -htparse_hooks htp_hooks = { - htp_msg_begin, /*htparse_hook on_msg_begin;*/ - htp_methodcb, /*htparse_data_hook method;*/ - 0, /* htparse_data_hook scheme;*/ - 0, /* htparse_data_hook host; */ - 0, /* htparse_data_hook port; */ - 0, /* htparse_data_hook path; */ - 0, /* htparse_data_hook args; */ - htp_uricb, /* htparse_data_hook uri; */ - htp_hdrs_begincb, /* htparse_hook on_hdrs_begin; */ - htp_hdr_keycb, /* htparse_data_hook hdr_key; */ - htp_hdr_valcb, /* htparse_data_hook hdr_val; */ - 0, /* htparse_data_hook hostname; */ - htp_hdrs_completecb, /* htparse_hook on_hdrs_complete; */ - 0, /*htparse_hook on_new_chunk;*/ - 0, /*htparse_hook on_chunk_complete;*/ - 0, /*htparse_hook on_chunks_complete;*/ - htp_bodycb, /* htparse_data_hook body; */ - htp_msg_completecb /* htparse_hook on_msg_complete;*/ +http_parser_settings htp_hooks = { + htp_msg_begin, /*http_cb on_message_begin;*/ + htp_uricb, /*http_data_cb on_url;*/ + htp_hdr_keycb, /*http_data_cb on_header_field;*/ + htp_hdr_valcb, /*http_data_cb on_header_value;*/ + htp_hdrs_completecb, /*http_cb on_headers_complete;*/ + htp_bodycb, /*http_data_cb on_body;*/ + htp_msg_completecb /*http_cb on_message_complete;*/ }; } // namespace + // on_read() does not consume all available data in input buffer if // one http request is fully received. int HttpsUpstream::on_read() @@ -235,15 +214,19 @@ int HttpsUpstream::on_read() unsigned char *mem = evbuffer_pullup(input, -1); - int nread = htparser_run(htp_, &htp_hooks, - reinterpret_cast(mem), - evbuffer_get_length(input)); + if(evbuffer_get_length(input) == 0) { + return 0; + } + + size_t nread = http_parser_execute(htp_, &htp_hooks, + reinterpret_cast(mem), + evbuffer_get_length(input)); evbuffer_drain(input, nread); // Well, actually header length + some body bytes current_header_length_ += nread; - htpparse_error htperr = htparser_get_error(htp_); Downstream *downstream = get_top_downstream(); - if(htperr == htparse_error_user) { + http_errno htperr = HTTP_PARSER_ERRNO(htp_); + if(htperr == HPE_PAUSED) { if(downstream->get_request_state() == Downstream::CONNECT_FAIL) { get_client_handler()->set_should_close_after_write(true); error_reply(503); @@ -259,7 +242,7 @@ int HttpsUpstream::on_read() pause_read(SHRPX_MSG_BLOCK); } } - } else if(htperr == htparse_error_none) { + } else if(htperr == HPE_OK) { // downstream can be NULL here. if(downstream) { if(downstream->get_request_state() == Downstream::INITIAL && @@ -278,7 +261,8 @@ int HttpsUpstream::on_read() } else { if(ENABLE_LOG) { LOG(INFO) << "Upstream http parse failure: " - << htparser_get_strerror(htp_); + << "(" << http_errno_name(htperr) << ") " + << http_errno_description(htperr); } get_client_handler()->set_should_close_after_write(true); error_reply(400); @@ -319,6 +303,7 @@ void HttpsUpstream::resume_read(IOCtrlReason reason) if(ioctrl_.resume_read(reason)) { // Process remaining data in input buffer here because these bytes // are not notified by readcb until new data arrive. + http_parser_pause(htp_, 0); on_read(); } } diff --git a/examples/shrpx_https_upstream.h b/examples/shrpx_https_upstream.h index ae679be..daf09de 100644 --- a/examples/shrpx_https_upstream.h +++ b/examples/shrpx_https_upstream.h @@ -32,7 +32,7 @@ #include extern "C" { -#include "htparse/htparse.h" +#include "http-parser/http_parser.h" } #include "shrpx_upstream.h" @@ -71,7 +71,7 @@ public: void reset_current_header_length(); private: ClientHandler *handler_; - htparser *htp_; + http_parser *htp_; size_t current_header_length_; std::deque downstream_queue_; IOControl ioctrl_; diff --git a/examples/shrpx_spdy_upstream.cc b/examples/shrpx_spdy_upstream.cc index a626ea5..f0d7766 100644 --- a/examples/shrpx_spdy_upstream.cc +++ b/examples/shrpx_spdy_upstream.cc @@ -351,9 +351,6 @@ ClientHandler* SpdyUpstream::get_client_handler() const namespace { void spdy_downstream_readcb(bufferevent *bev, void *ptr) { - if(ENABLE_LOG) { - LOG(INFO) << "spdy_downstream_readcb"; - } DownstreamConnection *dconn = reinterpret_cast(ptr); Downstream *downstream = dconn->get_downstream(); SpdyUpstream *upstream; @@ -674,9 +671,6 @@ int SpdyUpstream::on_downstream_header_complete(Downstream *downstream) int SpdyUpstream::on_downstream_body(Downstream *downstream, const uint8_t *data, size_t len) { - if(ENABLE_LOG) { - LOG(INFO) << "Downstream on_downstream_body"; - } evbuffer *body = downstream->get_response_body_buf(); evbuffer_add(body, data, len); spdylay_session_resume_data(session_, downstream->get_stream_id());