From fe50bfb82c0ee93eec7dc896097c3705b21ec0cd Mon Sep 17 00:00:00 2001 From: Olivier Houchard Date: Mon, 27 May 2019 12:09:19 +0200 Subject: [PATCH] MEDIUM: connections: Introduce a handshake pseudo-XPRT. Add a new XPRT that is used when using non-SSL handshakes, such as proxy protocol or Netscaler, instead of taking care of it in conn_fd_handler(). This XPRT is installed when any of those is used, and it removes itself once the handshake is done. This should allow us to remove the distinction between CO_FL_SOCK* and CO_FL_XPRT*. --- Makefile | 3 +- include/proto/connection.h | 35 +++++ include/types/connection.h | 1 + src/backend.c | 11 ++ src/checks.c | 4 + src/connection.c | 59 +------ src/session.c | 13 +- src/tcp_rules.c | 14 +- src/xprt_handshake.c | 306 +++++++++++++++++++++++++++++++++++++ 9 files changed, 385 insertions(+), 61 deletions(-) create mode 100644 src/xprt_handshake.c diff --git a/Makefile b/Makefile index 66959741d..5d838679c 100644 --- a/Makefile +++ b/Makefile @@ -788,7 +788,8 @@ OBJS = src/proto_http.o src/cfgparse-listen.o src/proto_htx.o src/stream.o \ src/xxhash.o src/hpack-enc.o src/h2.o src/freq_ctr.o src/lru.o \ src/protocol.o src/arg.o src/hpack-huff.o src/hdr_idx.o src/base64.o \ src/hash.o src/mailers.o src/activity.o src/http_msg.o src/version.o \ - src/mworker.o src/mworker-prog.o src/debug.o src/wdt.o src/dict.o + src/mworker.o src/mworker-prog.o src/debug.o src/wdt.o src/dict.o \ + src/xprt_handshake.o EBTREE_OBJS = $(EBTREE_DIR)/ebtree.o $(EBTREE_DIR)/eb32sctree.o \ $(EBTREE_DIR)/eb32tree.o $(EBTREE_DIR)/eb64tree.o \ diff --git a/include/proto/connection.h b/include/proto/connection.h index 3e4a1ef72..ac4e5de34 100644 --- a/include/proto/connection.h +++ b/include/proto/connection.h @@ -946,6 +946,41 @@ static inline struct xprt_ops *xprt_get(int id) return registered_xprt[id]; } +/* Try to add a handshake pseudo-XPRT. If the connection's first XPRT is + * raw_sock, then just use the new XPRT as the connection XPRT, otherwise + * call the xprt's add_xprt() method. + * Returns 0 on success, or non-zero on failure. + */ +static inline int xprt_add_hs(struct connection *conn) +{ + void *xprt_ctx = NULL; + const struct xprt_ops *ops = xprt_get(XPRT_HANDSHAKE); + void *nextxprt_ctx = NULL; + const struct xprt_ops *nextxprt_ops = NULL; + + if (conn->flags & CO_FL_ERROR) + return -1; + if (ops->init(conn, &xprt_ctx) < 0) + return -1; + if (conn->xprt == xprt_get(XPRT_RAW)) { + nextxprt_ctx = conn->xprt_ctx; + nextxprt_ops = conn->xprt; + conn->xprt_ctx = xprt_ctx; + conn->xprt = ops; + } else { + if (conn->xprt->add_xprt(conn, conn->xprt_ctx, xprt_ctx, ops, + &nextxprt_ctx, &nextxprt_ops) != 0) { + ops->close(conn, xprt_ctx); + return -1; + } + } + if (ops->add_xprt(conn, xprt_ctx, nextxprt_ctx, nextxprt_ops, NULL, NULL) != 0) { + ops->close(conn, xprt_ctx); + return -1; + } + return 0; +} + static inline int conn_get_alpn(const struct connection *conn, const char **str, int *len) { if (!conn_xprt_ready(conn) || !conn->xprt->get_alpn) diff --git a/include/types/connection.h b/include/types/connection.h index 74060b086..e706b87a0 100644 --- a/include/types/connection.h +++ b/include/types/connection.h @@ -301,6 +301,7 @@ enum { enum { XPRT_RAW = 0, XPRT_SSL = 1, + XPRT_HANDSHAKE = 2, XPRT_ENTRIES /* must be last one */ }; diff --git a/src/backend.c b/src/backend.c index cfd24625c..36936a520 100644 --- a/src/backend.c +++ b/src/backend.c @@ -1213,6 +1213,7 @@ int connect_server(struct stream *s) int reuse_orphan = 0; int init_mux = 0; int alloced_cs = 0; + int flags_hs = 0; int err; @@ -1483,6 +1484,7 @@ int connect_server(struct stream *s) return SF_ERR_INTERNAL; } + flags_hs = srv_conn->flags & CO_FL_HANDSHAKE_NOSSL; if (!conn_xprt_ready(srv_conn) && !srv_conn->mux) { /* set the correct protocol on the output stream interface */ if (srv) @@ -1590,6 +1592,15 @@ int connect_server(struct stream *s) srv_conn->mux->avail_streams(srv_conn) > 0) LIST_ADD(&srv->idle_conns[tid], &srv_conn->list); } + /* The CO_FL_SEND_PROXY flag may have been set by the connect method, + * if so, add our handshake pseudo-XPRT now. + */ + if (!flags_hs && (srv_conn->flags & CO_FL_SEND_PROXY)) { + if (xprt_add_hs(srv_conn) < 0) { + conn_full_close(srv_conn); + return SF_ERR_INTERNAL; + } + } #if USE_OPENSSL && (defined(OPENSSL_IS_BORINGSSL) || (HA_OPENSSL_VERSION_NUMBER >= 0x10101000L)) diff --git a/src/checks.c b/src/checks.c index c4d956ee0..308b4b724 100644 --- a/src/checks.c +++ b/src/checks.c @@ -1662,6 +1662,8 @@ static int connect_conn_chk(struct task *t) if (s->check.send_proxy && !(check->state & CHK_ST_AGENT)) { conn->send_proxy_ofs = 1; conn->flags |= CO_FL_SEND_PROXY; + if (xprt_add_hs(conn) < 0) + ret = SF_ERR_RESOURCE; } return ret; @@ -2885,6 +2887,8 @@ static int tcpcheck_main(struct check *check) if (check->current_step->conn_opts & TCPCHK_OPT_SEND_PROXY) { conn->send_proxy_ofs = 1; conn->flags |= CO_FL_SEND_PROXY; + if (xprt_add_hs(conn) < 0) + ret = SF_ERR_RESOURCE; } /* It can return one of : diff --git a/src/connection.c b/src/connection.c index 1c81a0c3f..838d8fda1 100644 --- a/src/connection.c +++ b/src/connection.c @@ -58,51 +58,12 @@ void conn_fd_handler(int fd) flags = conn->flags & ~CO_FL_ERROR; /* ensure to call the wake handler upon error */ - process_handshake: - /* The handshake callbacks are called in sequence. If either of them is - * missing something, it must enable the required polling at the socket - * layer of the connection. Polling state is not guaranteed when entering - * these handlers, so any handshake handler which does not complete its - * work must explicitly disable events it's not interested in. Error - * handling is also performed here in order to reduce the number of tests - * around. - */ - while (unlikely(conn->flags & (CO_FL_HANDSHAKE | CO_FL_ERROR))) { - if (unlikely(conn->flags & CO_FL_ERROR)) - goto leave; - - if (conn->flags & CO_FL_SOCKS4_SEND) - if (!conn_send_socks4_proxy_request(conn)) - goto leave; - - if (conn->flags & CO_FL_SOCKS4_RECV) - if (!conn_recv_socks4_proxy_response(conn)) - goto leave; - - if (conn->flags & CO_FL_ACCEPT_CIP) - if (!conn_recv_netscaler_cip(conn, CO_FL_ACCEPT_CIP)) - goto leave; - - if (conn->flags & CO_FL_ACCEPT_PROXY) - if (!conn_recv_proxy(conn, CO_FL_ACCEPT_PROXY)) - goto leave; - - if (conn->flags & CO_FL_SEND_PROXY) - if (!conn_si_send_proxy(conn, CO_FL_SEND_PROXY)) - goto leave; - /* sock polling may have been activated by the connection, - * so remove it if we don't want it. - */ - if (conn->flags & CO_FL_SSL_WAIT_HS) { - if (!conn->send_wait) - __conn_sock_stop_send(conn); - if (!conn->recv_wait) - __conn_sock_stop_recv(conn); - break; - } + if (conn->flags & CO_FL_HANDSHAKE) { + if (!conn->send_wait) + __conn_sock_stop_send(conn); + if (!conn->recv_wait) + __conn_sock_stop_recv(conn); } - - /* Once we're purely in the data phase, we disable handshake polling */ if (!(conn->flags & CO_FL_POLL_SOCK)) __conn_sock_stop_both(conn); @@ -113,7 +74,7 @@ void conn_fd_handler(int fd) * leave instead. The caller must immediately unregister itself once * called. */ - if (!(conn->flags & CO_FL_SSL_WAIT_HS) && + if (!(conn->flags & CO_FL_HANDSHAKE) && conn->xprt_done_cb && conn->xprt_done_cb(conn) < 0) return; @@ -151,12 +112,6 @@ void conn_fd_handler(int fd) __conn_xprt_stop_recv(conn); } - /* It may happen during the data phase that a handshake is - * enabled again (eg: SSL) - */ - if (unlikely(conn->flags & (CO_FL_HANDSHAKE_NOSSL | CO_FL_ERROR))) - goto process_handshake; - if (unlikely(conn->flags & CO_FL_WAIT_L4_CONN)) { /* still waiting for a connection to establish and nothing was * attempted yet to probe the connection. Then let's retry the @@ -198,7 +153,7 @@ void conn_fd_handler(int fd) if ((io_available || (((conn->flags ^ flags) & CO_FL_NOTIFY_DATA) || ((flags & (CO_FL_CONNECTED|CO_FL_HANDSHAKE)) != CO_FL_CONNECTED && (conn->flags & (CO_FL_CONNECTED|CO_FL_HANDSHAKE)) == CO_FL_CONNECTED))) && - conn->mux->wake && conn->mux->wake(conn) < 0) + conn->mux && conn->mux->wake && conn->mux->wake(conn) < 0) return; /* commit polling changes */ diff --git a/src/session.c b/src/session.c index 718b97fd5..0ce1c4e54 100644 --- a/src/session.c +++ b/src/session.c @@ -165,21 +165,22 @@ int session_accept_fd(struct listener *l, int cfd, struct sockaddr_storage *addr conn_ctrl_init(cli_conn); /* wait for a PROXY protocol header */ - if (l->options & LI_O_ACC_PROXY) { + if (l->options & LI_O_ACC_PROXY) cli_conn->flags |= CO_FL_ACCEPT_PROXY; - conn_sock_want_recv(cli_conn); - } /* wait for a NetScaler client IP insertion protocol header */ - if (l->options & LI_O_ACC_CIP) { + if (l->options & LI_O_ACC_CIP) cli_conn->flags |= CO_FL_ACCEPT_CIP; - conn_sock_want_recv(cli_conn); - } conn_xprt_want_recv(cli_conn); if (conn_xprt_init(cli_conn) < 0) goto out_free_conn; + /* Add the handshake pseudo-XPRT */ + if (cli_conn->flags & (CO_FL_ACCEPT_PROXY | CO_FL_ACCEPT_CIP)) { + if (xprt_add_hs(cli_conn) != 0) + goto out_free_conn; + } sess = session_new(p, l, &cli_conn->obj_type); if (!sess) goto out_free_conn; diff --git a/src/tcp_rules.c b/src/tcp_rules.c index 13057cebf..a70c4c076 100644 --- a/src/tcp_rules.c +++ b/src/tcp_rules.c @@ -455,12 +455,22 @@ int tcp_exec_l4_rules(struct session *sess) stream_track_stkctr(&sess->stkctr[trk_idx(rule->action)], t, ts); } else if (rule->action == ACT_TCP_EXPECT_PX) { + if (!(conn->flags & (CO_FL_HANDSHAKE_NOSSL))) { + if (xprt_add_hs(conn) < 0) { + result = 0; + break; + } + } conn->flags |= CO_FL_ACCEPT_PROXY; - conn_sock_want_recv(conn); } else if (rule->action == ACT_TCP_EXPECT_CIP) { + if (!(conn->flags & (CO_FL_HANDSHAKE_NOSSL))) { + if (xprt_add_hs(conn) < 0) { + result = 0; + break; + } + } conn->flags |= CO_FL_ACCEPT_CIP; - conn_sock_want_recv(conn); } else { /* Custom keywords. */ diff --git a/src/xprt_handshake.c b/src/xprt_handshake.c new file mode 100644 index 000000000..2770385c5 --- /dev/null +++ b/src/xprt_handshake.c @@ -0,0 +1,306 @@ +/* + * Pseudo-xprt to handle any handshake except the SSL handshake + * + * Copyright 2019 HAProxy Technologies, Olivier Houchard + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version + * 2 of the License, or (at your option) any later version. + * + */ + +#include +#include + +struct xprt_handshake_ctx { + struct connection *conn; + struct wait_event *send_wait; + struct wait_event *recv_wait; + struct wait_event wait_event; + const struct xprt_ops *xprt; + void *xprt_ctx; +}; + +DECLARE_STATIC_POOL(xprt_handshake_ctx_pool, "xprt_handshake_ctx_pool", sizeof(struct xprt_handshake_ctx)); + +/* This XPRT doesn't take care of sending or receiving data, once its handshake + * is done, it just removes itself + */ +static size_t xprt_handshake_from_buf(struct connection *conn, void *xprt_ctx, const struct buffer *buf, size_t count, int flags) +{ + return 0; +} + +static size_t xprt_handshake_to_buf(struct connection *conn, void *xprt_ctx, struct buffer *buf, size_t count, int flags) +{ + return 0; +} + +static struct task *xprt_handshake_io_cb(struct task *t, void *bctx, unsigned short state) +{ + struct xprt_handshake_ctx *ctx = bctx; + struct connection *conn = ctx->conn; + + if (conn->flags & CO_FL_SOCKS4_SEND) + if (!conn_send_socks4_proxy_request(conn)) { + ctx->xprt->subscribe(conn, ctx->xprt_ctx, SUB_RETRY_SEND, + &ctx->wait_event); + + goto out; + } + + if (conn->flags & CO_FL_SOCKS4_RECV) + if (!conn_recv_socks4_proxy_response(conn)) { + ctx->xprt->subscribe(conn, ctx->xprt_ctx, SUB_RETRY_RECV, + &ctx->wait_event); + goto out; + } + + if (conn->flags & CO_FL_ACCEPT_CIP) + if (!conn_recv_netscaler_cip(conn, CO_FL_ACCEPT_CIP)) { + ctx->xprt->subscribe(conn, ctx->xprt_ctx, SUB_RETRY_RECV, + &ctx->wait_event); + goto out; + } + + if (conn->flags & CO_FL_ACCEPT_PROXY) + if (!conn_recv_proxy(conn, CO_FL_ACCEPT_PROXY)) { + ctx->xprt->subscribe(conn, ctx->xprt_ctx, SUB_RETRY_RECV, + &ctx->wait_event); + goto out; + } + + if (conn->flags & CO_FL_SEND_PROXY) + if (!conn_si_send_proxy(conn, CO_FL_SEND_PROXY)) { + ctx->xprt->subscribe(conn, ctx->xprt_ctx, SUB_RETRY_SEND, + &ctx->wait_event); + goto out; + } + +out: + /* Wake the stream if we're done with the handshake, or we have a + * connection error + * */ + if ((conn->flags & CO_FL_ERROR) || + !(conn->flags & CO_FL_HANDSHAKE_NOSSL)) { + int ret = 0; + int woke = 0; + int was_conn_ctx = 0; + /* On error, wake any waiter */ + if (ctx->recv_wait) { + ctx->recv_wait->events &= ~SUB_RETRY_RECV; + tasklet_wakeup(ctx->recv_wait->task); + woke = 1; + ctx->recv_wait = NULL; + } + if (ctx->send_wait) { + ctx->send_wait->events &= ~SUB_RETRY_SEND; + tasklet_wakeup(ctx->send_wait->task); + woke = 1; + ctx->send_wait = NULL; + } + if (!(conn->flags & CO_FL_ERROR)) + conn->flags |= CO_FL_CONNECTED; + /* Remove ourself from the xprt chain */ + if (ctx->wait_event.events != 0) + ctx->xprt->unsubscribe(ctx->conn, + ctx->xprt_ctx, + ctx->wait_event.events, + &ctx->wait_event); + if (conn->xprt_ctx == ctx) { + conn->xprt_ctx = ctx->xprt_ctx; + conn->xprt = ctx->xprt; + was_conn_ctx = 1; + } else + conn->xprt->remove_xprt(conn, conn->xprt_ctx, ctx, + ctx->xprt, ctx->xprt_ctx); + /* If we're the first xprt for the connection, let the + * upper layers know. If xprt_done_cb() is set, call it, + * and if we have a mux, and it has a wake method, call it + * too. + */ + if (was_conn_ctx) { + if (ctx->conn->xprt_done_cb) + ret = ctx->conn->xprt_done_cb(ctx->conn); + if (ret >= 0 && !woke && ctx->conn->mux && ctx->conn->mux->wake) + ret = ctx->conn->mux->wake(ctx->conn); + } + tasklet_free(ctx->wait_event.task); + pool_free(xprt_handshake_ctx_pool, ctx); + } + return NULL; +} + +static int xprt_handshake_init(struct connection *conn, void **xprt_ctx) +{ + struct xprt_handshake_ctx *ctx; + /* already initialized */ + if (*xprt_ctx) + return 0; + if (!conn_ctrl_ready(conn)) + return 0; + + ctx = pool_alloc(xprt_handshake_ctx_pool); + if (!ctx) { + conn->err_code = CO_ER_SSL_NO_MEM; + return -1; + } + ctx->conn = conn; + ctx->wait_event.task = tasklet_new(); + if (!ctx->wait_event.task) { + conn->err_code = CO_ER_SSL_NO_MEM; + pool_free(xprt_handshake_ctx_pool, ctx); + return -1; + } + ctx->wait_event.task->process = xprt_handshake_io_cb; + ctx->wait_event.task->context = ctx; + ctx->wait_event.events = 0; + /* This XPRT expects the underlying XPRT to be provided later, + * with an add_xprt() call, so we start trying to do the handshake + * there, when we'll be provided an XPRT. + */ + ctx->xprt = NULL; + ctx->xprt_ctx = NULL; + ctx->send_wait = ctx->recv_wait = NULL; + *xprt_ctx = ctx; + + return 0; +} + +static void xprt_handshake_close(struct connection *conn, void *xprt_ctx) +{ + struct xprt_handshake_ctx *ctx = xprt_ctx; + + if (ctx) { + if (ctx->wait_event.events != 0) + ctx->xprt->unsubscribe(ctx->conn, ctx->xprt_ctx, + ctx->wait_event.events, + &ctx->wait_event); + if (ctx->send_wait) { + ctx->send_wait->events &= ~SUB_RETRY_SEND; + tasklet_wakeup(ctx->send_wait->task); + } + if (ctx->recv_wait) { + ctx->recv_wait->events &= ~SUB_RETRY_RECV; + tasklet_wakeup(ctx->recv_wait->task); + } + + if (ctx->xprt && ctx->xprt->close) + ctx->xprt->close(conn, ctx->xprt_ctx); + /* Remove any handshake flag, and if we were the connection + * xprt, get back to XPRT_RAW. If we're here because we + * failed an outoging connection, it will be retried using + * the same struct connection, and as xprt_handshake is a bit + * magic, because it requires a call to add_xprt(), it's better + * to fallback to the original XPRT to re-initiate the + * connection. + */ + conn->flags &= ~CO_FL_HANDSHAKE_NOSSL; + if (conn->xprt == xprt_get(XPRT_HANDSHAKE)) + conn->xprt = xprt_get(XPRT_RAW); + tasklet_free(ctx->wait_event.task); + pool_free(xprt_handshake_ctx_pool, ctx); + } +} + +static int xprt_handshake_subscribe(struct connection *conn, void *xprt_ctx, int event_type, void *param) +{ + struct wait_event *sw; + struct xprt_handshake_ctx *ctx = xprt_ctx; + + if (event_type & SUB_RETRY_RECV) { + sw = param; + BUG_ON(ctx->recv_wait != NULL || (sw->events & SUB_RETRY_RECV)); + sw->events |= SUB_RETRY_RECV; + ctx->recv_wait = sw; + event_type &= ~SUB_RETRY_RECV; + } + if (event_type & SUB_RETRY_SEND) { + sw = param; + BUG_ON(ctx->send_wait != NULL || (sw->events & SUB_RETRY_SEND)); + sw->events |= SUB_RETRY_SEND; + ctx->send_wait = sw; + event_type &= ~SUB_RETRY_SEND; + } + if (event_type != 0) + return -1; + return 0; + +} + +static int xprt_handshake_unsubscribe(struct connection *conn, void *xprt_ctx, int event_type, void *param) +{ + struct wait_event *sw; + struct xprt_handshake_ctx *ctx = xprt_ctx; + + if (event_type & SUB_RETRY_RECV) { + sw = param; + BUG_ON(ctx->recv_wait != sw); + ctx->recv_wait = NULL; + sw->events &= ~SUB_RETRY_RECV; + } + if (event_type & SUB_RETRY_SEND) { + sw = param; + BUG_ON(ctx->send_wait != sw); + ctx->send_wait = NULL; + sw->events &= ~SUB_RETRY_SEND; + } + return 0; +} + +/* Use the provided XPRT as an underlying XPRT, and provide the old one. + * Returns 0 on success, and non-zero on failure. + */ +static int xprt_handshake_add_xprt(struct connection *conn, void *xprt_ctx, void *toadd_ctx, const struct xprt_ops *toadd_ops, void **oldxprt_ctx, const struct xprt_ops **oldxprt_ops) +{ + struct xprt_handshake_ctx *ctx = xprt_ctx; + + if (oldxprt_ops) + *oldxprt_ops = ctx->xprt; + if (oldxprt_ctx) + *oldxprt_ctx = ctx->xprt_ctx; + ctx->xprt = toadd_ops; + ctx->xprt_ctx = toadd_ctx; + /* Ok we know have an xprt, so let's try to do the handshake */ + tasklet_wakeup(ctx->wait_event.task); + return 0; +} + +/* Remove the specified xprt. If if it our underlying XPRT, remove it and + * return 0, otherwise just call the remove_xprt method from the underlying + * XPRT. + */ +static int xprt_handshake_remove_xprt(struct connection *conn, void *xprt_ctx, void *toremove_ctx, const struct xprt_ops *newops, void *newctx) +{ + struct xprt_handshake_ctx *ctx = xprt_ctx; + + if (ctx->xprt_ctx == toremove_ctx) { + ctx->xprt_ctx = newctx; + ctx->xprt = newops; + return 0; + } + return (ctx->xprt->remove_xprt(conn, ctx->xprt_ctx, toremove_ctx, newops, newctx)); +} + +struct xprt_ops xprt_handshake = { + .snd_buf = xprt_handshake_from_buf, + .rcv_buf = xprt_handshake_to_buf, + .subscribe = xprt_handshake_subscribe, + .unsubscribe = xprt_handshake_unsubscribe, + .remove_xprt = xprt_handshake_remove_xprt, + .add_xprt = xprt_handshake_add_xprt, + .init = xprt_handshake_init, + .close= xprt_handshake_close, + .rcv_pipe = NULL, + .snd_pipe = NULL, + .shutr = NULL, + .shutw = NULL, + .name = "HS", +}; + +__attribute__((constructor)) +static void __xprt_handshake_init(void) +{ + xprt_register(XPRT_HANDSHAKE, &xprt_handshake); +}