From ba7bc164f740002b5f4536af8408cd8b42156d97 Mon Sep 17 00:00:00 2001 From: Christopher Faulet Date: Mon, 7 Nov 2016 21:07:38 +0100 Subject: [PATCH] MINOR: spoe/checks: Add support for SPOP health checks A new "option spop-check" statement has been added to enable server health checks based on SPOP HELLO handshake. SPOP is the protocol used by SPOE filters to talk to servers. --- contrib/spoa_example/spoa.c | 32 ++++++++++++++++-- doc/SPOE.txt | 23 +++++++++++++ doc/configuration.txt | 18 ++++++++++ include/proto/checks.h | 5 +++ include/types/proxy.h | 3 +- src/cfgparse.c | 28 ++++++++++++++++ src/checks.c | 20 ++++++++++++ src/flt_spoe.c | 65 +++++++++++++++++++++++++++++++++++++ 8 files changed, 190 insertions(+), 4 deletions(-) diff --git a/contrib/spoa_example/spoa.c b/contrib/spoa_example/spoa.c index 1bd822ebc..ce59c04a0 100644 --- a/contrib/spoa_example/spoa.c +++ b/contrib/spoa_example/spoa.c @@ -143,6 +143,7 @@ struct worker { int status_code; unsigned int stream_id; unsigned int frame_id; + bool healthcheck; int ip_score; /* -1 if unset, else between 0 and 100 */ }; @@ -580,6 +581,24 @@ check_max_frame_size(struct worker *w, int idx) return idx; } +/* Check healthcheck value. It returns -1 if an error occurred, the number of + * read bytes otherwise. */ +static int +check_healthcheck(struct worker *w, int idx) +{ + int type; + + /* Get the "healthcheck" value of HAProxy */ + type = w->buf[idx++]; + if ((type & SPOE_DATA_T_MASK) != SPOE_DATA_T_BOOL) { + w->status_code = SPOE_FRM_ERR_INVALID; + return -1; + } + w->healthcheck = ((type & SPOE_DATA_FL_TRUE) == SPOE_DATA_FL_TRUE); + return idx; +} + + /* Decode a HELLO frame received from HAProxy. It returns -1 if an error * occurred, 0 if the frame must be skipped, otherwise the number of read * bytes. */ @@ -627,6 +646,12 @@ handle_hahello(struct worker *w) goto error; idx = i; } + /* Check "healthcheck" K/V item "*/ + else if (!memcmp(str, "healthcheck", sz)) { + if ((i = check_healthcheck(w, idx)) == -1) + goto error; + idx = i; + } /* Skip "capabilities" K/V item for now */ else { /* Silently ignore unknown item */ @@ -927,8 +952,8 @@ hello_handshake(int sock, struct worker *w) LOG("Failed to write Agent frame"); goto error; } - DEBUG("Hello handshake done: version=%s - max-frame-size=%u", - SPOP_VERSION, w->size); + DEBUG("Hello handshake done: version=%s - max-frame-size=%u - healthcheck=%s", + SPOP_VERSION, w->size, (w->healthcheck ? "true" : "false")); return 0; error: return -1; @@ -993,7 +1018,8 @@ worker(void *data) if (hello_handshake(csock, &w) < 0) goto disconnect; - + if (w.healthcheck == true) + goto close; while (1) { w.ip_score = -1; if (notify_ack_roundtip(csock, &w) < 0) diff --git a/doc/SPOE.txt b/doc/SPOE.txt index 538bb2684..fa0a53367 100644 --- a/doc/SPOE.txt +++ b/doc/SPOE.txt @@ -493,12 +493,24 @@ Unknown frames may be silently skipped. HAPROXY AGENT SRV | HAPROXY-HELLO | + | (healthcheck: false) | | --------------------------> | | | | AGENT-HELLO | | <-------------------------- | | | + * Successful HELLO healthcheck: + + HAPROXY AGENT SRV + | HAPROXY-HELLO | + | (healthcheck: true) | + | --------------------------> | + | | + | AGENT-HELLO + close() | + | <-------------------------- | + | | + * Error encountered by agent during the HELLO handshake: @@ -581,6 +593,13 @@ Following items are mandatory in the KV-LIST: This a comma-separated list of capabilities supported by HAProxy. Spaces must be ignored, if any. +Following optional items can be added in the KV-LIST: + + * "healthcheck" + + If this item is set to TRUE, then the HAPROXY-HELLO frame is sent during a + SPOE health check. When set to FALSE, this item can be ignored. + To finish the HELLO handshake, the agent must return an AGENT-HELLO frame with its supported SPOP version, the lower value between its maximum size allowed for a frame and the HAProxy one and capabilities it supports. If an error @@ -617,6 +636,10 @@ At this time, if everything is ok for HAProxy (supported version and valid max-frame-size value), the HELLO handshake is successfully completed. Else, HAProxy sends a HAPROXY-DISCONNECT frame with the corresponding error. +If "healthcheck" item was set to TRUE in the HAPROXY-HELLO frame, the agent can +safely close the connection without DISCONNECT frame. In all cases, HAProxy +will close the connexion at the end of the health check. + 3.2.6. Frame: NOTIFY --------------------- diff --git a/doc/configuration.txt b/doc/configuration.txt index 313965052..dad015253 100644 --- a/doc/configuration.txt +++ b/doc/configuration.txt @@ -1898,6 +1898,7 @@ option socket-stats (*) X X X - option splice-auto (*) X X X X option splice-request (*) X X X X option splice-response (*) X X X X +option spop-check - - - X option srvtcpka (*) X - X X option ssl-hello-chk X - X X -- keyword -------------------------- defaults - frontend - listen -- backend - @@ -6367,6 +6368,23 @@ no option splice-response "nosplice" and "maxpipes" +option spop-check + Use SPOP health checks for server testing + May be used in sections : defaults | frontend | listen | backend + no | no | no | yes + Arguments : none + + It is possible to test that the server correctly talks SPOP protocol instead + of just testing that it accepts the TCP connection. When this option is set, + a HELLO handshake is performed between HAProxy and the server, and the + response is analyzed to check no error is reported. + + Example : + option spop-check + + See also : "option httpchk" + + option srvtcpka no option srvtcpka Enable or disable the sending of TCP keepalive packets on the server side diff --git a/include/proto/checks.h b/include/proto/checks.h index ecd4a5ce4..bf771ea5e 100644 --- a/include/proto/checks.h +++ b/include/proto/checks.h @@ -51,6 +51,11 @@ void free_check(struct check *check); void send_email_alert(struct server *s, int priority, const char *format, ...) __attribute__ ((format(printf, 3, 4))); int srv_check_healthcheck_port(struct check *chk); + +/* Declared here, but the definitions are in flt_spoe.c */ +int prepare_spoe_healthcheck_request(char **req, int *len); +int handle_spoe_healthcheck_response(char *frame, size_t size, char *err, int errlen); + #endif /* _PROTO_CHECKS_H */ /* diff --git a/include/types/proxy.h b/include/types/proxy.h index 80d6a6448..27aa15740 100644 --- a/include/types/proxy.h +++ b/include/types/proxy.h @@ -173,7 +173,8 @@ enum PR_SRV_STATE_FILE { #define PR_O2_LB_AGENT_CHK 0x80000000 /* use a TCP connection to obtain a metric of server health */ #define PR_O2_TCPCHK_CHK 0x90000000 /* use TCPCHK check for server health */ #define PR_O2_EXT_CHK 0xA0000000 /* use external command for server health */ -/* unused: 0xB0000000 to 0xF000000, reserved for health checks */ +#define PR_O2_SPOP_CHK 0xB0000000 /* use SPOP for server health */ +/* unused: 0xC0000000 to 0xF000000, reserved for health checks */ #define PR_O2_CHK_ANY 0xF0000000 /* Mask to cover any check */ /* end of proxy->options2 */ diff --git a/src/cfgparse.c b/src/cfgparse.c index acd570d31..7b05727a7 100644 --- a/src/cfgparse.c +++ b/src/cfgparse.c @@ -5118,6 +5118,34 @@ stats_error_parsing: if (alertif_too_many_args_idx(0, 1, file, linenum, args, &err_code)) goto out; } + else if (!strcmp(args[1], "spop-check")) { + if (curproxy == &defproxy) { + Alert("parsing [%s:%d] : '%s %s' not allowed in 'defaults' section.\n", + file, linenum, args[0], args[1]); + err_code |= ERR_ALERT | ERR_FATAL; + goto out; + } + if (curproxy->cap & PR_CAP_FE) { + Alert("parsing [%s:%d] : '%s %s' not allowed in 'frontend' and 'listen' sections.\n", + file, linenum, args[0], args[1]); + err_code |= ERR_ALERT | ERR_FATAL; + goto out; + } + + /* use SPOE request to check servers' health */ + free(curproxy->check_req); + curproxy->check_req = NULL; + curproxy->options2 &= ~PR_O2_CHK_ANY; + curproxy->options2 |= PR_O2_SPOP_CHK; + + if (prepare_spoe_healthcheck_request(&curproxy->check_req, &curproxy->check_len)) { + Alert("parsing [%s:%d] : failed to prepare SPOP healthcheck request.\n", file, linenum); + err_code |= ERR_ALERT | ERR_FATAL; + goto out; + } + if (alertif_too_many_args_idx(0, 1, file, linenum, args, &err_code)) + goto out; + } else if (!strcmp(args[1], "tcp-check")) { /* use raw TCPCHK send/expect to check servers' health */ if (warnifnotcap(curproxy, PR_CAP_BE, file, linenum, args[1], NULL)) diff --git a/src/checks.c b/src/checks.c index 65d003743..84a0f583c 100644 --- a/src/checks.c +++ b/src/checks.c @@ -1317,6 +1317,26 @@ static void event_srv_chk_r(struct connection *conn) } break; + case PR_O2_SPOP_CHK: { + unsigned int framesz; + char err[HCHK_DESC_LEN]; + + if (!done && check->bi->i < 4) + goto wait_more_data; + + memcpy(&framesz, check->bi->data, 4); + framesz = ntohl(framesz); + + if (!done && check->bi->i < (4+framesz)) + goto wait_more_data; + + if (!handle_spoe_healthcheck_response(check->bi->data+4, framesz, err, HCHK_DESC_LEN-1)) + set_server_check_status(check, HCHK_STATUS_L7OKD, "SPOA server is ok"); + else + set_server_check_status(check, HCHK_STATUS_L7STS, err); + break; + } + default: /* for other checks (eg: pure TCP), delegate to the main task */ break; diff --git a/src/flt_spoe.c b/src/flt_spoe.c index 1ebdbdaf0..12e589ebc 100644 --- a/src/flt_spoe.c +++ b/src/flt_spoe.c @@ -414,6 +414,7 @@ enum spoe_data_type { #define VERSION_KEY "version" #define MAX_FRAME_SIZE_KEY "max-frame-size" #define CAPABILITIES_KEY "capabilities" +#define HEALTHCHECK_KEY "healthcheck" #define STATUS_CODE_KEY "status-code" #define MSG_KEY "message" @@ -1075,6 +1076,70 @@ handle_spoe_agentack_frame(struct appctx *appctx, char *frame, size_t size) return idx; } +/* This function is used in cfgparse.c and declared in proto/checks.h. It + * prepare the request to send to agents during a healthcheck. It returns 0 on + * success and -1 if an error occurred. */ +int +prepare_spoe_healthcheck_request(char **req, int *len) +{ + struct appctx a; + char *frame, buf[global.tune.bufsize]; + unsigned int framesz; + int idx; + + memset(&a, 0, sizeof(a)); + memset(buf, 0, sizeof(buf)); + APPCTX_SPOE(&a).max_frame_size = global.tune.bufsize; + + frame = buf+4; + idx = prepare_spoe_hahello_frame(&a, frame, global.tune.bufsize-4); + if (idx <= 0) + return -1; + if (idx + SLEN(HEALTHCHECK_KEY) + 1 > global.tune.bufsize-4) + return -1; + + /* "healthcheck" K/V item */ + idx += encode_spoe_string(HEALTHCHECK_KEY, SLEN(HEALTHCHECK_KEY), frame+idx); + frame[idx++] = (SPOE_DATA_T_BOOL | SPOE_DATA_FL_TRUE); + + framesz = htonl(idx); + memcpy(buf, (char *)&framesz, 4); + + if ((*req = malloc(idx+4)) == NULL) + return -1; + memcpy(*req, buf, idx+4); + *len = idx+4; + return 0; +} + +/* This function is used in checks.c and declared in proto/checks.h. It decode + * the response received from an agent during a healthcheck. It returns 0 on + * success and -1 if an error occurred. */ +int +handle_spoe_healthcheck_response(char *frame, size_t size, char *err, int errlen) +{ + struct appctx a; + int r; + + memset(&a, 0, sizeof(a)); + APPCTX_SPOE(&a).max_frame_size = global.tune.bufsize; + + if (handle_spoe_agentdiscon_frame(&a, frame, size) != 0) + goto error; + if ((r = handle_spoe_agenthello_frame(&a, frame, size)) <= 0) { + if (r == 0) + spoe_status_code = SPOE_FRM_ERR_INVALID; + goto error; + } + + return 0; + + error: + if (spoe_status_code >= SPOE_FRM_ERRS) + spoe_status_code = SPOE_FRM_ERR_UNKNOWN; + strncpy(err, spoe_frm_err_reasons[spoe_status_code], errlen); + return -1; +} /******************************************************************** * Functions that manage the SPOE applet