[MAJOR] huge rework of the HTTP request FSM

The HTTP parser has been rewritten for better compliance to RFC2616.
The same parser is now usable for both requests and responses, and
it now supports HTTP/0.9 as well as multi-line headers. It has also
been improved for speed ; a typicial HTTP request is parsed in about
2 microseconds on a 1 GHz processor.

The monitor-uri check has been moved so that the requests are not
logged. The httpclose option now tries to change as little as
possible in the request, and does not affect the first header if
it is already set to 'close'. HTTP/0.9 requests are converted to
HTTP/1.0 before being forwarded.

Headers and request transformations are now distinct. The headers
list is updated after each insertion/removal/transformation. The
request is re-parsed and checked after each transformation. It is
not possible anymore to remove a request, and requests which lead
to invalid request lines are now rejected.

include/proto/buffers.h

@@ -87,6 +87,7 @@ int buffer_write_chunk(struct buffer *buf, struct chunk *chunk);
 int buffer_replace(struct buffer *b, char *pos, char *end, char *str);
 int buffer_replace2(struct buffer *b, char *pos, char *end, char *str, int len);
 int chunk_printf(struct chunk *chk, int size, const char *fmt, ...);
+void buffer_dump(FILE *o, struct buffer *b, int from, int to);
 
 /*
  * frees the destination chunk if already allocated, allocates a new string,

include/proto/hdr_idx.h

@@ -40,6 +40,38 @@ static inline void hdr_idx_init(struct hdr_idx *list)
 	list->used = list->last = 1;
 }
 
+/*
+ * Return index of the first entry in the list. Usually, it means the index of
+ * the first header just after the request or response. If zero is returned, it
+ * means that the list is empty.
+ */
+static inline int hdr_idx_first_idx(struct hdr_idx *list)
+{
+	return list->v[0].next;
+}
+
+/*
+ * Return position of the first entry in the list. Usually, it means the
+ * position of the first header just after the request, but it can also be the
+ * end of the headers if the request has no header. hdr_idx_start_idx() should
+ * be checked before to ensure there is a valid header.
+ */
+static inline int hdr_idx_first_pos(struct hdr_idx *list)
+{
+	return list->v[0].len + list->v[0].cr + 1;
+}
+
+/*
+ * Sets the information about the start line. Its length and the presence of
+ * the CR are registered so that hdr_idx_first_pos() knows exactly where to
+ * find the first header.
+ */
+static inline void hdr_idx_set_start(struct hdr_idx *list, int len, int cr)
+{
+	list->v[0].len = len;
+	list->v[0].cr = cr;
+}
+
 /*
  * Add a header entry to <list> after element <after>. <after> is ignored when
  * the list is empty or full. Common usage is to set <after> to list->tail.

include/proto/proto_http.h

@@ -31,9 +31,26 @@
  * some macros used for the request parsing.
  * from RFC2616:
  *   CTL                 = <any US-ASCII control character (octets 0 - 31) and DEL (127)>
+ *   SEP                 = one of the 17 defined separators or SP or HT
+ *   LWS                 = CR, LF, SP or HT
+ *   SPHT                = SP or HT. Use this macro and not a boolean expression for best speed.
+ *   CRLF                = CR or LF. Use this macro and not a boolean expression for best speed.
+ *   token               = any CHAR except CTL or SEP. Use this macro and not a boolean expression for best speed.
  */
-static inline int IS_CTL(const unsigned char x) { return (x < 32)||(x == 127);}
 
+extern const char http_is_ctl[256];
+extern const char http_is_sep[256];
+extern const char http_is_lws[256];
+extern const char http_is_spht[256];
+extern const char http_is_crlf[256];
+extern const char http_is_token[256];
+
+#define HTTP_IS_CTL(x)   (http_is_ctl[(unsigned char)(x)])
+#define HTTP_IS_SEP(x)   (http_is_sep[(unsigned char)(x)])
+#define HTTP_IS_LWS(x)   (http_is_lws[(unsigned char)(x)])
+#define HTTP_IS_SPHT(x)  (http_is_spht[(unsigned char)(x)])
+#define HTTP_IS_CRLF(x)  (http_is_crlf[(unsigned char)(x)])
+#define HTTP_IS_TOKEN(x) (http_is_token[(unsigned char)(x)])
 
 int event_accept(int fd);
 int process_session(struct task *t);
@@ -49,8 +66,10 @@ int produce_content(struct session *s);
 int produce_content_stats(struct session *s);
 int produce_content_stats_proxy(struct session *s, struct proxy *px);
 void debug_hdr(const char *dir, struct session *t, const char *start, const char *end);
-void get_srv_from_appsession(struct session *t, const char *begin, const char *end);
-void apply_filters_to_session(struct session *t, struct buffer *req, struct hdr_exp *exp);
+void get_srv_from_appsession(struct session *t, const char *begin, int len);
+int apply_filter_to_req_headers(struct session *t, struct buffer *req, struct hdr_exp *exp);
+int apply_filter_to_req_line(struct session *t, struct buffer *req, struct hdr_exp *exp);
+int apply_filters_to_request(struct session *t, struct buffer *req, struct hdr_exp *exp);
 void manage_client_side_cookies(struct session *t, struct buffer *req);
 int stats_check_uri_auth(struct session *t, struct proxy *backend);
 void init_proto_http();

include/types/proto_http.h

@@ -51,19 +51,64 @@
 #define SV_STCLOSE	6
 
 
+/* The HTTP parser is more complex than it looks like, because we have to
+ * support multi-line headers and any number of spaces between the colon and
+ * the value.
+ *
+ * All those examples must work :
+
+ Hdr1:val1\r\n
+ Hdr1: val1\r\n
+ Hdr1:\t val1\r\n
+ Hdr1: \r\n
+  val1\r\n
+ Hdr1:\r\n
+  val1\n
+ \tval2\r\n
+  val3\n
+
+ *
+ */
+
 /* Possible states while parsing HTTP messages (request|response) */
-#define HTTP_PA_EMPTY      0    /* leading LF, before start line */
-#define HTTP_PA_START      1    /* inside start line */
-#define HTTP_PA_STRT_LF    2    /* LF after start line */
-#define HTTP_PA_HEADER     3    /* inside a header */
-#define HTTP_PA_HDR_LF     4    /* LF after a header */
-#define HTTP_PA_HDR_LWS    5    /* LWS after a header */
-#define HTTP_PA_LFLF       6    /* after double LF/CRLF at the end of headers */
-#define HTTP_PA_ERROR      7    /* syntax error in the message */
-#define HTTP_PA_CR_SKIP 0x10    /* ORed with other values when a CR was skipped */
-#define HTTP_PA_LF_EXP  0x20    /* ORed with other values when a CR is seen and
-				 * an LF is expected before entering the
-				 * designated state. */
+#define HTTP_MSG_RQBEFORE      0 // request: leading LF, before start line
+#define HTTP_MSG_RQBEFORE_CR   1 // request: leading CRLF, before start line
+
+/* these ones define a request start line */
+#define HTTP_MSG_RQMETH        2 // parsing the Method
+#define HTTP_MSG_RQMETH_SP     3 // space(s) after the ethod
+#define HTTP_MSG_RQURI         4 // parsing the Request URI
+#define HTTP_MSG_RQURI_SP      5 // space(s) after the Request URI
+#define HTTP_MSG_RQVER         6 // parsing the Request Version
+#define HTTP_MSG_RQLINE_END    7 // end of request line (CR or LF)
+
+#define HTTP_MSG_RPBEFORE      8 // response: leading LF, before start line
+#define HTTP_MSG_RPBEFORE_CR   9 // response: leading CRLF, before start line
+
+/* these ones define a response start line */
+#define HTTP_MSG_RPVER        10 // parsing the Response Version
+#define HTTP_MSG_RPVER_SP     11 // space(s) after the Response Version
+#define HTTP_MSG_RPCODE       12 // response code
+#define HTTP_MSG_RPCODE_SP    13 // space(s) after the response code
+#define HTTP_MSG_RPREASON     14 // response reason
+#define HTTP_MSG_RPLINE_END   15 // end of response line (CR or LF)
+
+/* common header processing */
+
+#define HTTP_MSG_HDR_FIRST    16 // waiting for first header or last CRLF (no LWS possible)
+#define HTTP_MSG_HDR_NAME     17 // parsing header name
+#define HTTP_MSG_HDR_COL      18 // parsing header colon
+#define HTTP_MSG_HDR_L1_SP    19 // parsing header LWS (SP|HT) before value
+#define HTTP_MSG_HDR_L1_LF    20 // parsing header LWS (LF) before value
+#define HTTP_MSG_HDR_L1_LWS   21 // checking whether it's a new header or an LWS
+#define HTTP_MSG_HDR_VAL      22 // parsing header value
+#define HTTP_MSG_HDR_L2_LF    23 // parsing header LWS (LF) inside/after value
+#define HTTP_MSG_HDR_L2_LWS   24 // checking whether it's a new header or an LWS
+
+#define HTTP_MSG_LAST_LF      25 // parsing last LF
+#define HTTP_MSG_BODY         26 // parsing body at end of headers
+#define HTTP_MSG_ERROR        27 // an error occurred
+
 
 /* various data sources for the responses */
 #define DATA_SRC_NONE	0

include/types/session.h

@@ -93,6 +93,7 @@
 #define	SN_CACHE_SHIFT	20		/* bit shift */
 
 /* various other session flags, bits values 0x400000 and above */
+#define SN_CONN_CLOSED	0x00000800	/* "Connection: close" was present or added */
 #define SN_MONITOR	0x00400000	/* this session comes from a monitoring system */
 #define SN_ASSIGNED	0x00800000	/* no need to assign a server to this session */
 #define SN_ADDR_SET	0x01000000	/* this session's server address has been set */
@@ -134,9 +135,25 @@ typedef enum {
  */
 struct http_msg {
 	int hdr_state;                  /* where we are in the current header parsing */
-	int sor, eoh;			/* Start Of Request and End Of Headers, relative to buffer */
-	int eol;			/* end of line */
+	char *sol, *eol;		/* start of line, end of line */
+	int sor;			/* Start Of Request, relative to buffer */
+	int col, sov;			/* current header: colon, start of value */
+	int eoh;			/* End Of Headers, relative to buffer */
 	char **cap;			/* array of captured request headers (may be NULL) */
+	union {				/* useful start line pointers, relative to buffer */
+		struct {
+			int l;		/* request line length (not including CR) */
+			int m_l;	/* METHOD length (method starts at ->sor) */
+			int u, u_l;	/* URI, length */
+			int v, v_l;	/* VERSION, length */
+		} rq;			/* request line : field, length */
+		struct {
+			int l;		/* status line length (not including CR) */
+			int v_l;	/* VERSION length (version starts at ->sor) */
+			int c, c_l;	/* CODE, length */
+			int r, r_l;	/* REASON, length */
+		} st;			/* status line : field, length */
+	} sl;				/* start line */
 };
 
 /* This is an HTTP request, as described in RFC2616. It contains both a request
@@ -146,7 +163,6 @@ struct http_req {
 	int req_state;                  /* what we are currently parsing */
 	http_meth_t meth;		/* HTTP method */
 	struct hdr_idx hdr_idx;         /* array of header indexes (max: MAX_HTTP_HDR) */
-	struct chunk start;		/* points to first line, called "start line" in RFC2616 */
 	struct chunk auth_hdr;		/* points to 'Authorization:' header */
 	struct http_msg req, rsp;	/* HTTP request and response messages */
 };

src/buffers.c

@@ -154,6 +154,31 @@ int chunk_printf(struct chunk *chk, int size, const char *fmt, ...)
 	return chk->len;
 }
 
+/*
+ * Dumps part or all of a buffer.
+ */
+void buffer_dump(FILE *o, struct buffer *b, int from, int to)
+{
+	fprintf(o, "Dumping buffer %p\n", b);
+	fprintf(o, "  data=%p l=%d r=%p w=%p h=%p lr=%p\n",
+		b->data, b->l, b->r, b->w, b->h, b->lr);
+
+	if (!to || to > b->l)
+		to = b->l;
+
+	fprintf(o, "Dumping contents from byte %d to byte %d\n", from, to);
+	for (; from < to; from++) {
+		if ((from & 15) == 0)
+			fprintf(o, "  %04x: ", from);
+		fprintf(o, "%02x ", b->data[from]);
+		if ((from & 15) == 7)
+			fprintf(o, "- ");
+		else if (((from & 15) == 15) && (from != to-1))
+			fprintf(o, "\n");
+	}
+	fprintf(o, "\n--\n");
+}
+
 
 /*
  * Local variables:

src/cfgparse.c

@@ -601,10 +601,9 @@ int cfg_parse_listen(const char *file, int linenum, char **args)
 		if (curproxy->monitor_uri != NULL)
 			free(curproxy->monitor_uri);
 
-		curproxy->monitor_uri_len = strlen(args[1]) + 2; /* include leading and trailing spaces */
+		curproxy->monitor_uri_len = strlen(args[1]);
 		curproxy->monitor_uri = (char *)calloc(1, curproxy->monitor_uri_len + 1);
-		memcpy(curproxy->monitor_uri + 1, args[1], curproxy->monitor_uri_len - 2);
-		curproxy->monitor_uri[curproxy->monitor_uri_len-1] = curproxy->monitor_uri[0] = ' ';
+		memcpy(curproxy->monitor_uri, args[1], curproxy->monitor_uri_len);
 		curproxy->monitor_uri[curproxy->monitor_uri_len] = '\0';
 
 		return 0;

src/client.c

@@ -205,9 +205,9 @@ int event_accept(int fd) {
 		hreq->hdr_idx.size = hreq->hdr_idx.used = 0;
 
 		if (p->mode == PR_MODE_HTTP) {
-			hreq->req.hdr_state = HTTP_PA_EMPTY; /* at the very beginning of the request */
-			hreq->req.eol = hreq->req.sor = hreq->req.eoh = 0; /* relative to the buffer */
-			hreq->start.len = -1;
+			hreq->req.hdr_state = HTTP_MSG_RQBEFORE; /* at the very beginning of the request */
+			hreq->req.sol = hreq->req.eol = NULL;
+			hreq->req.sor = hreq->req.eoh = 0; /* relative to the buffer */
 			hreq->auth_hdr.len = -1;
 
 			hreq->hdr_idx.size = MAX_HTTP_HDR;

src/hdr_idx.c

@@ -14,22 +14,6 @@
 #include <types/hdr_idx.h>
 
 
-/*
- * Initialize the list pointers.
- * list->size must already be set. If list->size is set and list->v is
- * non-null, list->v is also initialized..
- */
-static inline void hdr_idx_init(struct hdr_idx *list)
-{
-
-	if (list->size && list->v) {
-		register struct hdr_idx_elem e = { .len=0, .cr=0, .next=0};
-		list->v[0] = e;
-	}
-	list->tail = 0;
-	list->used = list->last = 1;
-}
-
 /*
  * Add a header entry to <list> after element <after>. <after> is ignored when
  * the list is empty or full. Common usage is to set <after> to list->tail.

src/log.c

@@ -213,6 +213,10 @@ void send_log(struct proxy *p, int level, const char *message, ...)
 	}
 
 	va_start(argp, message);
+	/*
+	 * FIXME: we take a huge performance hit here. We might have to replace
+	 * vsnprintf() for a hard-coded log writer.
+	 */
 	data_len = vsnprintf(dataptr, logmsg + sizeof(logmsg) - dataptr, message, argp);
 	if (data_len < 0 || data_len > (logmsg + sizeof(logmsg) - dataptr))
 		data_len = logmsg + sizeof(logmsg) - dataptr;

src/session.c

@@ -21,6 +21,7 @@
 #include <types/proxy.h>
 #include <types/server.h>
 
+#include <proto/hdr_idx.h>
 #include <proto/session.h>
 #include <proto/queue.h>