From 4dff63c25ebf8963065323964cf1c24e290b6ced Mon Sep 17 00:00:00 2001 From: Karsten Graul Date: Tue, 12 Feb 2019 16:29:50 +0100 Subject: [PATCH 1/7] net/smc: reset cursor update required flag When an updated rx_cursor_confirmed field was sent to the peer then reset the cons_curs_upd_req flag. And remove the duplicate reset and cursor update in smc_tx_consumer_update(). Signed-off-by: Karsten Graul Signed-off-by: Ursula Braun Signed-off-by: David S. Miller --- net/smc/smc_cdc.c | 5 ++++- net/smc/smc_tx.c | 3 --- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/net/smc/smc_cdc.c b/net/smc/smc_cdc.c index a712c9f8699b..99d9d6e85dfb 100644 --- a/net/smc/smc_cdc.c +++ b/net/smc/smc_cdc.c @@ -105,8 +105,10 @@ int smc_cdc_msg_send(struct smc_connection *conn, &conn->local_tx_ctrl, conn); smc_curs_copy(&cfed, &((struct smc_host_cdc_msg *)wr_buf)->cons, conn); rc = smc_wr_tx_send(link, (struct smc_wr_tx_pend_priv *)pend); - if (!rc) + if (!rc) { smc_curs_copy(&conn->rx_curs_confirmed, &cfed, conn); + conn->local_rx_ctrl.prod_flags.cons_curs_upd_req = 0; + } return rc; } @@ -194,6 +196,7 @@ int smcd_cdc_msg_send(struct smc_connection *conn) if (rc) return rc; smc_curs_copy(&conn->rx_curs_confirmed, &curs, conn); + conn->local_rx_ctrl.prod_flags.cons_curs_upd_req = 0; /* Calculate transmitted data and increment free send buffer space */ diff = smc_curs_diff(conn->sndbuf_desc->len, &conn->tx_curs_fin, &conn->tx_curs_sent); diff --git a/net/smc/smc_tx.c b/net/smc/smc_tx.c index f93f3580c100..ce9586bce364 100644 --- a/net/smc/smc_tx.c +++ b/net/smc/smc_tx.c @@ -610,9 +610,6 @@ void smc_tx_consumer_update(struct smc_connection *conn, bool force) SMC_TX_WORK_DELAY); return; } - smc_curs_copy(&conn->rx_curs_confirmed, - &conn->local_tx_ctrl.cons, conn); - conn->local_rx_ctrl.prod_flags.cons_curs_upd_req = 0; } if (conn->local_rx_ctrl.prod_flags.write_blocked && !atomic_read(&conn->bytes_to_rcv)) From 5bc056d8d0e3f5f896eb747d42d1e3aa8ceaf1b0 Mon Sep 17 00:00:00 2001 From: Karsten Graul Date: Tue, 12 Feb 2019 16:29:51 +0100 Subject: [PATCH 2/7] net/smc: move wake up of close waiter Move the call to smc_close_wake_tx_prepared() (which wakes up a possibly waiting close processing that might wait for 'all data sent') to smc_tx_sndbuf_nonempty() (which is the main function to send data). Signed-off-by: Karsten Graul Signed-off-by: Ursula Braun Signed-off-by: David S. Miller --- net/smc/smc_cdc.c | 2 -- net/smc/smc_tx.c | 7 +++++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/net/smc/smc_cdc.c b/net/smc/smc_cdc.c index 99d9d6e85dfb..780b36c69292 100644 --- a/net/smc/smc_cdc.c +++ b/net/smc/smc_cdc.c @@ -290,8 +290,6 @@ static void smc_cdc_msg_recv_action(struct smc_sock *smc, /* trigger sndbuf consumer: RDMA write into peer RMBE and CDC */ if (diff_cons && smc_tx_prepared_sends(conn)) { smc_tx_sndbuf_nonempty(conn); - /* trigger socket release if connection closed */ - smc_close_wake_tx_prepared(smc); } if (diff_cons && conn->urg_tx_pend && atomic_read(&conn->peer_rmbe_space) == conn->peer_rmbe_size) { diff --git a/net/smc/smc_tx.c b/net/smc/smc_tx.c index ce9586bce364..dd10a913b38e 100644 --- a/net/smc/smc_tx.c +++ b/net/smc/smc_tx.c @@ -24,6 +24,7 @@ #include "smc.h" #include "smc_wr.h" #include "smc_cdc.h" +#include "smc_close.h" #include "smc_ism.h" #include "smc_tx.h" @@ -554,6 +555,12 @@ int smc_tx_sndbuf_nonempty(struct smc_connection *conn) else rc = smcr_tx_sndbuf_nonempty(conn); + if (!rc) { + /* trigger socket release if connection is closing */ + struct smc_sock *smc = container_of(conn, struct smc_sock, + conn); + smc_close_wake_tx_prepared(smc); + } return rc; } From 16297d143989e3f5acd75c1ca0a771b78aa12b46 Mon Sep 17 00:00:00 2001 From: Karsten Graul Date: Tue, 12 Feb 2019 16:29:52 +0100 Subject: [PATCH 3/7] net/smc: no delay for free tx buffer wait When no free transfer buffers are available then a work to call smc_tx_work() is scheduled. Set the schedule delay to zero, because for the out-of-buffers condition the work can start immediately and will block in the called function smc_wr_tx_get_free_slot(), waiting for free buffers. Signed-off-by: Karsten Graul Signed-off-by: Ursula Braun Signed-off-by: David S. Miller --- net/smc/smc_tx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/smc/smc_tx.c b/net/smc/smc_tx.c index dd10a913b38e..a3bff08ff8c8 100644 --- a/net/smc/smc_tx.c +++ b/net/smc/smc_tx.c @@ -28,7 +28,7 @@ #include "smc_ism.h" #include "smc_tx.h" -#define SMC_TX_WORK_DELAY HZ +#define SMC_TX_WORK_DELAY 0 #define SMC_TX_CORK_DELAY (HZ >> 2) /* 250 ms */ /***************************** sndbuf producer *******************************/ From cf0cfe535845cf60830e1482f7e618d80140e8d9 Mon Sep 17 00:00:00 2001 From: Karsten Graul Date: Tue, 12 Feb 2019 16:29:53 +0100 Subject: [PATCH 4/7] net/smc: reduce amount of status updates to peer In smc_cdc_msg_recv_action() the received cdc message is evaluated. To reduce the number of messaged triggered by this evaluation the logic is streamlined. For the write_blocked condition we do not need to send a response immediately. The remaining conditions can be put together into one if clause. Signed-off-by: Karsten Graul Signed-off-by: Ursula Braun Signed-off-by: David S. Miller --- net/smc/smc_cdc.c | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/net/smc/smc_cdc.c b/net/smc/smc_cdc.c index 780b36c69292..28bbdb04bc35 100644 --- a/net/smc/smc_cdc.c +++ b/net/smc/smc_cdc.c @@ -273,24 +273,18 @@ static void smc_cdc_msg_recv_action(struct smc_sock *smc, smp_mb__after_atomic(); smc->sk.sk_data_ready(&smc->sk); } else { - if (conn->local_rx_ctrl.prod_flags.write_blocked || - conn->local_rx_ctrl.prod_flags.cons_curs_upd_req || - conn->local_rx_ctrl.prod_flags.urg_data_pending) { - if (conn->local_rx_ctrl.prod_flags.urg_data_pending) - conn->urg_state = SMC_URG_NOTYET; - /* force immediate tx of current consumer cursor, but - * under send_lock to guarantee arrival in seqno-order - */ - if (smc->sk.sk_state != SMC_INIT) - smc_tx_sndbuf_nonempty(conn); - } + if (conn->local_rx_ctrl.prod_flags.write_blocked) + smc->sk.sk_data_ready(&smc->sk); + if (conn->local_rx_ctrl.prod_flags.urg_data_pending) + conn->urg_state = SMC_URG_NOTYET; } - /* piggy backed tx info */ /* trigger sndbuf consumer: RDMA write into peer RMBE and CDC */ - if (diff_cons && smc_tx_prepared_sends(conn)) { + if ((diff_cons && smc_tx_prepared_sends(conn)) || + conn->local_rx_ctrl.prod_flags.cons_curs_upd_req || + conn->local_rx_ctrl.prod_flags.urg_data_pending) smc_tx_sndbuf_nonempty(conn); - } + if (diff_cons && conn->urg_tx_pend && atomic_read(&conn->peer_rmbe_space) == conn->peer_rmbe_size) { /* urg data confirmed by peer, indicate we're ready for more */ From e78b2622266ce962b18efad8f29acdc6052db010 Mon Sep 17 00:00:00 2001 From: Karsten Graul Date: Tue, 12 Feb 2019 16:29:54 +0100 Subject: [PATCH 5/7] net/smc: check connections in smc_lgr_free_work Remove the shortcut that smc_lgr_free() would skip the check for existing connections when the link group is not in the link group list. Signed-off-by: Karsten Graul Signed-off-by: Ursula Braun Signed-off-by: David S. Miller --- net/smc/smc_core.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/net/smc/smc_core.c b/net/smc/smc_core.c index 349d789a9728..53a17cfa61af 100644 --- a/net/smc/smc_core.c +++ b/net/smc/smc_core.c @@ -160,8 +160,6 @@ static void smc_lgr_free_work(struct work_struct *work) bool conns; spin_lock_bh(&smc_lgr_list.lock); - if (list_empty(&lgr->list)) - goto free; read_lock_bh(&lgr->conns_lock); conns = RB_EMPTY_ROOT(&lgr->conns_all); read_unlock_bh(&lgr->conns_lock); @@ -169,8 +167,8 @@ static void smc_lgr_free_work(struct work_struct *work) spin_unlock_bh(&smc_lgr_list.lock); return; } - list_del_init(&lgr->list); /* remove from smc_lgr_list */ -free: + if (!list_empty(&lgr->list)) + list_del_init(&lgr->list); /* remove from smc_lgr_list */ spin_unlock_bh(&smc_lgr_list.lock); if (!lgr->is_smcd && !lgr->terminating) { From 81cf6430526531bc688d4c3d09506dba7b456fb1 Mon Sep 17 00:00:00 2001 From: Karsten Graul Date: Tue, 12 Feb 2019 16:29:55 +0100 Subject: [PATCH 6/7] net/smc: check port_idx of ib event For robustness protect of higher port numbers than expected to avoid setting bits behind our port_event_mask. In case of an DEVICE_FATAL event all ports must be checked. The IB_EVENT_GID_CHANGE event is provided in the global event handler, so handle it there. And handle a QP_FATAL event instead of an DEVICE_FATAL event in the qp handler. Signed-off-by: Karsten Graul Signed-off-by: Ursula Braun Signed-off-by: David S. Miller --- net/smc/smc_ib.c | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/net/smc/smc_ib.c b/net/smc/smc_ib.c index 76487a16934e..0b244be24fe0 100644 --- a/net/smc/smc_ib.c +++ b/net/smc/smc_ib.c @@ -257,13 +257,21 @@ static void smc_ib_global_event_handler(struct ib_event_handler *handler, smcibdev = container_of(handler, struct smc_ib_device, event_handler); switch (ibevent->event) { - case IB_EVENT_PORT_ERR: case IB_EVENT_DEVICE_FATAL: - case IB_EVENT_PORT_ACTIVE: - port_idx = ibevent->element.port_num - 1; - set_bit(port_idx, &smcibdev->port_event_mask); + /* terminate all ports on device */ + for (port_idx = 0; port_idx < SMC_MAX_PORTS; port_idx++) + set_bit(port_idx, &smcibdev->port_event_mask); schedule_work(&smcibdev->port_event_work); break; + case IB_EVENT_PORT_ERR: + case IB_EVENT_PORT_ACTIVE: + case IB_EVENT_GID_CHANGE: + port_idx = ibevent->element.port_num - 1; + if (port_idx < SMC_MAX_PORTS) { + set_bit(port_idx, &smcibdev->port_event_mask); + schedule_work(&smcibdev->port_event_work); + } + break; default: break; } @@ -294,13 +302,13 @@ static void smc_ib_qp_event_handler(struct ib_event *ibevent, void *priv) u8 port_idx; switch (ibevent->event) { - case IB_EVENT_DEVICE_FATAL: - case IB_EVENT_GID_CHANGE: - case IB_EVENT_PORT_ERR: + case IB_EVENT_QP_FATAL: case IB_EVENT_QP_ACCESS_ERR: port_idx = ibevent->element.qp->port - 1; - set_bit(port_idx, &smcibdev->port_event_mask); - schedule_work(&smcibdev->port_event_work); + if (port_idx < SMC_MAX_PORTS) { + set_bit(port_idx, &smcibdev->port_event_mask); + schedule_work(&smcibdev->port_event_work); + } break; default: break; From 994c6e29564b7faa852f4b746989012dc6694082 Mon Sep 17 00:00:00 2001 From: Ursula Braun Date: Tue, 12 Feb 2019 16:29:56 +0100 Subject: [PATCH 7/7] MAINTAINERS: add Karsten as SMC maintainer Add Karsten as additional maintainer for Shared Memory Communications (SMC) Sockets. Acked-by: Karsten Graul Signed-off-by: Ursula Braun Signed-off-by: David S. Miller --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index 604bca2fc05d..f3af5cde6456 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -13760,6 +13760,7 @@ F: drivers/misc/sgi-xp/ SHARED MEMORY COMMUNICATIONS (SMC) SOCKETS M: Ursula Braun +M: Karsten Graul L: linux-s390@vger.kernel.org W: http://www.ibm.com/developerworks/linux/linux390/ S: Supported