Kernel/linux
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2025-01-01 10:43:43 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

svcrdma: Handle device removal outside of the CM event handler

Browse Source 
Synchronously wait for all disconnects to complete to ensure the
transports have divested all hardware resources before the
underlying RDMA device can safely be removed.

Reviewed-by: Sagi Grimberg <sagi@grimberg.me>
Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
This commit is contained in:
Chuck Lever 2024-07-29 16:52:32 -04:00
parent 438f81e0e9
commit c4de97f7c4
3 changed files with 40 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/linux/sunrpc/svc_rdma.h
View File

@ -48,6 +48,7 @@
#include <linux/sunrpc/rpc_rdma.h>
#include <linux/sunrpc/rpc_rdma_cid.h>
#include <linux/sunrpc/svc_rdma_pcl.h>
#include <linux/sunrpc/rdma_rn.h>
#include <linux/percpu_counter.h>
#include <rdma/ib_verbs.h>
@ -76,6 +77,7 @@ struct svcxprt_rdma {
struct svc_xprt sc_xprt; /* SVC transport structure */
struct rdma_cm_id *sc_cm_id; /* RDMA connection id */
struct list_head sc_accept_q; /* Conn. waiting accept */
struct rpcrdma_notification sc_rn; /* removal notification */
int sc_ord; /* RDMA read limit */
int sc_max_send_sges;
bool sc_snd_w_inv; /* OK to use Send With Invalidate */

23
include/trace/events/rpcrdma.h
View File

@ -2172,6 +2172,29 @@ TRACE_EVENT(svcrdma_qp_error,
)
);
TRACE_EVENT(svcrdma_device_removal,
TP_PROTO(
const struct rdma_cm_id *id
),
TP_ARGS(id),
TP_STRUCT__entry(
__string(name, id->device->name)
__array(unsigned char, addr, sizeof(struct sockaddr_in6))
),
TP_fast_assign(
__assign_str(name);
memcpy(__entry->addr, &id->route.addr.dst_addr,
sizeof(struct sockaddr_in6));
),
TP_printk("device %s to be removed, disconnecting %pISpc\n",
__get_str(name), __entry->addr
)
);
DECLARE_EVENT_CLASS(svcrdma_sendqueue_class,
TP_PROTO(
const struct svcxprt_rdma *rdma,

16
net/sunrpc/xprtrdma/svc_rdma_transport.c
View File

@ -339,7 +339,6 @@ static int svc_rdma_cma_handler(struct rdma_cm_id *cma_id,
svc_xprt_enqueue(xprt);
break;
case RDMA_CM_EVENT_DISCONNECTED:
case RDMA_CM_EVENT_DEVICE_REMOVAL:
svc_xprt_deferred_close(xprt);
break;
default:
@ -384,6 +383,16 @@ static struct svc_xprt *svc_rdma_create(struct svc_serv *serv,
return &cma_xprt->sc_xprt;
}
static void svc_rdma_xprt_done(struct rpcrdma_notification *rn)
{
struct svcxprt_rdma *rdma = container_of(rn, struct svcxprt_rdma,
sc_rn);
struct rdma_cm_id *id = rdma->sc_cm_id;
trace_svcrdma_device_removal(id);
svc_xprt_close(&rdma->sc_xprt);
}
/*
* This is the xpo_recvfrom function for listening endpoints. Its
* purpose is to accept incoming connections. The CMA callback handler
@ -425,6 +434,9 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt)
dev = newxprt->sc_cm_id->device;
newxprt->sc_port_num = newxprt->sc_cm_id->port_num;
if (rpcrdma_rn_register(dev, &newxprt->sc_rn, svc_rdma_xprt_done))
goto errout;
newxprt->sc_max_req_size = svcrdma_max_req_size;
newxprt->sc_max_requests = svcrdma_max_requests;
newxprt->sc_max_bc_requests = svcrdma_max_bc_requests;
@ -580,6 +592,7 @@ static void __svc_rdma_free(struct work_struct *work)
{
struct svcxprt_rdma *rdma =
container_of(work, struct svcxprt_rdma, sc_work);
struct ib_device *device = rdma->sc_cm_id->device;
/* This blocks until the Completion Queues are empty */
if (rdma->sc_qp && !IS_ERR(rdma->sc_qp))
@ -608,6 +621,7 @@ static void __svc_rdma_free(struct work_struct *work)
/* Destroy the CM ID */
rdma_destroy_id(rdma->sc_cm_id);
rpcrdma_rn_unregister(device, &rdma->sc_rn);
kfree(rdma);
}