From owner-dev-commits-src-all@freebsd.org Mon Jan 25 14:21:46 2021 Return-Path: Delivered-To: dev-commits-src-all@mailman.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.nyi.freebsd.org (Postfix) with ESMTP id 4A8AD4E3CA8; Mon, 25 Jan 2021 14:21:46 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256 client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "R3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 4DPX9k1cmQz4TtX; Mon, 25 Jan 2021 14:21:46 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org (gitrepo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:5]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 2A92318155; Mon, 25 Jan 2021 14:21:46 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org ([127.0.1.44]) by gitrepo.freebsd.org (8.16.1/8.16.1) with ESMTP id 10PELkWU074536; Mon, 25 Jan 2021 14:21:46 GMT (envelope-from git@gitrepo.freebsd.org) Received: (from git@localhost) by gitrepo.freebsd.org (8.16.1/8.16.1/Submit) id 10PELknQ074535; Mon, 25 Jan 2021 14:21:46 GMT (envelope-from git) Date: Mon, 25 Jan 2021 14:21:46 GMT Message-Id: <202101251421.10PELknQ074535@gitrepo.freebsd.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-branches@FreeBSD.org From: Mark Johnston Subject: git: 5ade41213f19 - stable/12 - safexcel: Dispatch requests to the current CPU's ring MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit X-Git-Committer: markj X-Git-Repository: src X-Git-Refname: refs/heads/stable/12 X-Git-Reftype: branch X-Git-Commit: 5ade41213f19c340ad022b2add816404442b1160 Auto-Submitted: auto-generated X-BeenThere: dev-commits-src-all@freebsd.org X-Mailman-Version: 2.1.34 Precedence: list List-Id: Commit messages for all branches of the src repository List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 25 Jan 2021 14:21:46 -0000 The branch stable/12 has been updated by markj: URL: https://cgit.FreeBSD.org/src/commit/?id=5ade41213f19c340ad022b2add816404442b1160 commit 5ade41213f19c340ad022b2add816404442b1160 Author: Mark Johnston AuthorDate: 2021-01-18 22:07:55 +0000 Commit: Mark Johnston CommitDate: 2021-01-25 14:19:20 +0000 safexcel: Dispatch requests to the current CPU's ring This gives better performance in some tests than the previous policy of statically binding each session to a ring. Sponsored by: Rubicon Communications, LLC (Netgate) (cherry picked from commit e934d455ba37ea777bd32cdcb0f9754865f9e818) --- sys/dev/safexcel/safexcel.c | 25 ++++++++++++++----------- sys/dev/safexcel/safexcel_var.h | 3 +-- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/sys/dev/safexcel/safexcel.c b/sys/dev/safexcel/safexcel.c index c1736fc5f080..ea48cf1b8879 100644 --- a/sys/dev/safexcel/safexcel.c +++ b/sys/dev/safexcel/safexcel.c @@ -36,6 +36,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include @@ -189,6 +190,8 @@ safexcel_rdr_intr(struct safexcel_softc *sc, int ringidx) req = STAILQ_FIRST(&ring->queued_requests); KASSERT(req != NULL, ("%s: expected %d pending requests", __func__, nreqs)); + KASSERT(req->ringidx == ringidx, + ("%s: ring index mismatch", __func__)); STAILQ_REMOVE_HEAD(&ring->queued_requests, link); mtx_unlock(&ring->mtx); @@ -748,9 +751,7 @@ safexcel_execute(struct safexcel_softc *sc, struct safexcel_ring *ring, mtx_assert(&ring->mtx, MA_OWNED); - ringidx = req->sess->ringidx; - if (STAILQ_EMPTY(&ring->ready_requests)) - return; + ringidx = req->ringidx; busy = !STAILQ_EMPTY(&ring->queued_requests); ncdescs = nrdescs = nreqs = 0; while ((req = STAILQ_FIRST(&ring->ready_requests)) != NULL && @@ -1024,7 +1025,7 @@ safexcel_init_hw(struct safexcel_softc *sc) static int safexcel_setup_dev_interrupts(struct safexcel_softc *sc) { - int i, j; + int error, i, j; for (i = 0; i < SAFEXCEL_MAX_RINGS && sc->sc_intr[i] != NULL; i++) { sc->sc_ih[i].sc = sc; @@ -1037,6 +1038,11 @@ safexcel_setup_dev_interrupts(struct safexcel_softc *sc) "couldn't setup interrupt %d\n", i); goto err; } + + error = bus_bind_intr(sc->sc_dev, sc->sc_intr[i], i % mp_ncpus); + if (error != 0) + device_printf(sc->sc_dev, + "failed to bind ring %d\n", error); } return (0); @@ -1181,6 +1187,7 @@ safexcel_attach(device_t dev) for (i = 0; i < SAFEXCEL_REQUESTS_PER_RING; i++) { req = &ring->requests[i]; req->sc = sc; + req->ringidx = ringidx; if (bus_dmamap_create(ring->data_dtag, BUS_DMA_COHERENT, &req->dmap) != 0) { for (j = 0; j < i; j++) @@ -1770,7 +1777,7 @@ safexcel_set_token(struct safexcel_request *req) cdesc = req->cdesc; sc = req->sc; - ringidx = req->sess->ringidx; + ringidx = req->ringidx; safexcel_set_command(req, cdesc); @@ -1983,7 +1990,7 @@ safexcel_create_chain_cb(void *arg, bus_dma_segment_t *segs, int nseg, crp = req->crp; sess = req->sess; - ring = &req->sc->sc_ring[sess->ringidx]; + ring = &req->sc->sc_ring[req->ringidx]; mtx_assert(&ring->mtx, MA_OWNED); @@ -2577,10 +2584,6 @@ safexcel_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri) } } - /* Bind each session to a fixed ring to minimize lock contention. */ - sess->ringidx = atomic_fetchadd_int(&sc->sc_ringidx, 1); - sess->ringidx %= sc->sc_config.rings; - return (0); } @@ -2655,7 +2658,7 @@ safexcel_process(device_t dev, struct cryptop *crp, int hint) } } - ring = &sc->sc_ring[sess->ringidx]; + ring = &sc->sc_ring[curcpu % sc->sc_config.rings]; mtx_lock(&ring->mtx); req = safexcel_alloc_request(sc, ring); if (__predict_false(req == NULL)) { diff --git a/sys/dev/safexcel/safexcel_var.h b/sys/dev/safexcel/safexcel_var.h index 3ced5055545a..089a2bc1171c 100644 --- a/sys/dev/safexcel/safexcel_var.h +++ b/sys/dev/safexcel/safexcel_var.h @@ -339,7 +339,6 @@ struct safexcel_res_descr_ring { }; struct safexcel_session { - int ringidx; uint32_t alg; /* cipher algorithm */ uint32_t digest; /* digest type */ uint32_t hash; /* hash algorithm */ @@ -366,6 +365,7 @@ struct safexcel_softc; struct safexcel_request { STAILQ_ENTRY(safexcel_request) link; bool dmap_loaded; + int ringidx; bus_dmamap_t dmap; int error; int cdescs, rdescs; @@ -414,7 +414,6 @@ struct safexcel_softc { struct safexcel_intr_handle sc_ih[SAFEXCEL_MAX_RINGS]; struct safexcel_ring sc_ring[SAFEXCEL_MAX_RINGS]; - int sc_ringidx; int32_t sc_cid; struct safexcel_reg_offsets sc_offsets;