From nobody Tue Dec 14 23:30:24 2021
X-Original-To: dev-commits-src-main@mlmmj.nyi.freebsd.org
Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1])
	by mlmmj.nyi.freebsd.org (Postfix) with ESMTP id 5FCA818DF500;
	Tue, 14 Dec 2021 23:30:25 +0000 (UTC)
	(envelope-from git@FreeBSD.org)
Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3])
	(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
	 key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256
	 client-signature RSA-PSS (4096 bits) client-digest SHA256)
	(Client CN "mxrelay.nyi.freebsd.org", Issuer "R3" (verified OK))
	by mx1.freebsd.org (Postfix) with ESMTPS id 4JDF3j04T2z4t56;
	Tue, 14 Dec 2021 23:30:25 +0000 (UTC)
	(envelope-from git@FreeBSD.org)
Received: from gitrepo.freebsd.org (gitrepo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:5])
	(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
	 key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256)
	(Client did not present a certificate)
	by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id D313711840;
	Tue, 14 Dec 2021 23:30:24 +0000 (UTC)
	(envelope-from git@FreeBSD.org)
Received: from gitrepo.freebsd.org ([127.0.1.44])
	by gitrepo.freebsd.org (8.16.1/8.16.1) with ESMTP id 1BENUO0S039828;
	Tue, 14 Dec 2021 23:30:24 GMT
	(envelope-from git@gitrepo.freebsd.org)
Received: (from git@localhost)
	by gitrepo.freebsd.org (8.16.1/8.16.1/Submit) id 1BENUObx039827;
	Tue, 14 Dec 2021 23:30:24 GMT
	(envelope-from git)
Date: Tue, 14 Dec 2021 23:30:24 GMT
Message-Id: <202112142330.1BENUObx039827@gitrepo.freebsd.org>
To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org,
        dev-commits-src-main@FreeBSD.org
From: Chuck Tuffli <chuck@FreeBSD.org>
Subject: git: 9f1fa1a46117 - main - bhyve nvme: Add AEN support to NVMe emulation
List-Id: Commit messages for the main branch of the src repository <dev-commits-src-main.freebsd.org>
List-Archive: https://lists.freebsd.org/archives/dev-commits-src-main
List-Help: <mailto:dev-commits-src-main+help@freebsd.org>
List-Post: <mailto:dev-commits-src-main@freebsd.org>
List-Subscribe: <mailto:dev-commits-src-main+subscribe@freebsd.org>
List-Unsubscribe: <mailto:dev-commits-src-main+unsubscribe@freebsd.org>
Sender: owner-dev-commits-src-main@freebsd.org
X-BeenThere: dev-commits-src-main@freebsd.org
MIME-Version: 1.0
Content-Type: text/plain; charset=utf-8
Content-Transfer-Encoding: 8bit
X-Git-Committer: chuck
X-Git-Repository: src
X-Git-Refname: refs/heads/main
X-Git-Reftype: branch
X-Git-Commit: 9f1fa1a46117a418e7f7fbfb1be437100fad531f
Auto-Submitted: auto-generated
ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=freebsd.org;
	s=dkim; t=1639524625;
	h=from:from:reply-to:subject:subject:date:date:message-id:message-id:
	 to:to:cc:mime-version:mime-version:content-type:content-type:
	 content-transfer-encoding:content-transfer-encoding;
	bh=jR9RXdoQcEf4dRvWvDpH0bbGagEyJS29Zkij0FRB+r0=;
	b=IPNfPeuUi3UhDnp3g/9OBxuidXMY9KqHjZ1lw9aG8O6uqWKnlXIsYH3bY/Pql56wjIxrn+
	bKRRemqG20Ij6syrVtwZU1NIE2y6COCT+I83lIRsUE+RkIhx67WtkrIP+44AJMVV6qPj4d
	qCnjlr8Aud13hPsbY6KpZDhlc/RRpYJkO1h0CMurZeFQFcNgrKpGtWrKfVEP/PcE6k0UlS
	WzQ22MPIUptVwwpap+xkym3IO6zc/tpPh557R54+0n6WXv/ZZTPPW/gzK/eFr/5pzOFi5F
	scP9Y89Fwp9seZG9OXWEWFbbPUNELrUJ6sYF8lbnjh5jCaSFxofYBlGF0GosHA==
ARC-Seal: i=1; s=dkim; d=freebsd.org; t=1639524625; a=rsa-sha256; cv=none;
	b=A92f7iYypAsdTPK9sF+UwkQ7zBUPQL2STKYIhFp/rhdwdtsFmDTGeuAkeDvhTkyTyLJ40C
	zBxwjc6gdMjFkd4Ku3jn4UWlh1jcjNRYO+vXAef31j2tAo/AqA1+2Q+3gAdFVEbNOGPNCI
	m74MvtM89npuILy24slyqTzzYmDLIf1c46BOegfbBb8F2eqiHvd1lcWxeWxCWn41LZbYFj
	7uCtic+LDFGaAhjxzKz5qHBR7eBmS4W255HDC4M9HcCZbrGEI+oHlv1ZLnO1JtlSQ5jtFo
	cKqvMnrBic1uDtoOeMZKp1f+gUK+65iatyOP8w0xq5JjlagOVYIv6LGVkDUBkg==
ARC-Authentication-Results: i=1;
	mx1.freebsd.org;
	none
X-ThisMailContainsUnwantedMimeParts: N

The branch main has been updated by chuck:

URL: https://cgit.FreeBSD.org/src/commit/?id=9f1fa1a46117a418e7f7fbfb1be437100fad531f

commit 9f1fa1a46117a418e7f7fbfb1be437100fad531f
Author:     Chuck Tuffli <chuck@FreeBSD.org>
AuthorDate: 2021-12-15 07:16:49 +0000
Commit:     Chuck Tuffli <chuck@FreeBSD.org>
CommitDate: 2021-12-15 07:16:49 +0000

    bhyve nvme: Add AEN support to NVMe emulation
    
    Add Asynchronous Event Notification infrastructure to the NVMe
    emulation.
    
    Reviewed by:    imp, grehan
    MFC after:      1 month
    Differential Revision:  https://reviews.freebsd.org/D32952
---
 usr.sbin/bhyve/pci_nvme.c | 273 ++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 266 insertions(+), 7 deletions(-)

diff --git a/usr.sbin/bhyve/pci_nvme.c b/usr.sbin/bhyve/pci_nvme.c
index 034c9409cea3..6fd1aa14b9e3 100644
--- a/usr.sbin/bhyve/pci_nvme.c
+++ b/usr.sbin/bhyve/pci_nvme.c
@@ -65,6 +65,7 @@ __FBSDID("$FreeBSD$");
 
 #include <assert.h>
 #include <pthread.h>
+#include <pthread_np.h>
 #include <semaphore.h>
 #include <stdbool.h>
 #include <stddef.h>
@@ -250,11 +251,39 @@ struct nvme_feature_obj {
 
 #define NVME_FID_MAX		(NVME_FEAT_ENDURANCE_GROUP_EVENT_CONFIGURATION + 1)
 
+typedef enum {
+	PCI_NVME_AE_TYPE_ERROR = 0,
+	PCI_NVME_AE_TYPE_SMART,
+	PCI_NVME_AE_TYPE_NOTICE,
+	PCI_NVME_AE_TYPE_IO_CMD = 6,
+	PCI_NVME_AE_TYPE_VENDOR = 7,
+	PCI_NVME_AE_TYPE_MAX		/* Must be last */
+} pci_nvme_async_type;
+
+/* Asynchronous Event Requests */
 struct pci_nvme_aer {
 	STAILQ_ENTRY(pci_nvme_aer) link;
 	uint16_t	cid;	/* Command ID of the submitted AER */
 };
 
+typedef enum {
+	PCI_NVME_AE_INFO_NS_ATTR_CHANGED = 0,
+	PCI_NVME_AE_INFO_FW_ACTIVATION,
+	PCI_NVME_AE_INFO_TELEMETRY_CHANGE,
+	PCI_NVME_AE_INFO_ANA_CHANGE,
+	PCI_NVME_AE_INFO_PREDICT_LATENCY_CHANGE,
+	PCI_NVME_AE_INFO_LBA_STATUS_ALERT,
+	PCI_NVME_AE_INFO_ENDURANCE_GROUP_CHANGE,
+	PCI_NVME_AE_INFO_MAX,
+} pci_nvme_async_info;
+
+/* Asynchronous Event Notifications */
+struct pci_nvme_aen {
+	pci_nvme_async_type atype;
+	uint32_t	event_data;
+	bool		posted;
+};
+
 struct pci_nvme_softc {
 	struct pci_devinst *nsc_pi;
 
@@ -302,10 +331,21 @@ struct pci_nvme_softc {
 	uint32_t	write_dunits_remainder;
 
 	STAILQ_HEAD(, pci_nvme_aer) aer_list;
+	pthread_mutex_t	aer_mtx;
 	uint32_t	aer_count;
+	struct pci_nvme_aen aen[PCI_NVME_AE_TYPE_MAX];
+	pthread_t	aen_tid;
+	pthread_mutex_t	aen_mtx;
+	pthread_cond_t	aen_cond;
 };
 
 
+static void pci_nvme_cq_update(struct pci_nvme_softc *sc,
+    struct nvme_completion_queue *cq,
+    uint32_t cdw0,
+    uint16_t cid,
+    uint16_t sqid,
+    uint16_t status);
 static struct pci_nvme_ioreq *pci_nvme_get_ioreq(struct pci_nvme_softc *);
 static void pci_nvme_release_ioreq(struct pci_nvme_softc *, struct pci_nvme_ioreq *);
 static void pci_nvme_io_done(struct blockif_req *, int);
@@ -360,6 +400,8 @@ static void nvme_feature_iv_config(struct pci_nvme_softc *,
     struct nvme_command *,
     struct nvme_completion *);
 
+static void *aen_thr(void *arg);
+
 static __inline void
 cpywithpad(char *dst, size_t dst_size, const char *src, char pad)
 {
@@ -623,6 +665,8 @@ pci_nvme_init_features(struct pci_nvme_softc *sc)
 	sc->feat[NVME_FEAT_NUMBER_OF_QUEUES].set = nvme_feature_num_queues;
 	sc->feat[NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION].set =
 	    nvme_feature_iv_config;
+	/* Enable all AENs by default */
+	sc->feat[NVME_FEAT_ASYNC_EVENT_CONFIGURATION].cdw11 = 0x31f;
 	sc->feat[NVME_FEAT_PREDICTABLE_LATENCY_MODE_CONFIG].get =
 	    nvme_feature_invalid_cb;
 	sc->feat[NVME_FEAT_PREDICTABLE_LATENCY_MODE_WINDOW].get =
@@ -630,32 +674,42 @@ pci_nvme_init_features(struct pci_nvme_softc *sc)
 }
 
 static void
-pci_nvme_aer_init(struct pci_nvme_softc *sc)
+pci_nvme_aer_reset(struct pci_nvme_softc *sc)
 {
 
 	STAILQ_INIT(&sc->aer_list);
 	sc->aer_count = 0;
 }
 
+static void
+pci_nvme_aer_init(struct pci_nvme_softc *sc)
+{
+
+	pthread_mutex_init(&sc->aer_mtx, NULL);
+	pci_nvme_aer_reset(sc);
+}
+
 static void
 pci_nvme_aer_destroy(struct pci_nvme_softc *sc)
 {
 	struct pci_nvme_aer *aer = NULL;
 
+	pthread_mutex_lock(&sc->aer_mtx);
 	while (!STAILQ_EMPTY(&sc->aer_list)) {
 		aer = STAILQ_FIRST(&sc->aer_list);
 		STAILQ_REMOVE_HEAD(&sc->aer_list, link);
 		free(aer);
 	}
+	pthread_mutex_unlock(&sc->aer_mtx);
 
-	pci_nvme_aer_init(sc);
+	pci_nvme_aer_reset(sc);
 }
 
 static bool
 pci_nvme_aer_available(struct pci_nvme_softc *sc)
 {
 
-	return (!STAILQ_EMPTY(&sc->aer_list));
+	return (sc->aer_count != 0);
 }
 
 static bool
@@ -687,11 +741,13 @@ pci_nvme_aer_add(struct pci_nvme_softc *sc, uint16_t cid)
 	if (aer == NULL)
 		return (-1);
 
-	sc->aer_count++;
-
 	/* Save the Command ID for use in the completion message */
 	aer->cid = cid;
+
+	pthread_mutex_lock(&sc->aer_mtx);
+	sc->aer_count++;
 	STAILQ_INSERT_TAIL(&sc->aer_list, aer, link);
+	pthread_mutex_unlock(&sc->aer_mtx);
 
 	return (0);
 }
@@ -707,15 +763,208 @@ pci_nvme_aer_get(struct pci_nvme_softc *sc)
 {
 	struct pci_nvme_aer *aer = NULL;
 
+	pthread_mutex_lock(&sc->aer_mtx);
 	aer = STAILQ_FIRST(&sc->aer_list);
 	if (aer != NULL) {
 		STAILQ_REMOVE_HEAD(&sc->aer_list, link);
 		sc->aer_count--;
 	}
+	pthread_mutex_unlock(&sc->aer_mtx);
 	
 	return (aer);
 }
 
+static void
+pci_nvme_aen_reset(struct pci_nvme_softc *sc)
+{
+	uint32_t	atype;
+
+	memset(sc->aen, 0, PCI_NVME_AE_TYPE_MAX * sizeof(struct pci_nvme_aen));
+
+	for (atype = 0; atype < PCI_NVME_AE_TYPE_MAX; atype++) {
+		sc->aen[atype].atype = atype;
+	}
+}
+
+static void
+pci_nvme_aen_init(struct pci_nvme_softc *sc)
+{
+	char nstr[80];
+
+	pci_nvme_aen_reset(sc);
+
+	pthread_mutex_init(&sc->aen_mtx, NULL);
+	pthread_create(&sc->aen_tid, NULL, aen_thr, sc);
+	snprintf(nstr, sizeof(nstr), "nvme-aen-%d:%d", sc->nsc_pi->pi_slot,
+	    sc->nsc_pi->pi_func);
+	pthread_set_name_np(sc->aen_tid, nstr);
+}
+
+static void
+pci_nvme_aen_destroy(struct pci_nvme_softc *sc)
+{
+
+	pci_nvme_aen_reset(sc);
+}
+
+/* Notify the AEN thread of pending work */
+static void
+pci_nvme_aen_notify(struct pci_nvme_softc *sc)
+{
+
+	pthread_cond_signal(&sc->aen_cond);
+}
+
+/*
+ * Post an Asynchronous Event Notification
+ */
+static int32_t
+pci_nvme_aen_post(struct pci_nvme_softc *sc, pci_nvme_async_type atype,
+		uint32_t event_data)
+{
+	struct pci_nvme_aen *aen;
+
+	if (atype >= PCI_NVME_AE_TYPE_MAX) {
+		return(EINVAL);
+	}
+
+	pthread_mutex_lock(&sc->aen_mtx);
+	aen = &sc->aen[atype];
+
+	/* Has the controller already posted an event of this type? */
+	if (aen->posted) {
+		pthread_mutex_unlock(&sc->aen_mtx);
+		return(EALREADY);
+	}
+
+	aen->event_data = event_data;
+	aen->posted = true;
+	pthread_mutex_unlock(&sc->aen_mtx);
+
+	pci_nvme_aen_notify(sc);
+
+	return(0);
+}
+
+static void
+pci_nvme_aen_process(struct pci_nvme_softc *sc)
+{
+	struct pci_nvme_aer *aer;
+	struct pci_nvme_aen *aen;
+	pci_nvme_async_type atype;
+	uint32_t mask;
+	uint16_t status;
+	uint8_t lid;
+
+	assert(pthread_mutex_isowned_np(&sc->aen_mtx));
+	for (atype = 0; atype < PCI_NVME_AE_TYPE_MAX; atype++) {
+		aen = &sc->aen[atype];
+		/* Previous iterations may have depleted the available AER's */
+		if (!pci_nvme_aer_available(sc)) {
+			DPRINTF("%s: no AER", __func__);
+			break;
+		}
+
+		if (!aen->posted) {
+			DPRINTF("%s: no AEN posted for atype=%#x", __func__, atype);
+			continue;
+		}
+
+		status = NVME_SC_SUCCESS;
+
+		/* Is the event masked? */
+		mask =
+		    sc->feat[NVME_FEAT_ASYNC_EVENT_CONFIGURATION].cdw11;
+
+		DPRINTF("%s: atype=%#x mask=%#x event_data=%#x", __func__, atype, mask, aen->event_data);
+		switch (atype) {
+		case PCI_NVME_AE_TYPE_ERROR:
+			lid = NVME_LOG_ERROR;
+			break;
+		case PCI_NVME_AE_TYPE_SMART:
+			mask &= 0xff;
+			if ((mask & aen->event_data) == 0)
+				continue;
+			lid = NVME_LOG_HEALTH_INFORMATION;
+			break;
+		case PCI_NVME_AE_TYPE_NOTICE:
+			if (aen->event_data >= PCI_NVME_AE_INFO_MAX) {
+				EPRINTLN("%s unknown AEN notice type %u",
+				    __func__, aen->event_data);
+				status = NVME_SC_INTERNAL_DEVICE_ERROR;
+				break;
+			}
+			mask >>= 8;
+			if (((1 << aen->event_data) & mask) == 0)
+				continue;
+			switch (aen->event_data) {
+			case PCI_NVME_AE_INFO_NS_ATTR_CHANGED:
+				lid = NVME_LOG_CHANGED_NAMESPACE;
+				break;
+			case PCI_NVME_AE_INFO_FW_ACTIVATION:
+				lid = NVME_LOG_FIRMWARE_SLOT;
+				break;
+			case PCI_NVME_AE_INFO_TELEMETRY_CHANGE:
+				lid = NVME_LOG_TELEMETRY_CONTROLLER_INITIATED;
+				break;
+			case PCI_NVME_AE_INFO_ANA_CHANGE:
+				lid = NVME_LOG_ASYMMETRIC_NAMESPAVE_ACCESS; //TODO spelling
+				break;
+			case PCI_NVME_AE_INFO_PREDICT_LATENCY_CHANGE:
+				lid = NVME_LOG_PREDICTABLE_LATENCY_EVENT_AGGREGATE;
+				break;
+			case PCI_NVME_AE_INFO_LBA_STATUS_ALERT:
+				lid = NVME_LOG_LBA_STATUS_INFORMATION;
+				break;
+			case PCI_NVME_AE_INFO_ENDURANCE_GROUP_CHANGE:
+				lid = NVME_LOG_ENDURANCE_GROUP_EVENT_AGGREGATE;
+				break;
+			default:
+				lid = 0;
+			}
+			break;
+		default:
+			/* bad type?!? */
+			EPRINTLN("%s unknown AEN type %u", __func__, atype);
+			status = NVME_SC_INTERNAL_DEVICE_ERROR;
+			break;
+		}
+
+		aer = pci_nvme_aer_get(sc);
+		assert(aer != NULL);
+
+		DPRINTF("%s: CID=%#x CDW0=%#x", __func__, aer->cid, (lid << 16) | (aen->event_data << 8) | atype);
+		pci_nvme_cq_update(sc, &sc->compl_queues[0],
+		    (lid << 16) | (aen->event_data << 8) | atype, /* cdw0 */
+		    aer->cid,
+		    0,		/* SQID */
+		    status);
+
+		aen->event_data = 0;
+		aen->posted = false;
+
+		pci_generate_msix(sc->nsc_pi, 0);
+	}
+}
+
+static void *
+aen_thr(void *arg)
+{
+	struct pci_nvme_softc *sc;
+
+	sc = arg;
+
+	pthread_mutex_lock(&sc->aen_mtx);
+	for (;;) {
+		pci_nvme_aen_process(sc);
+		pthread_cond_wait(&sc->aen_cond, &sc->aen_mtx);
+	}
+	pthread_mutex_unlock(&sc->aen_mtx);
+
+	pthread_exit(NULL);
+	return (NULL);
+}
+
 static void
 pci_nvme_reset_locked(struct pci_nvme_softc *sc)
 {
@@ -756,6 +1005,7 @@ pci_nvme_reset_locked(struct pci_nvme_softc *sc)
 	sc->num_q_is_set = false;
 
 	pci_nvme_aer_destroy(sc);
+	pci_nvme_aen_destroy(sc);
 }
 
 static void
@@ -1406,8 +1656,13 @@ nvme_opc_set_features(struct pci_nvme_softc *sc, struct nvme_command *command,
 	if (feat->set)
 		feat->set(sc, feat, command, compl);
 
-	if (compl->status == NVME_SC_SUCCESS)
+	DPRINTF("%s: status=%#x cdw11=%#x", __func__, compl->status, command->cdw11);
+	if (compl->status == NVME_SC_SUCCESS) {
 		feat->cdw11 = command->cdw11;
+		if ((fid == NVME_FEAT_ASYNC_EVENT_CONFIGURATION) &&
+		    (command->cdw11 != 0))
+			pci_nvme_aen_notify(sc);
+	}
 
 	return (0);
 }
@@ -1525,7 +1780,8 @@ static int
 nvme_opc_async_event_req(struct pci_nvme_softc* sc,
 	struct nvme_command* command, struct nvme_completion* compl)
 {
-	DPRINTF("%s async event request 0x%x", __func__, command->cdw11);
+	DPRINTF("%s async event request count=%u aerl=%u cid=%#x", __func__,
+	    sc->aer_count, sc->ctrldata.aerl, command->cid);
 
 	/* Don't exceed the Async Event Request Limit (AERL). */
 	if (pci_nvme_aer_limit_reached(sc)) {
@@ -1546,6 +1802,7 @@ nvme_opc_async_event_req(struct pci_nvme_softc* sc,
 	 * there is an event reflective of the request to get event.
 	 */
 	compl->status = NVME_NO_STATUS;
+	pci_nvme_aen_notify(sc);
 
 	return (0);
 }
@@ -2025,6 +2282,7 @@ nvme_opc_write_read(struct pci_nvme_softc *sc,
 
 	lba = ((uint64_t)cmd->cdw11 << 32) | cmd->cdw10;
 	nblocks = (cmd->cdw12 & 0xFFFF) + 1;
+
 	if (pci_nvme_out_of_range(nvstore, lba, nblocks)) {
 		WPRINTF("%s command would exceed LBA range", __func__);
 		pci_nvme_status_genc(status, NVME_SC_LBA_OUT_OF_RANGE);
@@ -2789,6 +3047,7 @@ pci_nvme_init(struct vmctx *ctx, struct pci_devinst *pi, nvlist_t *nvl)
 	pci_nvme_init_features(sc);
 
 	pci_nvme_aer_init(sc);
+	pci_nvme_aen_init(sc);
 
 	pci_nvme_reset(sc);