Date: Sat, 15 May 2021 11:05:49 GMT From: Edward Tomasz Napierala <trasz@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org Subject: git: 3394d4239b85 - main - cam: allocate CCBs from UMA for SCSI and ATA IO Message-ID: <202105151105.14FB5nv7005915@gitrepo.freebsd.org>
next in thread | raw e-mail | index | archive | help
The branch main has been updated by trasz: URL: https://cgit.FreeBSD.org/src/commit/?id=3394d4239b85b5577845d9e6de4e97b18d3dba58 commit 3394d4239b85b5577845d9e6de4e97b18d3dba58 Author: Edward Tomasz Napierala <trasz@FreeBSD.org> AuthorDate: 2021-05-15 10:17:22 +0000 Commit: Edward Tomasz Napierala <trasz@FreeBSD.org> CommitDate: 2021-05-15 11:03:49 +0000 cam: allocate CCBs from UMA for SCSI and ATA IO This patch makes it possible for CAM to use small CCBs allocated from an periph-specific UMA zone instead of the usual, huge ones. The end result is that CCBs issued via da(4) take 544B (size of ccb_scsiio) instead of the usual 2kB (size of 'union ccb', ~1.5kB, rounded up by malloc(9)). For ATA it's 272B. We waste less memory, we avoid zeroing the unused 1kB, and it should be easier to allocate those CCBs in low memory conditions. It should also be possible to use uma_zone_reserve(9) to improve behaviour in low memory conditions even further. Note that this does not change the size, or the layout, of CCBs as such. CCBs get allocated in various different ways, in particular on the stack, and I don't want to redo all that. Instead, this provides an opt-in mechanism for the periph to declare "my start() callback is fine with receiving a CCB allocated from this UMA zone". In other words, most of the code works exactly as it used to; the change only happens to IOs issued by xpt_run_allockq(), which is - conveniently - pretty much all that matters for performance. The reason for doing it this way is that it's pretty small, localized change, and can be implemented gradually and iteratively: take a periph, make sure its start() callback only casts the CCBs it takes to a particular type of CCB, for example ccb_scsiio, and that it only casts CCBs returned by cam_periph_getccb() to that type, then add UMA zone for that size, and declare it safe to XPT. This is disabled by default. Set 'kern.cam.ada.enable_uma_ccbs=1' and 'kern.cam.da.enable_uma_ccbs=1' tunables to enable it. Testing is welcome; I will flip the default to enable in two weeks from now. Reviewed By: imp Sponsored by: NetApp, Inc. Sponsored by: Klara, Inc. Differential Revision: https://reviews.freebsd.org/D28674 --- sys/cam/ata/ata_da.c | 18 ++++++++++++++++++ sys/cam/ata/ata_xpt.c | 7 +++++++ sys/cam/cam_ccb.h | 14 +++++++++++++- sys/cam/cam_periph.h | 3 +++ sys/cam/cam_xpt.c | 32 +++++++++++++++++++++++++++++--- sys/cam/scsi/scsi_da.c | 18 ++++++++++++++++++ sys/cam/scsi/scsi_xpt.c | 7 +++++++ 7 files changed, 95 insertions(+), 4 deletions(-) diff --git a/sys/cam/ata/ata_da.c b/sys/cam/ata/ata_da.c index c29235e64e81..a8b67b2254a6 100644 --- a/sys/cam/ata/ata_da.c +++ b/sys/cam/ata/ata_da.c @@ -297,6 +297,8 @@ struct ada_softc { char announce_buffer[ADA_ANNOUNCE_SZ]; }; +static uma_zone_t ada_ccb_zone; + struct ada_quirk_entry { struct scsi_inquiry_pattern inq_pat; ada_quirks quirks; @@ -902,6 +904,7 @@ static int ada_spindown_suspend = ADA_DEFAULT_SPINDOWN_SUSPEND; static int ada_read_ahead = ADA_DEFAULT_READ_AHEAD; static int ada_write_cache = ADA_DEFAULT_WRITE_CACHE; static int ada_enable_biospeedup = 1; +static int ada_enable_uma_ccbs = 0; static SYSCTL_NODE(_kern_cam, OID_AUTO, ada, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "CAM Direct Access Disk driver"); @@ -921,6 +924,8 @@ SYSCTL_INT(_kern_cam_ada, OID_AUTO, write_cache, CTLFLAG_RWTUN, &ada_write_cache, 0, "Enable disk write cache"); SYSCTL_INT(_kern_cam_ada, OID_AUTO, enable_biospeedup, CTLFLAG_RDTUN, &ada_enable_biospeedup, 0, "Enable BIO_SPEEDUP processing"); +SYSCTL_INT(_kern_cam_ada, OID_AUTO, enable_uma_ccbs, CTLFLAG_RWTUN, + &ada_enable_uma_ccbs, 0, "Use UMA for CCBs"); /* * ADA_ORDEREDTAG_INTERVAL determines how often, relative @@ -1178,6 +1183,10 @@ adainit(void) { cam_status status; + ada_ccb_zone = uma_zcreate("ada_ccb", + sizeof(struct ccb_ataio), NULL, NULL, NULL, NULL, + UMA_ALIGN_PTR, 0); + /* * Install a global async callback. This callback will * receive async callbacks like "new device found". @@ -1855,6 +1864,15 @@ adaregister(struct cam_periph *periph, void *arg) "kern.cam.ada.%d.write_cache", periph->unit_number); TUNABLE_INT_FETCH(announce_buf, &softc->write_cache); + /* + * Let XPT know we can use UMA-allocated CCBs. + */ + if (ada_enable_uma_ccbs) { + KASSERT(ada_ccb_zone != NULL, + ("%s: NULL ada_ccb_zone", __func__)); + periph->ccb_zone = ada_ccb_zone; + } + /* * Set support flags based on the Identify data and quirks. */ diff --git a/sys/cam/ata/ata_xpt.c b/sys/cam/ata/ata_xpt.c index 0f94e556745a..c13c7b493c78 100644 --- a/sys/cam/ata/ata_xpt.c +++ b/sys/cam/ata/ata_xpt.c @@ -1795,6 +1795,13 @@ static void ata_action(union ccb *start_ccb) { + if (start_ccb->ccb_h.func_code != XPT_ATA_IO) { + KASSERT((start_ccb->ccb_h.alloc_flags & CAM_CCB_FROM_UMA) == 0, + ("%s: ccb %p, func_code %#x should not be allocated " + "from UMA zone\n", + __func__, start_ccb, start_ccb->ccb_h.func_code)); + } + switch (start_ccb->ccb_h.func_code) { case XPT_SET_TRAN_SETTINGS: { diff --git a/sys/cam/cam_ccb.h b/sys/cam/cam_ccb.h index 221b24a7c187..2545e40e192d 100644 --- a/sys/cam/cam_ccb.h +++ b/sys/cam/cam_ccb.h @@ -58,6 +58,12 @@ /* Struct definitions for CAM control blocks */ /* Common CCB header */ + +/* CCB memory allocation flags */ +typedef enum { + CAM_CCB_FROM_UMA = 0x00000001,/* CCB from a periph UMA zone */ +} ccb_alloc_flags; + /* CAM CCB flags */ typedef enum { CAM_CDB_POINTER = 0x00000001,/* The CDB field is a pointer */ @@ -341,7 +347,13 @@ struct ccb_hdr { camq_entry xpt_links; /* For chaining in the XPT layer */ camq_entry sim_links; /* For chaining in the SIM layer */ camq_entry periph_links; /* For chaining in the type driver */ - u_int32_t retry_count; +#if BYTE_ORDER == LITTLE_ENDIAN + u_int16_t retry_count; + u_int16_t alloc_flags; /* ccb_alloc_flags */ +#else + u_int16_t alloc_flags; /* ccb_alloc_flags */ + u_int16_t retry_count; +#endif void (*cbfcnp)(struct cam_periph *, union ccb *); /* Callback on completion function */ xpt_opcode func_code; /* XPT function code */ diff --git a/sys/cam/cam_periph.h b/sys/cam/cam_periph.h index 15a239decf0a..9c323394797c 100644 --- a/sys/cam/cam_periph.h +++ b/sys/cam/cam_periph.h @@ -42,6 +42,8 @@ #include <sys/sysctl.h> #include <sys/taskqueue.h> +#include <vm/uma.h> + #include <cam/cam_xpt.h> struct devstat; @@ -147,6 +149,7 @@ struct cam_periph { ac_callback_t *deferred_callback; ac_code deferred_ac; struct task periph_run_task; + uma_zone_t ccb_zone; }; #define CAM_PERIPH_MAXMAPS 2 diff --git a/sys/cam/cam_xpt.c b/sys/cam/cam_xpt.c index b7bc2b74da6c..33361cfb68a5 100644 --- a/sys/cam/cam_xpt.c +++ b/sys/cam/cam_xpt.c @@ -4693,7 +4693,17 @@ xpt_alloc_ccb_nowait(void) void xpt_free_ccb(union ccb *free_ccb) { - free(free_ccb, M_CAMCCB); + struct cam_periph *periph; + + if (free_ccb->ccb_h.alloc_flags & CAM_CCB_FROM_UMA) { + /* + * Looks like a CCB allocated from a periph UMA zone. + */ + periph = free_ccb->ccb_h.path->periph; + uma_zfree(periph->ccb_zone, free_ccb); + } else { + free(free_ccb, M_CAMCCB); + } } /* Private XPT functions */ @@ -4707,10 +4717,18 @@ static union ccb * xpt_get_ccb_nowait(struct cam_periph *periph) { union ccb *new_ccb; + int alloc_flags; - new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_NOWAIT); + if (periph->ccb_zone != NULL) { + alloc_flags = CAM_CCB_FROM_UMA; + new_ccb = uma_zalloc(periph->ccb_zone, M_ZERO|M_NOWAIT); + } else { + alloc_flags = 0; + new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_NOWAIT); + } if (new_ccb == NULL) return (NULL); + new_ccb->ccb_h.alloc_flags = alloc_flags; periph->periph_allocated++; cam_ccbq_take_opening(&periph->path->device->ccbq); return (new_ccb); @@ -4720,9 +4738,17 @@ static union ccb * xpt_get_ccb(struct cam_periph *periph) { union ccb *new_ccb; + int alloc_flags; cam_periph_unlock(periph); - new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_WAITOK); + if (periph->ccb_zone != NULL) { + alloc_flags = CAM_CCB_FROM_UMA; + new_ccb = uma_zalloc(periph->ccb_zone, M_ZERO|M_WAITOK); + } else { + alloc_flags = 0; + new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_WAITOK); + } + new_ccb->ccb_h.alloc_flags = alloc_flags; cam_periph_lock(periph); periph->periph_allocated++; cam_ccbq_take_opening(&periph->path->device->ccbq); diff --git a/sys/cam/scsi/scsi_da.c b/sys/cam/scsi/scsi_da.c index baba5d9ed839..05dd9b6e7566 100644 --- a/sys/cam/scsi/scsi_da.c +++ b/sys/cam/scsi/scsi_da.c @@ -403,6 +403,8 @@ struct da_softc { softc->delete_available &= ~(1 << delete_method); \ } +static uma_zone_t da_ccb_zone; + struct da_quirk_entry { struct scsi_inquiry_pattern inq_pat; da_quirks quirks; @@ -1557,6 +1559,7 @@ static sbintime_t da_default_softtimeout = DA_DEFAULT_SOFTTIMEOUT; static int da_send_ordered = DA_DEFAULT_SEND_ORDERED; static int da_disable_wp_detection = 0; static int da_enable_biospeedup = 1; +static int da_enable_uma_ccbs = 0; static SYSCTL_NODE(_kern_cam, OID_AUTO, da, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "CAM Direct Access Disk driver"); @@ -1573,6 +1576,8 @@ SYSCTL_INT(_kern_cam_da, OID_AUTO, disable_wp_detection, CTLFLAG_RWTUN, "Disable detection of write-protected disks"); SYSCTL_INT(_kern_cam_da, OID_AUTO, enable_biospeedup, CTLFLAG_RDTUN, &da_enable_biospeedup, 0, "Enable BIO_SPEEDUP processing"); +SYSCTL_INT(_kern_cam_da, OID_AUTO, enable_uma_ccbs, CTLFLAG_RWTUN, + &da_enable_uma_ccbs, 0, "Use UMA for CCBs"); SYSCTL_PROC(_kern_cam_da, OID_AUTO, default_softtimeout, CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, NULL, 0, @@ -2011,6 +2016,10 @@ dainit(void) NULL, SHUTDOWN_PRI_DEFAULT)) == NULL) printf("dainit: shutdown event registration failed!\n"); } + + da_ccb_zone = uma_zcreate("da_ccb", + sizeof(struct ccb_scsiio), NULL, NULL, NULL, NULL, + UMA_ALIGN_PTR, 0); } /* @@ -2848,6 +2857,15 @@ daregister(struct cam_periph *periph, void *arg) TASK_INIT(&softc->sysctl_task, 0, dasysctlinit, periph); + /* + * Let XPT know we can use UMA-allocated CCBs. + */ + if (da_enable_uma_ccbs) { + KASSERT(da_ccb_zone != NULL, + ("%s: NULL da_ccb_zone", __func__)); + periph->ccb_zone = da_ccb_zone; + } + /* * Take an exclusive section lock on the periph while dastart is called * to finish the probe. The lock will be dropped in dadone at the end diff --git a/sys/cam/scsi/scsi_xpt.c b/sys/cam/scsi/scsi_xpt.c index 67b94488dff0..bdc23e4b51b7 100644 --- a/sys/cam/scsi/scsi_xpt.c +++ b/sys/cam/scsi/scsi_xpt.c @@ -2625,6 +2625,13 @@ static void scsi_action(union ccb *start_ccb) { + if (start_ccb->ccb_h.func_code != XPT_SCSI_IO) { + KASSERT((start_ccb->ccb_h.alloc_flags & CAM_CCB_FROM_UMA) == 0, + ("%s: ccb %p, func_code %#x should not be allocated " + "from UMA zone\n", + __func__, start_ccb, start_ccb->ccb_h.func_code)); + } + switch (start_ccb->ccb_h.func_code) { case XPT_SET_TRAN_SETTINGS: {
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202105151105.14FB5nv7005915>