Date: Sat, 29 Apr 2006 14:56:59 -0400 From: "Sean Bryant" <bryants@gmail.com> To: freebsd-stable@freebsd.org Subject: [patch[ twa, bus_dma.h, busdma_machdep.c (amd64) Message-ID: <f6791cc60604291156m39f27a3em615cd5e64a414043@mail.gmail.com>
next in thread | raw e-mail | index | archive | help
[-- Attachment #1 --]
I loaded up 6.1-Stable on a Dual Dual Core Opteron, with a 3ware
9550sx-4lp raid card in it and noticed a horrid boot time.
Well after investigating the issue it turns out the twa driver had
problems creating the bus_dmamaps. It was creating 255 request buffers
and giving each of them a dma map.
Yippie! This leaves me with about a 7 - 10 minute boot time.
To correct this problem I created a bus_dmamap_create_v (should probably
be called bus_dmamap_create_many as mux suggested) and this creates a
large amount of dma maps all at once.
Obviously to test I modified the twa driver to use the new function.
I've seen no problems so far. let me know if you see any glaring errors.
As there are probably some. Disk performance remained the same just
speed up boot time considerably. This is only tested with amd64 code.
The code *should* work with i386, but I wasn't able to test it. Please
test and let me know of any problems.
And I apologize ahead of time if this is useless I don't know much about
the topic but was in need of a quick fix.
--
Sean Bryant
[-- Attachment #2 --]
--- /sys/amd64/amd64/busdma_machdep.c Tue Mar 28 01:28:37 2006
+++ /sys/amd64/amd64/busdma_machdep.c Thu Apr 27 11:55:55 2006
@@ -355,6 +355,107 @@
}
/*
+ * Allocate a vector of handles for mapping from kva/uva/physical
+ * address space into bus device space
+ */
+int
+bus_dmamap_create_v(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp, size_t num)
+{
+ int error;
+
+ error = 0;
+
+ int i;
+
+ i = 0;
+
+ if (dmat->segments == NULL) {
+ dmat->segments = (bus_dma_segment_t *)malloc(
+ sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF,
+ M_NOWAIT);
+ if (dmat->segments == NULL) {
+ CTR3(KTR_BUSDMA, "%s: tag %p error %d",
+ __func__, dmat, ENOMEM);
+ return (ENOMEM);
+ }
+ }
+
+ /*
+ * Bouncing might be required if the driver asks for an active
+ * exclusion region, a data alignment that is stricter than 1, and/or
+ * an active address boundary.
+ */
+ if (dmat->flags & BUS_DMA_COULD_BOUNCE) {
+
+ /* Must bounce */
+ struct bounce_zone *bz;
+ int maxpages;
+
+ if (dmat->bounce_zone == NULL) {
+ if ((error = alloc_bounce_zone(dmat)) != 0)
+ return (error);
+ }
+ bz = dmat->bounce_zone;
+
+ *mapp = (bus_dmamap_t)malloc(sizeof(**mapp) * num, M_DEVBUF,
+ M_NOWAIT | M_ZERO);
+
+ if (*mapp == NULL) {
+ CTR3(KTR_BUSDMA, "%s: tag %p error %d",
+ __func__, dmat, ENOMEM);
+ return (ENOMEM);
+ }
+
+
+
+ /* Initialize the new map */
+
+ bus_dmamap_t *tempp = mapp;
+
+ for(; i < num; i++) {
+ STAILQ_INIT(&((*mapp)->bpages));
+ (*mapp)++;
+ }
+
+ mapp = tempp;
+
+ /*
+ * Attempt to add pages to our pool on a per-instance
+ * basis up to a sane limit.
+ */
+ if (dmat->alignment > 1)
+ maxpages = MAX_BPAGES;
+ else
+ maxpages = MIN(MAX_BPAGES, Maxmem -atop(dmat->lowaddr));
+ if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0
+ || (dmat->map_count > 0 && bz->total_bpages < maxpages)) {
+ int pages;
+
+ pages = MAX(atop(dmat->maxsize), 1);
+ pages = MIN(maxpages - bz->total_bpages, pages);
+ pages = MAX(pages, 1);
+ if (alloc_bounce_pages(dmat, pages) < pages)
+ error = ENOMEM;
+
+ if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) {
+ if (error == 0)
+ dmat->flags |= BUS_DMA_MIN_ALLOC_COMP;
+ } else {
+ error = 0;
+ }
+ }
+ } else {
+ *mapp = NULL;
+ }
+ if (error == 0)
+ dmat->map_count += num;
+ CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
+ __func__, dmat, dmat->flags, error);
+ return (error);
+}
+
+
+/*
* Allocate a handle for mapping from kva/uva/physical
* address space into bus device space.
*/
[-- Attachment #3 --]
--- /sys/sys/bus_dma.h Tue Mar 15 09:57:29 2005
+++ /sys/sys/bus_dma.h Tue Apr 25 21:49:13 2006
@@ -225,6 +225,12 @@
int bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp);
/*
+ * Allocate a vector of handles
+ *
+ */
+int bus_dmamap_create_v(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp, size_t num);
+
+/*
* Destroy a handle for mapping from kva/uva/physical
* address space into bus device space.
*/
[-- Attachment #4 --]
--- /sys/dev/twa/tw_osl_freebsd.c Wed Dec 7 13:18:05 2005
+++ /sys/dev/twa/tw_osl_freebsd.c Thu Apr 27 12:50:12 2006
@@ -89,6 +89,7 @@
TW_INT32 unit = minor(dev);
struct twa_softc *sc = devclass_get_softc(twa_devclass, unit);
+
tw_osli_dbg_dprintf(5, sc, "entered");
sc->state |= TW_OSLI_CTLR_STATE_OPEN;
return(0);
@@ -144,7 +145,6 @@
{
struct twa_softc *sc = (struct twa_softc *)(dev->si_drv1);
TW_INT32 error;
-
tw_osli_dbg_dprintf(5, sc, "entered");
switch (cmd) {
@@ -224,6 +224,7 @@
{
static TW_UINT8 first_ctlr = 1;
+
tw_osli_dbg_printf(3, "entered");
if (tw_cl_ctlr_supported(pci_get_vendor(dev), pci_get_device(dev))) {
@@ -264,6 +265,7 @@
TW_INT32 bar_size;
TW_INT32 error;
+
tw_osli_dbg_dprintf(3, sc, "entered");
sc->ctlr_handle.osl_ctlr_ctxt = sc;
@@ -279,6 +281,8 @@
mtx_init(sc->q_lock, "tw_osl_q_lock", NULL, MTX_SPIN);
sysctl_ctx_init(&sc->sysctl_ctxt);
+
+
sc->sysctl_tree = SYSCTL_ADD_NODE(&sc->sysctl_ctxt,
SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
device_get_nameunit(dev), CTLFLAG_RD, 0, "");
@@ -308,6 +312,7 @@
return(ENXIO);
}
+
/* Force the busmaster enable bit on, in case the BIOS forgot. */
command |= PCIM_CMD_BUSMASTEREN;
pci_write_config(dev, PCIR_COMMAND, command, 2);
@@ -340,6 +345,7 @@
sc->bus_tag = rman_get_bustag(sc->reg_res);
sc->bus_handle = rman_get_bushandle(sc->reg_res);
+
/* Allocate and register our interrupt. */
sc->irq_res_id = 0;
if ((sc->irq_res = bus_alloc_resource(sc->bus_dev, SYS_RES_IRQ,
@@ -386,6 +392,7 @@
return(error);
}
+
/* Initialize the Common Layer for this controller. */
if ((error = tw_cl_init_ctlr(&sc->ctlr_handle, sc->flags, sc->device_id,
TW_OSLI_MAX_NUM_IOS, TW_OSLI_MAX_NUM_AENS,
@@ -467,7 +474,8 @@
TW_UINT32 per_req_dma_mem_size;
#endif /* TW_OSL_DMA_MEM_ALLOC_PER_REQUEST */
TW_INT32 error;
- TW_INT32 i;
+ TW_INT32 i = 0;
+
tw_osli_dbg_dprintf(3, sc, "entered");
@@ -700,10 +708,11 @@
tw_osli_req_q_init(sc, TW_OSLI_FREE_Q);
tw_osli_req_q_init(sc, TW_OSLI_BUSY_Q);
+
if ((sc->req_ctxt_buf = (struct tw_osli_req_context *)
malloc((sizeof(struct tw_osli_req_context) *
TW_OSLI_MAX_NUM_IOS),
- TW_OSLI_MALLOC_CLASS, M_WAITOK)) == NULL) {
+ TW_OSLI_MALLOC_CLASS, M_WAITOK | M_ZERO)) == NULL) {
tw_osli_printf(sc, "error = %d",
TW_CL_SEVERITY_ERROR_STRING,
TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
@@ -712,21 +721,39 @@
ENOMEM);
return(ENOMEM);
}
- bzero(sc->req_ctxt_buf,
- sizeof(struct tw_osli_req_context) * TW_OSLI_MAX_NUM_IOS);
+
+ bus_dmamap_t* dma_v = (bus_dmamap_t*)malloc(sizeof(bus_dmamap_t) * TW_OSLI_MAX_NUM_IOS, M_DEVBUF, M_WAITOK | M_ZERO);
+ if(dma_v == NULL) {
+ return(ENOMEM);
+ }
+
+ if(bus_dmamap_create_v(sc->dma_tag, 0, dma_v, TW_OSLI_MAX_NUM_IOS)) {
+ tw_osli_printf(sc, "request # = %d, error = %d",
+ TW_CL_SEVERITY_ERROR_STRING,
+ TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
+ 0x2013,
+ "Can't create dma map",
+ i, ENOMEM);
+
+ return(ENOMEM);
+ }
+
+ // XXX should be i < ...
for (i = 0; i < TW_OSLI_MAX_NUM_IOS; i++) {
req = &(sc->req_ctxt_buf[i]);
req->ctlr = sc;
- if (bus_dmamap_create(sc->dma_tag, 0, &req->dma_map)) {
+ req->dma_map = dma_v[i];
+ /* if (bus_dmamap_create(sc->dma_tag, 0, &req->dma_map)) {
tw_osli_printf(sc, "request # = %d, error = %d",
TW_CL_SEVERITY_ERROR_STRING,
TW_CL_MESSAGE_SOURCE_FREEBSD_DRIVER,
0x2013,
"Can't create dma map",
i, ENOMEM);
+
return(ENOMEM);
- }
+ }*/
#ifdef TW_OSL_DMA_MEM_ALLOC_PER_REQUEST
@@ -734,11 +761,12 @@
(i * per_req_dma_mem_size);
req->req_pkt.dma_mem_phys = sc->dma_mem_phys +
(i * per_req_dma_mem_size);
-
+
#endif /* TW_OSL_DMA_MEM_ALLOC_PER_REQUEST */
-
+
/* Insert request into the free queue. */
tw_osli_req_q_insert_tail(req, TW_OSLI_FREE_Q);
+
}
#ifdef TW_OSL_DMA_MEM_ALLOC_PER_REQUEST
@@ -962,10 +990,12 @@
lock = (struct mtx *)lock_arg;
switch (op) {
case BUS_DMA_LOCK:
+ printf("twa: Locking!\n");
mtx_lock_spin(lock);
break;
case BUS_DMA_UNLOCK:
+ printf("twa: Unlocking!\n");
mtx_unlock_spin(lock);
break;
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?f6791cc60604291156m39f27a3em615cd5e64a414043>