From owner-svn-src-projects@freebsd.org Fri Jul 17 08:06:41 2015 Return-Path: Delivered-To: svn-src-projects@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id DDE069A4C21 for ; Fri, 17 Jul 2015 08:06:41 +0000 (UTC) (envelope-from np@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id CEA901969; Fri, 17 Jul 2015 08:06:41 +0000 (UTC) (envelope-from np@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.70]) by repo.freebsd.org (8.14.9/8.14.9) with ESMTP id t6H86f3M064094; Fri, 17 Jul 2015 08:06:41 GMT (envelope-from np@FreeBSD.org) Received: (from np@localhost) by repo.freebsd.org (8.14.9/8.14.9/Submit) id t6H86f2u064093; Fri, 17 Jul 2015 08:06:41 GMT (envelope-from np@FreeBSD.org) Message-Id: <201507170806.t6H86f2u064093@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: np set sender to np@FreeBSD.org using -f From: Navdeep Parhar Date: Fri, 17 Jul 2015 08:06:41 +0000 (UTC) To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r285650 - projects/cxl_iscsi/sys/dev/cxgbe/cxgbei X-SVN-Group: projects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-projects@freebsd.org X-Mailman-Version: 2.1.20 Precedence: list List-Id: "SVN commit messages for the src " projects" tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 17 Jul 2015 08:06:42 -0000 Author: np Date: Fri Jul 17 08:06:40 2015 New Revision: 285650 URL: https://svnweb.freebsd.org/changeset/base/285650 Log: Create some worker threads. These will be used instead of the driver's ithreads to dispatch received PDUs. Modified: projects/cxl_iscsi/sys/dev/cxgbe/cxgbei/cxgbei.c Modified: projects/cxl_iscsi/sys/dev/cxgbe/cxgbei/cxgbei.c ============================================================================== --- projects/cxl_iscsi/sys/dev/cxgbe/cxgbei/cxgbei.c Fri Jul 17 07:30:00 2015 (r285649) +++ projects/cxl_iscsi/sys/dev/cxgbe/cxgbei/cxgbei.c Fri Jul 17 08:06:40 2015 (r285650) @@ -40,6 +40,8 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include +#include #include #include #include @@ -1204,6 +1206,114 @@ static struct uld_info cxgbei_uld_info = .deactivate = cxgbei_deactivate, }; +enum { + CWT_RUNNING = 1, + CWT_STOP = 2, + CWT_STOPPED = 3, +}; + +struct cxgbei_worker_thread_softc { + struct mtx cwt_lock; + struct cv cwt_cv; + volatile int cwt_state; +} __aligned(CACHE_LINE_SIZE); + +int worker_thread_count; +static struct cxgbei_worker_thread_softc *cwt_softc; +static struct proc *cxgbei_proc; + +static void +cwt_main(void *arg) +{ + struct cxgbei_worker_thread_softc *cwt = arg; + + MPASS(cwt != NULL); + + mtx_lock(&cwt->cwt_lock); + MPASS(cwt->cwt_state == 0); + cwt->cwt_state = CWT_RUNNING; + cv_signal(&cwt->cwt_cv); + for (;;) { + cv_wait(&cwt->cwt_cv, &cwt->cwt_lock); + if (cwt->cwt_state == CWT_STOP) + break; + } + + mtx_assert(&cwt->cwt_lock, MA_OWNED); + cwt->cwt_state = CWT_STOPPED; + cv_signal(&cwt->cwt_cv); + mtx_unlock(&cwt->cwt_lock); + kthread_exit(); +} + +static int +start_worker_threads(void) +{ + int i, rc; + struct cxgbei_worker_thread_softc *cwt; + + worker_thread_count = min(mp_ncpus, 32); + cwt_softc = malloc(worker_thread_count * sizeof(*cwt), M_CXGBE, + M_WAITOK | M_ZERO); + + MPASS(cxgbei_proc == NULL); + for (i = 0, cwt = &cwt_softc[0]; i < worker_thread_count; i++, cwt++) { + mtx_init(&cwt->cwt_lock, "cwt lock", NULL, MTX_DEF); + cv_init(&cwt->cwt_cv, "cwt cv"); + rc = kproc_kthread_add(cwt_main, cwt, &cxgbei_proc, NULL, 0, 0, + "cxgbei", "%d", i); + if (rc != 0) { + printf("cxgbei: failed to start thread #%d/%d (%d)\n", + i + 1, worker_thread_count, rc); + mtx_destroy(&cwt->cwt_lock); + cv_destroy(&cwt->cwt_cv); + bzero(&cwt, sizeof(*cwt)); + if (i == 0) { + free(cwt_softc, M_CXGBE); + worker_thread_count = 0; + + return (rc); + } + + /* Not fatal, carry on with fewer threads. */ + worker_thread_count = i; + rc = 0; + break; + } + + /* Wait for thread to start before moving on to the next one. */ + mtx_lock(&cwt->cwt_lock); + while (cwt->cwt_state != CWT_RUNNING) + cv_wait(&cwt->cwt_cv, &cwt->cwt_lock); + mtx_unlock(&cwt->cwt_lock); + } + + MPASS(cwt_softc != NULL); + MPASS(worker_thread_count > 0); + return (0); +} + +static void +stop_worker_threads(void) +{ + int i; + struct cxgbei_worker_thread_softc *cwt = &cwt_softc[0]; + + MPASS(worker_thread_count >= 0); + + for (i = 0, cwt = &cwt_softc[0]; i < worker_thread_count; i++, cwt++) { + mtx_lock(&cwt->cwt_lock); + MPASS(cwt->cwt_state == CWT_RUNNING); + cwt->cwt_state = CWT_STOP; + cv_signal(&cwt->cwt_cv); + do { + cv_wait(&cwt->cwt_cv, &cwt->cwt_lock); + } while (cwt->cwt_state != CWT_STOPPED); + mtx_unlock(&cwt->cwt_lock); + } + free(cwt_softc, M_CXGBE); +} + extern void (*cxgbei_fw4_ack)(struct toepcb *, int); extern void (*cxgbei_rx_data_ddp)(struct toepcb *, const struct cpl_rx_data_ddp *); @@ -1220,10 +1330,16 @@ cxgbei_mod_load(void) cxgbei_writeq_len = get_writeq_len; cxgbei_writeq_next = do_writeq_next; - rc = t4_register_uld(&cxgbei_uld_info); + rc = start_worker_threads(); if (rc != 0) return (rc); + rc = t4_register_uld(&cxgbei_uld_info); + if (rc != 0) { + stop_worker_threads(); + return (rc); + } + t4_iterate(cxgbei_activate_all, NULL); return (rc); @@ -1238,6 +1354,8 @@ cxgbei_mod_unload(void) if (t4_unregister_uld(&cxgbei_uld_info) == EBUSY) return (EBUSY); + stop_worker_threads(); + return (0); }