From owner-svn-src-all@freebsd.org Wed Aug 19 12:23:05 2020 Return-Path: Delivered-To: svn-src-all@mailman.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.nyi.freebsd.org (Postfix) with ESMTP id E06B83BD132; Wed, 19 Aug 2020 12:23:05 +0000 (UTC) (envelope-from hselasky@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256 client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 4BWn495XtQz3Sxc; Wed, 19 Aug 2020 12:23:05 +0000 (UTC) (envelope-from hselasky@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id A211F228BE; Wed, 19 Aug 2020 12:23:05 +0000 (UTC) (envelope-from hselasky@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id 07JCN5g8021223; Wed, 19 Aug 2020 12:23:05 GMT (envelope-from hselasky@FreeBSD.org) Received: (from hselasky@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id 07JCN56g021222; Wed, 19 Aug 2020 12:23:05 GMT (envelope-from hselasky@FreeBSD.org) Message-Id: <202008191223.07JCN56g021222@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: hselasky set sender to hselasky@FreeBSD.org using -f From: Hans Petter Selasky Date: Wed, 19 Aug 2020 12:23:05 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-12@freebsd.org Subject: svn commit: r364381 - stable/12/sys/kern X-SVN-Group: stable-12 X-SVN-Commit-Author: hselasky X-SVN-Commit-Paths: stable/12/sys/kern X-SVN-Commit-Revision: 364381 X-SVN-Commit-Repository: base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.33 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 19 Aug 2020 12:23:05 -0000 Author: hselasky Date: Wed Aug 19 12:23:05 2020 New Revision: 364381 URL: https://svnweb.freebsd.org/changeset/base/364381 Log: MFC r364019: Add full support support for dynamic allocation and freeing of epoch's. Make sure to reclaim epoch structures when they are freed to support dynamic allocation and freeing of epoch structures. While at it, move the 64 supported epoch control structures to the static memory domain. This overall simplifies the management and debugging of system epoch's. Reviewed by: kib, markj Differential Revision: https://reviews.freebsd.org/D25960 Sponsored by: Mellanox Technologies Modified: stable/12/sys/kern/subr_epoch.c Directory Properties: stable/12/ (props changed) Modified: stable/12/sys/kern/subr_epoch.c ============================================================================== --- stable/12/sys/kern/subr_epoch.c Wed Aug 19 12:16:11 2020 (r364380) +++ stable/12/sys/kern/subr_epoch.c Wed Aug 19 12:23:05 2020 (r364381) @@ -54,8 +54,6 @@ __FBSDID("$FreeBSD$"); #include -static MALLOC_DEFINE(M_EPOCH, "epoch", "epoch based reclamation"); - #ifdef __amd64__ #define EPOCH_ALIGN CACHE_LINE_SIZE*2 #else @@ -76,7 +74,7 @@ typedef struct epoch_record { struct epoch { struct ck_epoch e_epoch __aligned(EPOCH_ALIGN); epoch_record_t e_pcpu_record; - int e_idx; + int e_in_use; int e_flags; /* fields above are part of KBI and cannot be modified */ struct sx e_drain_sx; @@ -123,19 +121,23 @@ TAILQ_HEAD (threadlist, thread); CK_STACK_CONTAINER(struct ck_epoch_entry, stack_entry, ck_epoch_entry_container) -epoch_t allepochs[MAX_EPOCHS]; +static struct epoch epoch_array[MAX_EPOCHS]; DPCPU_DEFINE(struct grouptask, epoch_cb_task); DPCPU_DEFINE(int, epoch_cb_count); static __read_mostly int inited; -static __read_mostly int epoch_count; __read_mostly epoch_t global_epoch; __read_mostly epoch_t global_epoch_preempt; static void epoch_call_task(void *context __unused); static uma_zone_t pcpu_zone_record; +static struct sx epoch_sx; + +#define EPOCH_LOCK() sx_xlock(&epoch_sx) +#define EPOCH_UNLOCK() sx_xunlock(&epoch_sx) + static void epoch_init(void *arg __unused) { @@ -158,6 +160,7 @@ epoch_init(void *arg __unused) DPCPU_ID_PTR(cpu, epoch_cb_task), NULL, cpu, -1, "epoch call task"); } + sx_init(&epoch_sx, "epoch-sx"); inited = 1; global_epoch = epoch_alloc(0); global_epoch_preempt = epoch_alloc(EPOCH_PREEMPT); @@ -203,18 +206,47 @@ epoch_t epoch_alloc(int flags) { epoch_t epoch; + int i; + MPASS(name != NULL); + if (__predict_false(!inited)) panic("%s called too early in boot", __func__); - epoch = malloc(sizeof(struct epoch), M_EPOCH, M_ZERO | M_WAITOK); + + EPOCH_LOCK(); + + /* + * Find a free index in the epoch array. If no free index is + * found, try to use the index after the last one. + */ + for (i = 0;; i++) { + /* + * If too many epochs are currently allocated, + * return NULL. + */ + if (i == MAX_EPOCHS) { + epoch = NULL; + goto done; + } + if (epoch_array[i].e_in_use == 0) + break; + } + + epoch = epoch_array + i; ck_epoch_init(&epoch->e_epoch); epoch_ctor(epoch); - MPASS(epoch_count < MAX_EPOCHS - 2); epoch->e_flags = flags; - epoch->e_idx = epoch_count; sx_init(&epoch->e_drain_sx, "epoch-drain-sx"); mtx_init(&epoch->e_drain_mtx, "epoch-drain-mtx", NULL, MTX_DEF); - allepochs[epoch_count++] = epoch; + + /* + * Set e_in_use last, because when this field is set the + * epoch_call_task() function will start scanning this epoch + * structure. + */ + atomic_store_rel_int(&epoch->e_in_use, 1); +done: + EPOCH_UNLOCK(); return (epoch); } @@ -222,13 +254,24 @@ void epoch_free(epoch_t epoch) { + EPOCH_LOCK(); + + MPASS(epoch->e_in_use != 0); + epoch_drain_callbacks(epoch); - allepochs[epoch->e_idx] = NULL; + + atomic_store_rel_int(&epoch->e_in_use, 0); + /* + * Make sure the epoch_call_task() function see e_in_use equal + * to zero, by calling epoch_wait() on the global_epoch: + */ epoch_wait(global_epoch); uma_zfree_pcpu(pcpu_zone_record, epoch->e_pcpu_record); mtx_destroy(&epoch->e_drain_mtx); sx_destroy(&epoch->e_drain_sx); - free(epoch, M_EPOCH); + memset(epoch, 0, sizeof(*epoch)); + + EPOCH_UNLOCK(); } static epoch_record_t @@ -592,8 +635,10 @@ epoch_call_task(void *arg __unused) ck_stack_init(&cb_stack); critical_enter(); epoch_enter(global_epoch); - for (total = i = 0; i < epoch_count; i++) { - if (__predict_false((epoch = allepochs[i]) == NULL)) + for (total = i = 0; i != MAX_EPOCHS; i++) { + epoch = epoch_array + i; + if (__predict_false( + atomic_load_acq_int(&epoch->e_in_use) == 0)) continue; er = epoch_currecord(epoch); record = &er->er_record;