From owner-dev-commits-src-branches@freebsd.org Wed Aug 11 15:22:44 2021
Return-Path: <owner-dev-commits-src-branches@freebsd.org>
Delivered-To: dev-commits-src-branches@mailman.nyi.freebsd.org
Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1])
by mailman.nyi.freebsd.org (Postfix) with ESMTP id 215B8656065;
Wed, 11 Aug 2021 15:22:44 +0000 (UTC) (envelope-from git@FreeBSD.org)
Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org
[IPv6:2610:1c1:1:606c::19:3])
(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256
client-signature RSA-PSS (4096 bits) client-digest SHA256)
(Client CN "mxrelay.nyi.freebsd.org", Issuer "R3" (verified OK))
by mx1.freebsd.org (Postfix) with ESMTPS id 4GlD8g5ddZz3LMg;
Wed, 11 Aug 2021 15:22:43 +0000 (UTC) (envelope-from git@FreeBSD.org)
Received: from gitrepo.freebsd.org (gitrepo.freebsd.org
[IPv6:2610:1c1:1:6068::e6a:5])
(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256)
(Client did not present a certificate)
by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id A1DEC11A5F;
Wed, 11 Aug 2021 15:22:43 +0000 (UTC) (envelope-from git@FreeBSD.org)
Received: from gitrepo.freebsd.org ([127.0.1.44])
by gitrepo.freebsd.org (8.16.1/8.16.1) with ESMTP id 17BFMhl3064298;
Wed, 11 Aug 2021 15:22:43 GMT (envelope-from git@gitrepo.freebsd.org)
Received: (from git@localhost)
by gitrepo.freebsd.org (8.16.1/8.16.1/Submit) id 17BFMhuj064297;
Wed, 11 Aug 2021 15:22:43 GMT (envelope-from git)
Date: Wed, 11 Aug 2021 15:22:43 GMT
Message-Id: <202108111522.17BFMhuj064297@gitrepo.freebsd.org>
To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org,
dev-commits-src-branches@FreeBSD.org
From: Mateusz Guzik <mjg@FreeBSD.org>
Subject: git: fff80a6992f1 - stable/13 - pf: add hybrid 32- an 64- bit counters
MIME-Version: 1.0
Content-Type: text/plain; charset=utf-8
Content-Transfer-Encoding: 8bit
X-Git-Committer: mjg
X-Git-Repository: src
X-Git-Refname: refs/heads/stable/13
X-Git-Reftype: branch
X-Git-Commit: fff80a6992f1ba13f2261b066d70a3a97d8837e4
Auto-Submitted: auto-generated
X-BeenThere: dev-commits-src-branches@freebsd.org
X-Mailman-Version: 2.1.34
Precedence: list
List-Id: Commits to the stable branches of the FreeBSD src repository
<dev-commits-src-branches.freebsd.org>
List-Unsubscribe: <https://lists.freebsd.org/mailman/options/dev-commits-src-branches>,
<mailto:dev-commits-src-branches-request@freebsd.org?subject=unsubscribe>
List-Archive: <http://lists.freebsd.org/pipermail/dev-commits-src-branches/>
List-Post: <mailto:dev-commits-src-branches@freebsd.org>
List-Help: <mailto:dev-commits-src-branches-request@freebsd.org?subject=help>
List-Subscribe: <https://lists.freebsd.org/mailman/listinfo/dev-commits-src-branches>,
<mailto:dev-commits-src-branches-request@freebsd.org?subject=subscribe>
X-List-Received-Date: Wed, 11 Aug 2021 15:22:44 -0000
The branch stable/13 has been updated by mjg:
URL: https://cgit.FreeBSD.org/src/commit/?id=fff80a6992f1ba13f2261b066d70a3a97d8837e4
commit fff80a6992f1ba13f2261b066d70a3a97d8837e4
Author: Mateusz Guzik <mjg@FreeBSD.org>
AuthorDate: 2021-07-22 20:47:24 +0000
Commit: Mateusz Guzik <mjg@FreeBSD.org>
CommitDate: 2021-08-11 13:37:53 +0000
pf: add hybrid 32- an 64- bit counters
Numerous counters got migrated from straight uint64_t to the counter(9)
API. Unfortunately the implementation comes with a significiant
performance hit on some platforms and cannot be easily fixed.
Work around the problem by implementing a pf-specific variant.
Reviewed by: kp
Sponsored by: Rubicon Communications, LLC ("Netgate")
(cherry picked from commit defdcdd5648dc1ea789bc54bb45108fcab546a6b)
---
sys/net/pfvar.h | 218 ++++++++++++++++++++++++++++++++++++++++++++++++++++
sys/netpfil/pf/pf.c | 28 ++++++-
2 files changed, 245 insertions(+), 1 deletion(-)
diff --git a/sys/net/pfvar.h b/sys/net/pfvar.h
index 55ecbecf657a..e381279a113f 100644
--- a/sys/net/pfvar.h
+++ b/sys/net/pfvar.h
@@ -43,9 +43,11 @@
#include <sys/nv.h>
#include <sys/refcount.h>
#include <sys/sysctl.h>
+#include <sys/smp.h>
#include <sys/lock.h>
#include <sys/rmlock.h>
#include <sys/tree.h>
+#include <sys/seqc.h>
#include <vm/uma.h>
#include <net/radix.h>
@@ -64,6 +66,222 @@
#ifdef _KERNEL
+#if defined(__arm__)
+#define PF_WANT_32_TO_64_COUNTER
+#endif
+
+/*
+ * A hybrid of 32-bit and 64-bit counters which can be used on platforms where
+ * counter(9) is very expensive.
+ *
+ * As 32-bit counters are expected to overflow, a periodic job sums them up to
+ * a saved 64-bit state. Fetching the value still walks all CPUs to get the most
+ * current snapshot.
+ */
+#ifdef PF_WANT_32_TO_64_COUNTER
+struct pf_counter_u64_pcpu {
+ u_int32_t current;
+ u_int32_t snapshot;
+};
+
+struct pf_counter_u64 {
+ struct pf_counter_u64_pcpu *pfcu64_pcpu;
+ u_int64_t pfcu64_value;
+ seqc_t pfcu64_seqc;
+};
+
+static inline int
+pf_counter_u64_init(struct pf_counter_u64 *pfcu64, int flags)
+{
+
+ pfcu64->pfcu64_value = 0;
+ pfcu64->pfcu64_seqc = 0;
+ pfcu64->pfcu64_pcpu = uma_zalloc_pcpu(pcpu_zone_8, flags | M_ZERO);
+ if (__predict_false(pfcu64->pfcu64_pcpu == NULL))
+ return (ENOMEM);
+ return (0);
+}
+
+static inline void
+pf_counter_u64_deinit(struct pf_counter_u64 *pfcu64)
+{
+
+ uma_zfree_pcpu(pcpu_zone_8, pfcu64->pfcu64_pcpu);
+}
+
+static inline void
+pf_counter_u64_critical_enter(void)
+{
+
+ critical_enter();
+}
+
+static inline void
+pf_counter_u64_critical_exit(void)
+{
+
+ critical_exit();
+}
+
+static inline void
+pf_counter_u64_add_protected(struct pf_counter_u64 *pfcu64, uint32_t n)
+{
+ struct pf_counter_u64_pcpu *pcpu;
+ u_int32_t val;
+
+ MPASS(curthread->td_critnest > 0);
+ pcpu = zpcpu_get(pfcu64->pfcu64_pcpu);
+ val = atomic_load_int(&pcpu->current);
+ atomic_store_int(&pcpu->current, val + n);
+}
+
+static inline void
+pf_counter_u64_add(struct pf_counter_u64 *pfcu64, uint32_t n)
+{
+
+ critical_enter();
+ pf_counter_u64_add_protected(pfcu64, n);
+ critical_exit();
+}
+
+static inline u_int64_t
+pf_counter_u64_periodic(struct pf_counter_u64 *pfcu64)
+{
+ struct pf_counter_u64_pcpu *pcpu;
+ u_int64_t sum;
+ u_int32_t val;
+ int cpu;
+
+ MPASS(curthread->td_critnest > 0);
+ seqc_write_begin(&pfcu64->pfcu64_seqc);
+ sum = pfcu64->pfcu64_value;
+ CPU_FOREACH(cpu) {
+ pcpu = zpcpu_get_cpu(pfcu64->pfcu64_pcpu, cpu);
+ val = atomic_load_int(&pcpu->current);
+ sum += (uint32_t)(val - pcpu->snapshot);
+ pcpu->snapshot = val;
+ }
+ pfcu64->pfcu64_value = sum;
+ seqc_write_end(&pfcu64->pfcu64_seqc);
+ return (sum);
+}
+
+static inline u_int64_t
+pf_counter_u64_fetch(struct pf_counter_u64 *pfcu64)
+{
+ struct pf_counter_u64_pcpu *pcpu;
+ u_int64_t sum;
+ seqc_t seqc;
+ int cpu;
+
+ for (;;) {
+ seqc = seqc_read(&pfcu64->pfcu64_seqc);
+ sum = 0;
+ CPU_FOREACH(cpu) {
+ pcpu = zpcpu_get_cpu(pfcu64->pfcu64_pcpu, cpu);
+ sum += (uint32_t)(atomic_load_int(&pcpu->current) -pcpu->snapshot);
+ }
+ sum += pfcu64->pfcu64_value;
+ if (seqc_consistent(&pfcu64->pfcu64_seqc, seqc))
+ break;
+ }
+ return (sum);
+}
+
+static inline void
+pf_counter_u64_zero_protected(struct pf_counter_u64 *pfcu64)
+{
+ struct pf_counter_u64_pcpu *pcpu;
+ int cpu;
+
+ MPASS(curthread->td_critnest > 0);
+ seqc_write_begin(&pfcu64->pfcu64_seqc);
+ CPU_FOREACH(cpu) {
+ pcpu = zpcpu_get_cpu(pfcu64->pfcu64_pcpu, cpu);
+ pcpu->snapshot = atomic_load_int(&pcpu->current);
+ }
+ pfcu64->pfcu64_value = 0;
+ seqc_write_end(&pfcu64->pfcu64_seqc);
+}
+
+static inline void
+pf_counter_u64_zero(struct pf_counter_u64 *pfcu64)
+{
+
+ critical_enter();
+ pf_counter_u64_zero_protected(pfcu64);
+ critical_exit();
+}
+#else
+struct pf_counter_u64 {
+ counter_u64_t counter;
+};
+
+static inline int
+pf_counter_u64_init(struct pf_counter_u64 *pfcu64, int flags)
+{
+
+ pfcu64->counter = counter_u64_alloc(flags);
+ if (__predict_false(pfcu64->counter == NULL))
+ return (ENOMEM);
+ return (0);
+}
+
+static inline void
+pf_counter_u64_deinit(struct pf_counter_u64 *pfcu64)
+{
+
+ counter_u64_free(pfcu64->counter);
+}
+
+static inline void
+pf_counter_u64_critical_enter(void)
+{
+
+}
+
+static inline void
+pf_counter_u64_critical_exit(void)
+{
+
+}
+
+static inline void
+pf_counter_u64_add_protected(struct pf_counter_u64 *pfcu64, uint32_t n)
+{
+
+ counter_u64_add(pfcu64->counter, n);
+}
+
+static inline void
+pf_counter_u64_add(struct pf_counter_u64 *pfcu64, uint32_t n)
+{
+
+ pf_counter_u64_add_protected(pfcu64, n);
+}
+
+static inline u_int64_t
+pf_counter_u64_fetch(struct pf_counter_u64 *pfcu64)
+{
+
+ return (counter_u64_fetch(pfcu64->counter));
+}
+
+static inline void
+pf_counter_u64_zero_protected(struct pf_counter_u64 *pfcu64)
+{
+
+ counter_u64_zero(pfcu64->counter);
+}
+
+static inline void
+pf_counter_u64_zero(struct pf_counter_u64 *pfcu64)
+{
+
+ pf_counter_u64_zero_protected(pfcu64);
+}
+#endif
+
SYSCTL_DECL(_net_pf);
MALLOC_DECLARE(M_PFHASH);
diff --git a/sys/netpfil/pf/pf.c b/sys/netpfil/pf/pf.c
index 0db9eaed1ac4..b74491bb88d0 100644
--- a/sys/netpfil/pf/pf.c
+++ b/sys/netpfil/pf/pf.c
@@ -154,6 +154,11 @@ VNET_DECLARE(int, pf_vnet_active);
VNET_DEFINE_STATIC(uint32_t, pf_purge_idx);
#define V_pf_purge_idx VNET(pf_purge_idx)
+#ifdef PF_WANT_32_TO_64_COUNTER
+VNET_DEFINE_STATIC(uint32_t, pf_counter_periodic_iter);
+#define V_pf_counter_periodic_iter VNET(pf_counter_periodic_iter)
+#endif
+
/*
* Queue for pf_intr() sends.
*/
@@ -1509,6 +1514,25 @@ pf_intr(void *v)
CURVNET_RESTORE();
}
+#define pf_purge_thread_period (hz / 10)
+
+#ifdef PF_WANT_32_TO_64_COUNTER
+static void
+pf_counter_u64_periodic_main(void)
+{
+ PF_RULES_RLOCK_TRACKER;
+
+ V_pf_counter_periodic_iter++;
+
+ PF_RULES_RLOCK();
+ pf_counter_u64_critical_enter();
+ pf_counter_u64_critical_exit();
+ PF_RULES_RUNLOCK();
+}
+#else
+#define pf_counter_u64_periodic_main() do { } while (0)
+#endif
+
void
pf_purge_thread(void *unused __unused)
{
@@ -1516,7 +1540,7 @@ pf_purge_thread(void *unused __unused)
sx_xlock(&pf_end_lock);
while (pf_end_threads == 0) {
- sx_sleep(pf_purge_thread, &pf_end_lock, 0, "pftm", hz / 10);
+ sx_sleep(pf_purge_thread, &pf_end_lock, 0, "pftm", pf_purge_thread_period);
VNET_LIST_RLOCK();
VNET_FOREACH(vnet_iter) {
@@ -1528,6 +1552,8 @@ pf_purge_thread(void *unused __unused)
continue;
}
+ pf_counter_u64_periodic_main();
+
/*
* Process 1/interval fraction of the state
* table every run.