ontent-type:content-type:
	 content-transfer-encoding:content-transfer-encoding;
	bh=eMdC43xPX+7W44bsAhIgE+dFYhAALK0T+qT26215vRE=;
	b=qCEIYnMMQgehqrBjr0DTNFfF+NvLceICKSTSv0h4kuyuyVokByNv0IWqGtBeYccqL/t6SP
	5VJaA++KvF4Lrdq7fzsOtHmfdc0NFRh+mhQ1yB53Q6gY5qwzQoqqz6GIXKPXOeZ+OYqE9l
	+iLL9fV1m8HVG8whWRBn7rDMp1Fuu5+ZM+GDaDjn+3tCGzgAl5CbrhDrKa9fGrhtP2n9Yc
	9CjwPylSkKXRFa9XknM4qr8UhJmrwa09ta6w88HeXH+804i6GaIrIy+AStqgiQorY5SCUa
	ZsXavwNf45m6BqTAl96txCZZMHqltXLDC+fmjf97YruK44eLjhoHbnOHdKXw3Q==
Date: Thu, 16 Oct 2025 20:03:17 GMT
Message-Id: <202510162003.59GK3HD5016705@gitrepo.freebsd.org>
To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org,
        dev-commits-src-branches@FreeBSD.org
From: Colin Percival <cperciva@FreeBSD.org>
Subject: git: 3d8ec3273dbb - stable/15 - packages: Compress in
  parallel
Sender: owner-dev-commits-src-all@FreeBSD.org
MIME-Version: 1.0
Content-Type: text/plain; charset=utf-8
Content-Transfer-Encoding: 8bit
Auto-Submitted: auto-generated

The branch stable/15 has been updated by cperciva:

URL: https://cgit.FreeBSD.org/src/commit/?id=3d8ec3273dbb9edf9080ded97e071edff59dd708

commit 3d8ec3273dbb9edf9080ded97e071edff59dd708
Author:     Colin Percival <cperciva@FreeBSD.org>
AuthorDate: 2025-10-12 21:44:14 +0000
Commit:     Colin Percival <cperciva@FreeBSD.org>
CommitDate: 2025-10-16 20:02:56 +0000

    packages: Compress in parallel
    
    The FreeBSD-base pkg repository is currrently dominated by a handful of
    large packages: Out of a compressed repository size of ~960 MB, about
    160 MB is taken up by the FreeBSD-src package, 128 MB (on amd64) is
    used by the FreeBSD-kernel-generic-dbg package, and 91 MB is used by
    the FreeBSD-src-sys package.  Consequently, running 'make packages -jN'
    provides less benefit than one might hope, as most of the packages
    finish building quickly, ultimately leaving the FreeBSD-src package
    building by itself for a couple minutes while all the other CPUs are
    idle.
    
    Pass -T${PKG_CTHREADS} to the 'pkg create' commands (with a default
    of -T0) in order to instruct pkg's zstd compression to use multiple
    threads.
    
    Testing on an EC2 r7i.48xlarge instance with -j192, this reduces the
    time taken by 'make packages' from 6m17s to 1m39s; package creation
    time itself (excluding the initial process of installing into world
    and kernel staging directories) dropped from 5m37s to 59s.
    
    Reviewed by:    ivy
    MFC after:      3 days
    Sponsored by:   https://www.patreon.com/cperciva
    Differential Revision:  https://reviews.freebsd.org/D53053
    
    (cherry picked from commit 16155cccc4bf42a5a76806128e7ef637f9ae61fe)
---
 Makefile.inc1 | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/Makefile.inc1 b/Makefile.inc1
index d530ca4e9584..21adf215c0aa 100644
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -1977,6 +1977,7 @@ REPODIR?=	${OBJROOT}repo
 PKG_FORMAT?=	tzst
 PKG_LEVEL?=	-1
 PKG_CLEVEL?=	${"${PKG_FORMAT:Mtar}" != "":?:-l ${PKG_LEVEL}}
+PKG_CTHREADS?=	0
 PKG_REPO_SIGNING_KEY?=	# empty
 PKG_OUTPUT_DIR?=	${PKG_VERSION}
 PKG_ABI_FILE?=	${WSTAGEDIR}/usr/bin/uname
@@ -2157,7 +2158,7 @@ create-source-src-package: _pkgbootstrap .PHONY
 		${SSTAGEDIR}/src.ucl
 	${PKG_CMD} -o ABI=${PKG_ABI} \
 		-o OSVERSION="${SRCRELDATE}" \
-		create -f ${PKG_FORMAT} ${PKG_CLEVEL} \
+		create -f ${PKG_FORMAT} ${PKG_CLEVEL} -T${PKG_CTHREADS} \
 		-M ${SSTAGEDIR}/src.ucl \
 		-p ${SSTAGEDIR}/src.plist \
 		-r ${SRCDIR} \
@@ -2183,7 +2184,7 @@ create-source-src-sys-package: _pkgbootstrap .PHONY
 		${SSTAGEDIR}/src-sys.ucl
 	${PKG_CMD} -o ABI=${PKG_ABI} \
 		-o OSVERSION="${SRCRELDATE}" \
-		create -f ${PKG_FORMAT} ${PKG_CLEVEL} \
+		create -f ${PKG_FORMAT} ${PKG_CLEVEL} -T${PKG_CTHREADS} \
 		-M ${SSTAGEDIR}/src-sys.ucl \
 		-p ${SSTAGEDIR}/src-sys.plist \
 		-r ${SRCDIR} \
@@ -2226,7 +2227,7 @@ create-world-package-${pkgname}: .PHONY
 	fi
 	${PKG_CMD} -o ABI=${PKG_ABI} -o ALLOW_BASE_SHLIBS=yes \
 		-o OSVERSION="${SRCRELDATE}" \
-		create -f ${PKG_FORMAT} ${PKG_CLEVEL} \
+		create -f ${PKG_FORMAT} ${PKG_CLEVEL} -T${PKG_CTHREADS} \
 		-M ${WSTAGEDIR}/${pkgname}.ucl \
 		-p ${WSTAGEDIR}/${pkgname}.plist \
 		-r ${WSTAGEDIR} \
@@ -2245,7 +2246,7 @@ create-sets-packages: .PHONY
 	@for manifest in ${WSTAGEDIR}/set-*.ucl; do \
 		echo "--> Processing manifest: $$manifest"; \
 		${PKG_CMD} -o ABI=${PKG_ABI} -o OSVERSION="${SRCRELDATE}" \
-			create -f ${PKG_FORMAT} ${PKG_CLEVEL} \
+			create -f ${PKG_FORMAT} ${PKG_CLEVEL} -T${PKG_CTHREADS} \
 			-M $$manifest \
 			-o "${REPODIR}/${PKG_ABI}/${PKG_OUTPUT_DIR}" \
 			|| exit 1; \
@@ -2275,7 +2276,7 @@ create-dtb-package:	.PHONY
 		${KSTAGEDIR}/${DISTDIR}/dtb.ucl ; \
 	${PKG_CMD} -o ABI=${PKG_ABI} -o ALLOW_BASE_SHLIBS=yes \
 		-o OSVERSION="${SRCRELDATE}" \
-		create -f ${PKG_FORMAT} ${PKG_CLEVEL} \
+		create -f ${PKG_FORMAT} ${PKG_CLEVEL} -T${PKG_CTHREADS} \
 		-M ${KSTAGEDIR}/${DISTDIR}/dtb.ucl \
 		-p ${KSTAGEDIR}/${DISTDIR}/dtb.plist \
 		-r ${KSTAGEDIR}/${DISTDIR} \
@@ -2312,7 +2313,7 @@ create-kernel-packages-flavor${flavor:C,^""$,${_default_flavor},}: _pkgbootstrap
 		${KSTAGEDIR}/${DISTDIR}/kernel.${INSTALLKERNEL}${flavor}.ucl ; \
 	${PKG_CMD} -o ABI=${PKG_ABI} -o ALLOW_BASE_SHLIBS=yes \
 		-o OSVERSION="${SRCRELDATE}" \
-		create -f ${PKG_FORMAT} ${PKG_CLEVEL} \
+		create -f ${PKG_FORMAT} ${PKG_CLEVEL} -T${PKG_CTHREADS} \
 		-M ${KSTAGEDIR}/${DISTDIR}/kernel.${INSTALLKERNEL}${flavor}.ucl \
 		-p ${KSTAGEDIR}/${DISTDIR}/kernel.${INSTALLKERNEL}${flavor}.plist \
 		-r ${KSTAGEDIR}/${DISTDIR} \
@@ -2352,7 +2353,7 @@ create-kernel-packages-extra-flavor${flavor:C,^""$,${_default_flavor},}-${_kerne
 		${KSTAGEDIR}/kernel.${_kernel}/kernel.${_kernel}${flavor}.ucl ; \
 	${PKG_CMD} -o ABI=${PKG_ABI} -o ALLOW_BASE_SHLIBS=yes \
 		-o OSVERSION="${SRCRELDATE}" \
-		create -f ${PKG_FORMAT} ${PKG_CLEVEL} \
+		create -f ${PKG_FORMAT} ${PKG_CLEVEL} -T${PKG_CTHREADS} \
 		-M ${KSTAGEDIR}/kernel.${_kernel}/kernel.${_kernel}${flavor}.ucl \
 		-p ${KSTAGEDIR}/kernel.${_kernel}/kernel.${_kernel}${flavor}.plist \
 		-r ${KSTAGEDIR}/kernel.${_kernel} \