Date: Tue, 14 Oct 2025 02:52:51 GMT From: Colin Percival <cperciva@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org Subject: git: 16155cccc4bf - main - packages: Compress in parallel Message-ID: <202510140252.59E2qpmW046357@gitrepo.freebsd.org>
next in thread | raw e-mail | index | archive | help
The branch main has been updated by cperciva: URL: https://cgit.FreeBSD.org/src/commit/?id=16155cccc4bf42a5a76806128e7ef637f9ae61fe commit 16155cccc4bf42a5a76806128e7ef637f9ae61fe Author: Colin Percival <cperciva@FreeBSD.org> AuthorDate: 2025-10-12 21:44:14 +0000 Commit: Colin Percival <cperciva@FreeBSD.org> CommitDate: 2025-10-14 02:50:52 +0000 packages: Compress in parallel The FreeBSD-base pkg repository is currrently dominated by a handful of large packages: Out of a compressed repository size of ~960 MB, about 160 MB is taken up by the FreeBSD-src package, 128 MB (on amd64) is used by the FreeBSD-kernel-generic-dbg package, and 91 MB is used by the FreeBSD-src-sys package. Consequently, running 'make packages -jN' provides less benefit than one might hope, as most of the packages finish building quickly, ultimately leaving the FreeBSD-src package building by itself for a couple minutes while all the other CPUs are idle. Pass -T${PKG_CTHREADS} to the 'pkg create' commands (with a default of -T0) in order to instruct pkg's zstd compression to use multiple threads. Testing on an EC2 r7i.48xlarge instance with -j192, this reduces the time taken by 'make packages' from 6m17s to 1m39s; package creation time itself (excluding the initial process of installing into world and kernel staging directories) dropped from 5m37s to 59s. Reviewed by: ivy MFC after: 3 days Sponsored by: https://www.patreon.com/cperciva Differential Revision: https://reviews.freebsd.org/D53053 --- Makefile.inc1 | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Makefile.inc1 b/Makefile.inc1 index e079a23552f1..a86dead09aa1 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -1964,6 +1964,7 @@ REPODIR?= ${OBJROOT}repo PKG_FORMAT?= tzst PKG_LEVEL?= -1 PKG_CLEVEL?= ${"${PKG_FORMAT:Mtar}" != "":?:-l ${PKG_LEVEL}} +PKG_CTHREADS?= 0 PKG_REPO_SIGNING_KEY?= # empty PKG_OUTPUT_DIR?= ${PKG_VERSION} PKG_ABI_FILE?= ${WSTAGEDIR}/usr/bin/uname @@ -2144,7 +2145,7 @@ create-source-src-package: _pkgbootstrap .PHONY ${SSTAGEDIR}/src.ucl ${PKG_CMD} -o ABI=${PKG_ABI} \ -o OSVERSION="${SRCRELDATE}" \ - create -f ${PKG_FORMAT} ${PKG_CLEVEL} \ + create -f ${PKG_FORMAT} ${PKG_CLEVEL} -T${PKG_CTHREADS} \ -M ${SSTAGEDIR}/src.ucl \ -p ${SSTAGEDIR}/src.plist \ -r ${SRCDIR} \ @@ -2170,7 +2171,7 @@ create-source-src-sys-package: _pkgbootstrap .PHONY ${SSTAGEDIR}/src-sys.ucl ${PKG_CMD} -o ABI=${PKG_ABI} \ -o OSVERSION="${SRCRELDATE}" \ - create -f ${PKG_FORMAT} ${PKG_CLEVEL} \ + create -f ${PKG_FORMAT} ${PKG_CLEVEL} -T${PKG_CTHREADS} \ -M ${SSTAGEDIR}/src-sys.ucl \ -p ${SSTAGEDIR}/src-sys.plist \ -r ${SRCDIR} \ @@ -2210,7 +2211,7 @@ create-world-package-${pkgname}: .PHONY ' ${WSTAGEDIR}/${pkgname}.ucl ${PKG_CMD} -o ABI=${PKG_ABI} -o ALLOW_BASE_SHLIBS=yes \ -o OSVERSION="${SRCRELDATE}" \ - create -f ${PKG_FORMAT} ${PKG_CLEVEL} \ + create -f ${PKG_FORMAT} ${PKG_CLEVEL} -T${PKG_CTHREADS} \ -M ${WSTAGEDIR}/${pkgname}.ucl \ -p ${WSTAGEDIR}/${pkgname}.plist \ -r ${WSTAGEDIR} \ @@ -2229,7 +2230,7 @@ create-sets-packages: .PHONY @for manifest in ${WSTAGEDIR}/set-*.ucl; do \ echo "--> Processing manifest: $$manifest"; \ ${PKG_CMD} -o ABI=${PKG_ABI} -o OSVERSION="${SRCRELDATE}" \ - create -f ${PKG_FORMAT} ${PKG_CLEVEL} \ + create -f ${PKG_FORMAT} ${PKG_CLEVEL} -T${PKG_CTHREADS} \ -M $$manifest \ -o "${REPODIR}/${PKG_ABI}/${PKG_OUTPUT_DIR}" \ || exit 1; \ @@ -2259,7 +2260,7 @@ create-dtb-package: .PHONY ${KSTAGEDIR}/${DISTDIR}/dtb.ucl ; \ ${PKG_CMD} -o ABI=${PKG_ABI} -o ALLOW_BASE_SHLIBS=yes \ -o OSVERSION="${SRCRELDATE}" \ - create -f ${PKG_FORMAT} ${PKG_CLEVEL} \ + create -f ${PKG_FORMAT} ${PKG_CLEVEL} -T${PKG_CTHREADS} \ -M ${KSTAGEDIR}/${DISTDIR}/dtb.ucl \ -p ${KSTAGEDIR}/${DISTDIR}/dtb.plist \ -r ${KSTAGEDIR}/${DISTDIR} \ @@ -2296,7 +2297,7 @@ create-kernel-packages-flavor${flavor:C,^""$,${_default_flavor},}: _pkgbootstrap ${KSTAGEDIR}/${DISTDIR}/kernel.${INSTALLKERNEL}${flavor}.ucl ; \ ${PKG_CMD} -o ABI=${PKG_ABI} -o ALLOW_BASE_SHLIBS=yes \ -o OSVERSION="${SRCRELDATE}" \ - create -f ${PKG_FORMAT} ${PKG_CLEVEL} \ + create -f ${PKG_FORMAT} ${PKG_CLEVEL} -T${PKG_CTHREADS} \ -M ${KSTAGEDIR}/${DISTDIR}/kernel.${INSTALLKERNEL}${flavor}.ucl \ -p ${KSTAGEDIR}/${DISTDIR}/kernel.${INSTALLKERNEL}${flavor}.plist \ -r ${KSTAGEDIR}/${DISTDIR} \ @@ -2339,7 +2340,7 @@ create-kernel-packages-extra-flavor${flavor:C,^""$,${_default_flavor},}-${_kerne ${KSTAGEDIR}/kernel.${_kernel}/kernel.${_kernel}${flavor}.ucl ; \ ${PKG_CMD} -o ABI=${PKG_ABI} -o ALLOW_BASE_SHLIBS=yes \ -o OSVERSION="${SRCRELDATE}" \ - create -f ${PKG_FORMAT} ${PKG_CLEVEL} \ + create -f ${PKG_FORMAT} ${PKG_CLEVEL} -T${PKG_CTHREADS} \ -M ${KSTAGEDIR}/kernel.${_kernel}/kernel.${_kernel}${flavor}.ucl \ -p ${KSTAGEDIR}/kernel.${_kernel}/kernel.${_kernel}${flavor}.plist \ -r ${KSTAGEDIR}/kernel.${_kernel} \
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202510140252.59E2qpmW046357>