Date: Fri, 26 Nov 2021 10:34:46 GMT From: Tobias Kortkamp <tobik@FreeBSD.org> To: ports-committers@FreeBSD.org, dev-commits-ports-all@FreeBSD.org, dev-commits-ports-main@FreeBSD.org Subject: git: b6633a0113fd - main - ports-mgmt/parfetch-overlay: Add overlay to fetch distfiles in parallel and with pipelining Message-ID: <202111261034.1AQAYkZ1095148@gitrepo.freebsd.org>
next in thread | raw e-mail | index | archive | help
The branch main has been updated by tobik: URL: https://cgit.FreeBSD.org/ports/commit/?id=b6633a0113fd4d12b1310a02a865dc1e6c5f0169 commit b6633a0113fd4d12b1310a02a865dc1e6c5f0169 Author: Tobias Kortkamp <tobik@FreeBSD.org> AuthorDate: 2021-11-26 10:21:50 +0000 Commit: Tobias Kortkamp <tobik@FreeBSD.org> CommitDate: 2021-11-26 10:33:16 +0000 ports-mgmt/parfetch-overlay: Add overlay to fetch distfiles in parallel and with pipelining Ports that fetch multiple distfiles from the same host can especially benefit from this. This is a glue application between libcurl and the ports framework. It comes with a ports overlay that replaces the default do-fetch and checksum targets. This is an experiment. Use at your own risk. WWW: https://github.com/t6/parfetch Demo: https://asciinema.org/a/ogjs0MiCNZxqwYy2xliHfYEaM Some unscientific but repeatable numbers: tokei (152 distfiles) fetch is down from ~44s to ~3s. wlroots (34 distfiles) fetch is down from ~12s to ~3s. rust makesum (28 distfiles) is down from ~90s to ~50s. --- ports-mgmt/Makefile | 1 + ports-mgmt/parfetch-overlay/Makefile | 29 +++++++++++++++++++ ports-mgmt/parfetch-overlay/distinfo | 3 ++ ports-mgmt/parfetch-overlay/files/pkg-message.in | 37 ++++++++++++++++++++++++ ports-mgmt/parfetch-overlay/pkg-descr | 12 ++++++++ ports-mgmt/parfetch-overlay/pkg-plist | 4 +++ 6 files changed, 86 insertions(+) diff --git a/ports-mgmt/Makefile b/ports-mgmt/Makefile index 3809ff9946d0..fed51cb0637b 100644 --- a/ports-mgmt/Makefile +++ b/ports-mgmt/Makefile @@ -21,6 +21,7 @@ SUBDIR += p5-FreeBSD-Ports SUBDIR += p5-FreeBSD-Ports-INDEXhash SUBDIR += packagekit + SUBDIR += parfetch-overlay SUBDIR += pchecker SUBDIR += pkg SUBDIR += pkg-devel diff --git a/ports-mgmt/parfetch-overlay/Makefile b/ports-mgmt/parfetch-overlay/Makefile new file mode 100644 index 000000000000..a14ac93ca17a --- /dev/null +++ b/ports-mgmt/parfetch-overlay/Makefile @@ -0,0 +1,29 @@ +PORTNAME= parfetch +DISTVERSION= a20211126 +CATEGORIES= ports-mgmt +MASTER_SITES= https://github.com/t6/parfetch/releases/download/${DISTVERSION}/ +PKGNAMESUFFIX= -overlay + +MAINTAINER= tobik@FreeBSD.org +COMMENT= Fetch distfiles in parallel and with pipelining + +LICENSE= BSD2CLAUSE +LICENSE_FILE= ${WRKSRC}/LICENSE + +LIB_DEPENDS= libcurl.so:ftp/curl \ + libevent.so:devel/libevent + +USES= ninja pkgconfig tar:lz +HAS_CONFIGURE= yes +CONFIGURE_ARGS= AR="${AR}" \ + CC="${CC}" \ + CFLAGS="${CFLAGS}" \ + CPPFLAGS="${CPPFLAGS}" \ + DESTDIR="${STAGEDIR}" \ + INSTALL_DATA="${INSTALL_DATA}" \ + INSTALL_PROGRAM="${INSTALL_PROGRAM}" \ + LDFLAGS="${LDFLAGS}" \ + PREFIX="${PREFIX}" +SUB_FILES= pkg-message + +.include <bsd.port.mk> diff --git a/ports-mgmt/parfetch-overlay/distinfo b/ports-mgmt/parfetch-overlay/distinfo new file mode 100644 index 000000000000..bea39c4e4a8a --- /dev/null +++ b/ports-mgmt/parfetch-overlay/distinfo @@ -0,0 +1,3 @@ +TIMESTAMP = 1637920935 +SHA256 (parfetch-a20211126.tar.lz) = 55c3614b57aeb8664c11e6c7c8a0bffd5cace6fb3a131d2c504539e31dca4d90 +SIZE (parfetch-a20211126.tar.lz) = 83026 diff --git a/ports-mgmt/parfetch-overlay/files/pkg-message.in b/ports-mgmt/parfetch-overlay/files/pkg-message.in new file mode 100644 index 000000000000..896670f980e4 --- /dev/null +++ b/ports-mgmt/parfetch-overlay/files/pkg-message.in @@ -0,0 +1,37 @@ +[ +{ type: install + message: <<EOM +Local setup +=========== + +Add this to /etc/make.conf + +OVERLAYS+= %%DATADIR%%/overlay + +It replaces the default do-fetch and checksum targets. + +Poudriere setup +=============== + +This requires a Poudriere version that supports overlays like +poudriere-devel. + +Make the overlay available to Poudriere + +$ poudriere ports -c -p parfetch -m null -M %%DATADIR%%/overlay + +Use it + +$ poudriere bulk -O parfetch ... + +Additional configuration +======================== + +For additional configuration options please refer to +%%DATADIR%%/overlay/Mk/bsd.overlay.mk + +The overlay can be disabled at any time by defining +NO_PARFETCH. +EOM +} +] diff --git a/ports-mgmt/parfetch-overlay/pkg-descr b/ports-mgmt/parfetch-overlay/pkg-descr new file mode 100644 index 000000000000..63d2ee8e9f3d --- /dev/null +++ b/ports-mgmt/parfetch-overlay/pkg-descr @@ -0,0 +1,12 @@ +Fetch distfiles in parallel and with pipelining + +Ports that fetch multiple distfiles from the same host can +especially benefit from this. + +This is a glue application between libcurl and the ports +framework. It comes with a ports overlay that replaces the +default do-fetch and checksum targets. + +This is an experiment. Use at your own risk. + +WWW: https://github.com/t6/parfetch diff --git a/ports-mgmt/parfetch-overlay/pkg-plist b/ports-mgmt/parfetch-overlay/pkg-plist new file mode 100644 index 000000000000..cb6b1ea8958c --- /dev/null +++ b/ports-mgmt/parfetch-overlay/pkg-plist @@ -0,0 +1,4 @@ +bin/parfetch +bin/parfetch-static +%%DATADIR%%/overlay/Mk/bsd.overlay.mk +%%DATADIR%%/overlay/bin/parfetch-static
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202111261034.1AQAYkZ1095148>