Date: Mon, 26 May 2014 08:48:53 -0600 From: Ian Lepore <ian@FreeBSD.org> To: freebsd-arch <freebsd-arch@FreeBSD.org> Subject: CFR, CFT: Fine-grained SUBDIR dependencies for parallel builds Message-ID: <1401115733.1152.339.camel@revolution.hippie.lan>
next in thread | raw e-mail | index | archive | help
--=-iwxWkDDku4c4XVLMNASv
Content-Type: text/plain; charset="us-ascii"
Content-Transfer-Encoding: 7bit
Based on a request from Justin Gibbs and suggestions from Jilles and
Warner on how to accomplish it, I've cobbled together a mechanism for
fine grained build order controls on parallel subdir builds. This
augments the coarse .WAIT mechanism, which is still useful if you've got
a situation where "almost everything depends on A and B".
Because the parallel subdir mechanism uses non-obvious mangling of
target names, which should probably remain a private detail of the
implementation, it's not easy to do things like "libfoo: libbar", so
instead the new mechanism lets you set a variable that lists
dependencies:
SUBDIR_DEPEND_libfoo= libgroodah libpouet
Note that while I'm using libraries as an example here, it really has
nothing to do with the generated library files. This is really saying
"build in directory libfoo after building in the libgroodah and libpouet
directories." Is this kind of mechanism acceptable to everyone?
I've updated lib/Makefile with dependency information based on the old
almost-accurate comment block and by combing through lib/* makefiles
looking for LDADD dependencies to other libraries within lib/*.
This passed a universe run for me with -j12, but that takes about 5
hours so I can't run it through a lot of different -j numbers very
easily.
-- Ian
--=-iwxWkDDku4c4XVLMNASv
Content-Disposition: inline; filename="subdir_depend.diff"
Content-Type: text/x-patch; name="subdir_depend.diff"; charset="us-ascii"
Content-Transfer-Encoding: 7bit
Index: share/mk/bsd.subdir.mk
===================================================================
--- share/mk/bsd.subdir.mk (revision 266650)
+++ share/mk/bsd.subdir.mk (working copy)
@@ -80,7 +80,11 @@ __subdir_targets=
__subdir_targets+= .WAIT
.else
__subdir_targets+= ${__target}_subdir_${__dir}
-${__target}_subdir_${__dir}: .MAKE
+__deps=
+.for __dep in ${SUBDIR_DEPEND_${__dir}}
+__deps+= ${__target}_subdir_${__dep}
+.endfor
+${__target}_subdir_${__dir}: .MAKE ${__deps}
@${_+_}set -e; \
if test -d ${.CURDIR}/${__dir}.${MACHINE_ARCH}; then \
${ECHODIR} "===> ${DIRPRFX}${__dir}.${MACHINE_ARCH} (${__target:realinstall=install})"; \
Index: lib/Makefile
===================================================================
--- lib/Makefile (revision 266650)
+++ lib/Makefile (working copy)
@@ -3,82 +3,43 @@
.include <src.opts.mk>
-# To satisfy shared library or ELF linkage when only the libraries being
-# built are visible:
-#
-# csu must be built before all shared libaries for ELF.
-# libc must be built before all other shared libraries.
-# libbsm must be built before libauditd.
-# libcom_err must be built before libpam.
-# libcrypt must be built before libpam.
-# libkvm must be built before libdevstat.
-# libldns must be built before libunbound.
-# msun must be built before libg++ and libstdc++.
-# libmd must be built before libatm, libopie, libradius, and libtacplus.
-# ncurses must be built before libdialog, libedit and libreadline.
-# libnetgraph must be built before libbsnmp/modules/snmp_netgraph.
-# libopie must be built before libpam.
-# libradius must be built before libpam.
-# librpcsvc must be built before libpam.
-# libsbuf must be built before libcam.
-# libtacplus must be built before libpam.
-# libutil must be built before libpam.
-# libypclnt must be built before libpam.
-# libgssapi must be built before librpcsec_gss
-#
-# Otherwise, the SUBDIR list should be in alphabetical order.
-#
-# Except it appears bind needs to be compiled last
+# The SUBDIR_ORDERED list is a small set of libraries which are used by many
+# of the other libraries. These are built first with a .WAIT between them
+# and the main list to avoid needing a SUBDIR_DEPEND line on every library
+# naming just these few items.
SUBDIR_ORDERED= ${_csu} \
.WAIT \
libc \
libc_nonshared \
- .WAIT \
- msun \
- .WAIT \
- libbsm \
- libauditd \
- libutil \
- libpjdlog \
- libnv \
- ${_libcapsicum} \
libcompiler_rt \
- libcrypt \
+ ${_libcplusplus} \
+ ${_libcxxrt} \
libelf \
- ${_libiconv_modules} \
- libkvm \
- ${_libldns} \
- libmd \
- ncurses \
- ${_libnetgraph} \
- libradius \
- librpcsvc \
- libsbuf \
- libtacplus \
- ${_libypclnt} \
- ${_libcxxrt} \
- ${_libcplusplus}
+ msun
-.if ${MK_KERBEROS_SUPPORT} != "no"
-SUBDIR_ORDERED+= libcom_err
-.endif
+# The main list; please keep these sorted alphabetically.
SUBDIR= ${SUBDIR_ORDERED} \
.WAIT \
libalias \
libarchive \
${_libatm} \
+ libauditd \
libbegemot \
libblocksruntime \
${_libbluetooth} \
${_libbsnmp} \
libbsdstat \
+ libbsm \
libbz2 \
libcalendar \
libcam \
+ ${_libcapsicum} \
${_libcasper} \
+ ${_libcom_err} \
libcompat \
+ libcrypt \
libdevinfo \
libdevstat \
libdwarf \
@@ -91,26 +52,36 @@ SUBDIR= ${SUBDIR_ORDERED} \
${_libgpib} \
${_libgssapi} \
${_librpcsec_gss} \
+ ${_libiconv_modules} \
libipsec \
libjail \
libkiconv \
+ libkvm \
+ ${_libldns} \
liblzma \
libmagic \
libmandoc \
libmemstat \
+ libmd \
${_libmilter} \
${_libmp} \
${_libnandfs} \
libnetbsd \
+ ${_libnetgraph} \
${_libngatm} \
+ libnv \
libopie \
libpam \
libpcap \
+ libpjdlog \
${_libpmc} \
${_libproc} \
libprocstat \
+ libradius \
+ librpcsvc \
librt \
${_librtld_db} \
+ libsbuf \
${_libsdp} \
${_libsm} \
${_libsmb} \
@@ -119,6 +90,7 @@ SUBDIR= ${SUBDIR_ORDERED} \
libstand \
libstdbuf \
libstdthreads \
+ libtacplus \
${_libtelnet} \
${_libthr} \
libthread_db \
@@ -129,15 +101,52 @@ SUBDIR= ${SUBDIR_ORDERED} \
${_libunbound} \
${_libusbhid} \
${_libusb} \
+ libutil \
${_libvgl} \
${_libvmmapi} \
libwrap \
liby \
+ ${_libypclnt} \
libz \
+ ncurses \
${_atf} \
${_clang} \
${_tests}
+# Inter-library dependencies. When the makefile for a library contains LDADD
+# libraries, those libraries should be listed as build order dependencies here.
+
+SUBDIR_DEPEND_libarchive= libz libbz2 libexpat liblzma libmb
+SUBDIR_DEPEND_libatm= libmd
+SUBDIR_DEPEND_libauditdm= libbsm
+SUBDIR_DEPEND_libbsnmp= ${_libnetgraph}
+SUBDIR_DEPEND_libc++= libcxxrt
+SUBDIR_DEPEND_libc= libcompiler_rt
+SUBDIR_DEPEND_libcam= libsbuf
+SUBDIR_DEPEND_libcapsicum= libnv
+SUBDIR_DEPEND_libcasper= libcapsicum libnv libpjdlog
+SUBDIR_DEPEND_libdevstat= libkvm
+SUBDIR_DEPEND_libdevstat= libkvm
+SUBDIR_DEPEND_libdiaglog= ncurses
+SUBDIR_DEPEND_libedit= ncurses
+SUBDIR_DEPEND_libg++= msun
+SUBDIR_DEPEND_libgeom= libexpat libsbuf
+SUBDIR_DEPEND_liblibrpcsec_gss= libgssapi
+SUBDIR_DEPEND_libmagic= libz
+SUBDIR_DEPEND_libmemstat= libkvm
+SUBDIR_DEPEND_libopie= libmd
+SUBDIR_DEPEND_libpam= libcrypt libopie libradius librpcsvc libtacplus libutil ${_libypclnt} ${_libcom_err}
+SUBDIR_DEPEND_libpjdlog= libutil
+SUBDIR_DEPEND_libprocstat= libkvm libutil
+SUBDIR_DEPEND_libradius= libmd
+SUBDIR_DEPEND_libreadline= ncurses
+SUBDIR_DEPEND_libsmb= libkiconv
+SUBDIR_DEPEND_libstdc++= msun
+SUBDIR_DEPEND_libtacplus= libmb
+SUBDIR_DEPEND_libtacplus= libmd
+SUBDIR_DEPEND_libulog= libmd
+SUBDIR_DEPEND_libunbound= ${_libldns}
+
.if exists(${.CURDIR}/csu/${MACHINE_ARCH}-elf)
_csu=csu/${MACHINE_ARCH}-elf
.elif exists(${.CURDIR}/csu/${MACHINE_ARCH})
@@ -185,6 +194,10 @@ _librpcsec_gss= librpcsec_gss
_libiconv_modules= libiconv_modules
.endif
+.if ${MK_KERBEROS_SUPPORT} != "no"
+_libcom_err= libcom_err
+.endif
+
.if ${MK_LDNS} != "no"
_libldns= libldns
.endif
--=-iwxWkDDku4c4XVLMNASv--
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?1401115733.1152.339.camel>