Date: Wed, 28 May 2014 06:27:30 -0600 From: Ian Lepore <ian@FreeBSD.org> To: Jia-Shiun Li <jiashiun@gmail.com> Cc: freebsd-arch <freebsd-arch@FreeBSD.org> Subject: Re: CFR, CFT: Fine-grained SUBDIR dependencies for parallel builds Message-ID: <1401280050.1152.377.camel@revolution.hippie.lan> In-Reply-To: <CAHNYxxMkK5LpNdCOUyi33nh7xpU8qz4Xs8FvqMSyHmJo54TEeg@mail.gmail.com> References: <1401115733.1152.339.camel@revolution.hippie.lan> <CAHNYxxMkK5LpNdCOUyi33nh7xpU8qz4Xs8FvqMSyHmJo54TEeg@mail.gmail.com>
next in thread | previous in thread | raw e-mail | index | archive | help
--=-W4vrQItYiX5TYo1HaM5E
Content-Type: text/plain; charset="us-ascii"
Content-Transfer-Encoding: 7bit
On Wed, 2014-05-28 at 12:58 +0800, Jia-Shiun Li wrote:
> It failed cleandir at libmb with -j4. Test script attached and log
> snippet below.
>
> Tested with:
> - HW: i5-3450 CPU w/ 8GB memory
> - /usr/obj & src mounted on tmpfs.
>
> src uses ~1GB without .svn dir. /usr/obj uses another ~2GB for
> buildworld, not including buildkernel. If memory is not constrained I
> think it is easier to use tmpfs to uncover parallel timing/race issues
> hidden by slower I/O.
>
>
> -Jia-shiun.
Doh! There was a typo, libmb should have been libmd. More importantly,
it shows that my testing didn't test anything at all. I think I applied
the patch in one sandbox and then ran make universe in a different one.
Here's an updated patch (this time I'll try to test it correctly myself
too).
Thanks for testing.
-- Ian
--=-W4vrQItYiX5TYo1HaM5E
Content-Disposition: attachment; filename="subdir_depend.diff"
Content-Type: text/x-patch; name="subdir_depend.diff"; charset="us-ascii"
Content-Transfer-Encoding: 7bit
Index: share/mk/bsd.subdir.mk
===================================================================
--- share/mk/bsd.subdir.mk (revision 266650)
+++ share/mk/bsd.subdir.mk (working copy)
@@ -80,7 +80,11 @@ __subdir_targets=
__subdir_targets+= .WAIT
.else
__subdir_targets+= ${__target}_subdir_${__dir}
-${__target}_subdir_${__dir}: .MAKE
+__deps=
+.for __dep in ${SUBDIR_DEPEND_${__dir}}
+__deps+= ${__target}_subdir_${__dep}
+.endfor
+${__target}_subdir_${__dir}: .MAKE ${__deps}
@${_+_}set -e; \
if test -d ${.CURDIR}/${__dir}.${MACHINE_ARCH}; then \
${ECHODIR} "===> ${DIRPRFX}${__dir}.${MACHINE_ARCH} (${__target:realinstall=install})"; \
Index: lib/Makefile
===================================================================
--- lib/Makefile (revision 266650)
+++ lib/Makefile (working copy)
@@ -3,82 +3,43 @@
.include <src.opts.mk>
-# To satisfy shared library or ELF linkage when only the libraries being
-# built are visible:
-#
-# csu must be built before all shared libaries for ELF.
-# libc must be built before all other shared libraries.
-# libbsm must be built before libauditd.
-# libcom_err must be built before libpam.
-# libcrypt must be built before libpam.
-# libkvm must be built before libdevstat.
-# libldns must be built before libunbound.
-# msun must be built before libg++ and libstdc++.
-# libmd must be built before libatm, libopie, libradius, and libtacplus.
-# ncurses must be built before libdialog, libedit and libreadline.
-# libnetgraph must be built before libbsnmp/modules/snmp_netgraph.
-# libopie must be built before libpam.
-# libradius must be built before libpam.
-# librpcsvc must be built before libpam.
-# libsbuf must be built before libcam.
-# libtacplus must be built before libpam.
-# libutil must be built before libpam.
-# libypclnt must be built before libpam.
-# libgssapi must be built before librpcsec_gss
-#
-# Otherwise, the SUBDIR list should be in alphabetical order.
-#
-# Except it appears bind needs to be compiled last
+# The SUBDIR_ORDERED list is a small set of libraries which are used by many
+# of the other libraries. These are built first with a .WAIT between them
+# and the main list to avoid needing a SUBDIR_DEPEND line on every library
+# naming just these few items.
SUBDIR_ORDERED= ${_csu} \
.WAIT \
libc \
libc_nonshared \
- .WAIT \
- msun \
- .WAIT \
- libbsm \
- libauditd \
- libutil \
- libpjdlog \
- libnv \
- ${_libcapsicum} \
libcompiler_rt \
- libcrypt \
+ ${_libcplusplus} \
+ ${_libcxxrt} \
libelf \
- ${_libiconv_modules} \
- libkvm \
- ${_libldns} \
- libmd \
- ncurses \
- ${_libnetgraph} \
- libradius \
- librpcsvc \
- libsbuf \
- libtacplus \
- ${_libypclnt} \
- ${_libcxxrt} \
- ${_libcplusplus}
+ msun
-.if ${MK_KERBEROS_SUPPORT} != "no"
-SUBDIR_ORDERED+= libcom_err
-.endif
+# The main list; please keep these sorted alphabetically.
SUBDIR= ${SUBDIR_ORDERED} \
.WAIT \
libalias \
libarchive \
${_libatm} \
+ libauditd \
libbegemot \
libblocksruntime \
${_libbluetooth} \
${_libbsnmp} \
libbsdstat \
+ libbsm \
libbz2 \
libcalendar \
libcam \
+ ${_libcapsicum} \
${_libcasper} \
+ ${_libcom_err} \
libcompat \
+ libcrypt \
libdevinfo \
libdevstat \
libdwarf \
@@ -91,26 +52,36 @@ SUBDIR= ${SUBDIR_ORDERED} \
${_libgpib} \
${_libgssapi} \
${_librpcsec_gss} \
+ ${_libiconv_modules} \
libipsec \
libjail \
libkiconv \
+ libkvm \
+ ${_libldns} \
liblzma \
libmagic \
libmandoc \
libmemstat \
+ libmd \
${_libmilter} \
${_libmp} \
${_libnandfs} \
libnetbsd \
+ ${_libnetgraph} \
${_libngatm} \
+ libnv \
libopie \
libpam \
libpcap \
+ libpjdlog \
${_libpmc} \
${_libproc} \
libprocstat \
+ libradius \
+ librpcsvc \
librt \
${_librtld_db} \
+ libsbuf \
${_libsdp} \
${_libsm} \
${_libsmb} \
@@ -119,6 +90,7 @@ SUBDIR= ${SUBDIR_ORDERED} \
libstand \
libstdbuf \
libstdthreads \
+ libtacplus \
${_libtelnet} \
${_libthr} \
libthread_db \
@@ -129,15 +101,50 @@ SUBDIR= ${SUBDIR_ORDERED} \
${_libunbound} \
${_libusbhid} \
${_libusb} \
+ libutil \
${_libvgl} \
${_libvmmapi} \
libwrap \
liby \
+ ${_libypclnt} \
libz \
+ ncurses \
${_atf} \
${_clang} \
${_tests}
+# Inter-library dependencies. When the makefile for a library contains LDADD
+# libraries, those libraries should be listed as build order dependencies here.
+
+SUBDIR_DEPEND_libarchive= libz libbz2 libexpat liblzma libmd
+SUBDIR_DEPEND_libatm= libmd
+SUBDIR_DEPEND_libauditdm= libbsm
+SUBDIR_DEPEND_libbsnmp= ${_libnetgraph}
+SUBDIR_DEPEND_libc++= libcxxrt
+SUBDIR_DEPEND_libc= libcompiler_rt
+SUBDIR_DEPEND_libcam= libsbuf
+SUBDIR_DEPEND_libcapsicum= libnv
+SUBDIR_DEPEND_libcasper= libcapsicum libnv libpjdlog
+SUBDIR_DEPEND_libdevstat= libkvm
+SUBDIR_DEPEND_libdiaglog= ncurses
+SUBDIR_DEPEND_libedit= ncurses
+SUBDIR_DEPEND_libg++= msun
+SUBDIR_DEPEND_libgeom= libexpat libsbuf
+SUBDIR_DEPEND_liblibrpcsec_gss= libgssapi
+SUBDIR_DEPEND_libmagic= libz
+SUBDIR_DEPEND_libmemstat= libkvm
+SUBDIR_DEPEND_libopie= libmd
+SUBDIR_DEPEND_libpam= libcrypt libopie libradius librpcsvc libtacplus libutil ${_libypclnt} ${_libcom_err}
+SUBDIR_DEPEND_libpjdlog= libutil
+SUBDIR_DEPEND_libprocstat= libkvm libutil
+SUBDIR_DEPEND_libradius= libmd
+SUBDIR_DEPEND_libreadline= ncurses
+SUBDIR_DEPEND_libsmb= libkiconv
+SUBDIR_DEPEND_libstdc++= msun
+SUBDIR_DEPEND_libtacplus= libmd
+SUBDIR_DEPEND_libulog= libmd
+SUBDIR_DEPEND_libunbound= ${_libldns}
+
.if exists(${.CURDIR}/csu/${MACHINE_ARCH}-elf)
_csu=csu/${MACHINE_ARCH}-elf
.elif exists(${.CURDIR}/csu/${MACHINE_ARCH})
@@ -185,6 +192,10 @@ _librpcsec_gss= librpcsec_gss
_libiconv_modules= libiconv_modules
.endif
+.if ${MK_KERBEROS_SUPPORT} != "no"
+_libcom_err= libcom_err
+.endif
+
.if ${MK_LDNS} != "no"
_libldns= libldns
.endif
--=-W4vrQItYiX5TYo1HaM5E--
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?1401280050.1152.377.camel>