Date: Sun, 21 Jul 2013 16:59:44 +0000 (UTC) From: Nathan Whitehorn <nwhitehorn@FreeBSD.org> To: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: svn commit: r253521 - user/nwhitehorn/condorports Message-ID: <201307211659.r6LGxiIe094026@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: nwhitehorn Date: Sun Jul 21 16:59:44 2013 New Revision: 253521 URL: http://svnweb.freebsd.org/changeset/base/253521 Log: Try to cut down the ports tree to only necessary bits and then transfer subsets of it to the build nodes. This removes entirely the need for a shared filesystem but may be too fragile (see below). Several TODOs: - Handle ports with MASTERDIR and other kinds of includes, maybe with .MAKELIST.LIST and tar --include $PORTSDIR/*? - Figure out why nosuid is breaking unionfs - Set up lo0 in jail Added: user/nwhitehorn/condorports/postbuild.sh - copied, changed from r253500, user/nwhitehorn/condorports/fetchandlinks.sh user/nwhitehorn/condorports/stagebuildfiles.sh - copied, changed from r253500, user/nwhitehorn/condorports/fetchandlinks.sh Deleted: user/nwhitehorn/condorports/fetchandlinks.sh Modified: user/nwhitehorn/condorports/buildport.sh user/nwhitehorn/condorports/buildportsdag.sh user/nwhitehorn/condorports/package.sub user/nwhitehorn/condorports/prepbuildjail user/nwhitehorn/condorports/reapbuildjail Modified: user/nwhitehorn/condorports/buildport.sh ============================================================================== --- user/nwhitehorn/condorports/buildport.sh Sun Jul 21 15:26:25 2013 (r253520) +++ user/nwhitehorn/condorports/buildport.sh Sun Jul 21 16:59:44 2013 (r253521) @@ -5,6 +5,7 @@ export PATH=/sbin:/bin:/usr/sbin:/usr/bi port=$1 pkg=$2 release=$3 +tarballname=$(basename $4) scratchdir=$(pwd) @@ -13,8 +14,10 @@ scratchdir=$(pwd) set -e sudo /pkgscripts/prepbuildjail $_CONDOR_SLOT $release -sudo jexec $_CONDOR_SLOT make -C $port install USE_PACKAGE_DEPENDS=true BATCH=true WRKDIRPREFIX=/obj DISABLE_MAKE_JOBS=true -sudo jexec $_CONDOR_SLOT make -C $port package PKGREPOSITORY=/packages BATCH=true WRKDIRPREFIX=/obj NO_LATEST_LINK=true PKGFILE=/packages/$pkg +cat $tarballname | sudo jexec $_CONDOR_SLOT tar xvzf - +sudo jexec $_CONDOR_SLOT make -C $port install USE_PACKAGE_DEPENDS=true BATCH=true DISABLE_MAKE_JOBS=true +sudo jexec $_CONDOR_SLOT make -C $port package PKGREPOSITORY=/packages BATCH=true NO_LATEST_LINK=true +ls -lh /scratch/$_CONDOR_SLOT/packages cp /scratch/$_CONDOR_SLOT/packages/$pkg $scratchdir/$pkg sudo /pkgscripts/reapbuildjail $_CONDOR_SLOT set +e Modified: user/nwhitehorn/condorports/buildportsdag.sh ============================================================================== --- user/nwhitehorn/condorports/buildportsdag.sh Sun Jul 21 15:26:25 2013 (r253520) +++ user/nwhitehorn/condorports/buildportsdag.sh Sun Jul 21 16:59:44 2013 (r253521) @@ -2,9 +2,15 @@ : ${PORTSDIR=/usr/ports} : ${PKGSDIR=/usr/ports/packages} : ${ARCH=`uname -p`} +: ${RELEASE=$ARCH/`uname -r`} +: ${STAGEDIR=$(pwd)/stage-$ARCH/$PKGSDIR} + #(cd $PORTSDIR && make index) ports="$(cut -f 2 -d '|' $PORTSDIR/INDEX-`uname -r | cut -f 1 -d .`)" ports="/usr/ports/games/sl /usr/ports/ports-mgmt/pkg /usr/ports/japanese/nkf" + +mkdir -p $STAGEDIR + dagjobs=$(pwd)/ports.dagjobs dagdeps=$(pwd)/ports.dagdeps dag=$(pwd)/ports.dag @@ -13,13 +19,18 @@ for job in $ports; do cd $job echo JOB $job package.sub >> $dagjobs # One fetch for all archs with dummy fetch job? - echo SCRIPT PRE $job fetchandlinks.sh $job $PKGSDIR $ARCH >> $dagjobs export UNAME_p=$ARCH + PKGNAME=$(make package-name) + TARBALL=$STAGEDIR/$(make -V UNIQUENAME).tgz + echo SCRIPT PRE $job stagebuildfiles.sh $job $PKGSDIR $ARCH $TARBALL >> $dagjobs + echo SCRIPT POST $job postbuild.sh $TARBALL \$RETURN >> $dagjobs echo VARS $job BuildArch=\"$ARCH\" >> $dagjobs echo VARS $job port=\"$job\" >> $dagjobs - echo VARS $job pkg=\"`make package-name`.txz\" >> $dagjobs + echo VARS $job pkg=\"$PKGNAME.txz\" >> $dagjobs echo VARS $job pkgdir=\"$PKGSDIR/All\" >> $dagjobs + echo VARS $job stagetarball=\"$TARBALL\" >> $dagjobs + echo VARS $job release=\"$RELEASE\" >> $dagjobs deps=$(make all-depends-list) if [ ! -z "$deps" ]; then echo PARENT $deps CHILD $job >> $dagdeps Modified: user/nwhitehorn/condorports/package.sub ============================================================================== --- user/nwhitehorn/condorports/package.sub Sun Jul 21 15:26:25 2013 (r253520) +++ user/nwhitehorn/condorports/package.sub Sun Jul 21 16:59:44 2013 (r253521) @@ -13,10 +13,7 @@ error=logs/$(pkg).err should_transfer_files=YES when_to_transfer_output=ON_EXIT transfer_output_remaps="$(pkg)=$(pkgdir)/$(pkg)" +transfer_input_files=$(stagetarball) -# TODO: transfer input files? package dependencies and distfiles allow flocking -# A restricted segment of the ports tree (just this, depends, Mk, and distfiles)# would also totally eliminate the need for a shared file system of any kind -# File system tarball could be built by fetchandlinks.sh - -arguments= $(port) $(pkg) release +arguments= $(port) $(pkg) $(release) $(stagetarball) queue Copied and modified: user/nwhitehorn/condorports/postbuild.sh (from r253500, user/nwhitehorn/condorports/fetchandlinks.sh) ============================================================================== --- user/nwhitehorn/condorports/fetchandlinks.sh Sat Jul 20 14:54:43 2013 (r253500, copy source) +++ user/nwhitehorn/condorports/postbuild.sh Sun Jul 21 16:59:44 2013 (r253521) @@ -1,13 +1,7 @@ #!/bin/sh -# Build tarball of interesting subset of ports tree, relevant distfiles, -# dependent packages? +rm $1 || exit $? -set -e -export UNAME_p=$3 -cd $1 -make fetch -make package-links PACKAGES=$2 -mkdir -p $2/All -set +e +# Condor puts the job's error code in argument 2 +exit $2 Modified: user/nwhitehorn/condorports/prepbuildjail ============================================================================== --- user/nwhitehorn/condorports/prepbuildjail Sun Jul 21 15:26:25 2013 (r253520) +++ user/nwhitehorn/condorports/prepbuildjail Sun Jul 21 16:59:44 2013 (r253521) @@ -31,12 +31,10 @@ trap cleanup EXIT #TODO: localhost networking mkdir -p /scratch/$slot +# TODO: Setting nosuid seems to break "below" #mount_unionfs -o below /releases/$release /scratch/$slot -#mount_nullfs -o ro /ports /scratch/$slot/ports mount_unionfs -o below /scratch/$release /scratch/$slot -mount_nullfs -o ro /usr/ports /scratch/$slot/usr/ports -ls /scratch/$slot/usr/ports > /dev/null # Hack around bug in nullfs -devfs_mount_jail /scratch/$slot/dev 4 && mounts="/scratch/$slot/dev $mounts" +devfs_mount_jail /scratch/$slot/dev 4 jail -c name=$slot path=/scratch/$slot host.hostname=$slot.`hostname` persist=true trap true EXIT Modified: user/nwhitehorn/condorports/reapbuildjail ============================================================================== --- user/nwhitehorn/condorports/reapbuildjail Sun Jul 21 15:26:25 2013 (r253520) +++ user/nwhitehorn/condorports/reapbuildjail Sun Jul 21 16:59:44 2013 (r253521) @@ -18,8 +18,6 @@ fi set +e jail -r $slot umount -f /scratch/$slot/dev -#umount -f /scratch/$slot/ports -umount -f /scratch/$slot/usr/ports umount -f /scratch/$slot chflags -R noschg /scratch/$slot rm -rf /scratch/$slot Copied and modified: user/nwhitehorn/condorports/stagebuildfiles.sh (from r253500, user/nwhitehorn/condorports/fetchandlinks.sh) ============================================================================== --- user/nwhitehorn/condorports/fetchandlinks.sh Sat Jul 20 14:54:43 2013 (r253500, copy source) +++ user/nwhitehorn/condorports/stagebuildfiles.sh Sun Jul 21 16:59:44 2013 (r253521) @@ -5,9 +5,33 @@ set -e export UNAME_p=$3 + +# Fetch distfiles if unfetched cd $1 make fetch make package-links PACKAGES=$2 mkdir -p $2/All + + +# Make a tarball containing all the bits we need to build: the distfiles, +# relevant sections of the ports tree, all dependent packages +# Need POST script to clean up + +TARBALL=$4 + +# Find distfiles +DIST_SUBDIR=$(make -V DISTDIR)/$(make -V DIST_SUBDIR) +ALLFILES=$(for f in `make -V ALLFILES`; do echo $DIST_SUBDIR/$f; done) + +# Find packages +DEPENDS=`make all-depends-list` +PKGLIST=$(for x in $DEPENDS; do make -C $x PACKAGES=$2 -V PKGFILE; done) + +# Take an interesting subset of the ports tree +# TODO: how on Earth to do this robustly? +PORTSDIR=$(make -V PORTSDIR) +PORTS_SUBSET="$PORTSDIR/Mk $PORTSDIR/Templates $PORTSDIR/Tools $PORTSDIR/Keywords $PORTSDIR/UIDs $PORTSDIR/GIDs $PORTSDIR/*/Makefile.inc $DEPENDS" # Other important bits? + +tar cvzf $TARBALL $ALLFILES $PKGLIST $PORTS_SUBSET $1 set +e
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201307211659.r6LGxiIe094026>