From owner-svn-src-user@FreeBSD.ORG Sun Jul 21 22:20:45 2013 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by hub.freebsd.org (Postfix) with ESMTP id 45657E85; Sun, 21 Jul 2013 22:20:45 +0000 (UTC) (envelope-from nwhitehorn@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) by mx1.freebsd.org (Postfix) with ESMTP id 27F1E78A; Sun, 21 Jul 2013 22:20:45 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.7/8.14.7) with ESMTP id r6LMKjuV016512; Sun, 21 Jul 2013 22:20:45 GMT (envelope-from nwhitehorn@svn.freebsd.org) Received: (from nwhitehorn@localhost) by svn.freebsd.org (8.14.7/8.14.5/Submit) id r6LMKiWP016507; Sun, 21 Jul 2013 22:20:44 GMT (envelope-from nwhitehorn@svn.freebsd.org) Message-Id: <201307212220.r6LMKiWP016507@svn.freebsd.org> From: Nathan Whitehorn Date: Sun, 21 Jul 2013 22:20:44 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: svn commit: r253534 - user/nwhitehorn/condorports X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.14 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 21 Jul 2013 22:20:45 -0000 Author: nwhitehorn Date: Sun Jul 21 22:20:44 2013 New Revision: 253534 URL: http://svnweb.freebsd.org/changeset/base/253534 Log: Fancify tarball preparation, connect entire ports tree. I suspect the non-shared-FS thing may have to be abandoned: 2500 ports have a .include directive pointing to another directory and there does not seem at present to be any way to programmatically detect such dependencies. Modified: user/nwhitehorn/condorports/buildport.sh user/nwhitehorn/condorports/buildportsdag.sh user/nwhitehorn/condorports/stagebuildfiles.sh Modified: user/nwhitehorn/condorports/buildport.sh ============================================================================== --- user/nwhitehorn/condorports/buildport.sh Sun Jul 21 21:10:53 2013 (r253533) +++ user/nwhitehorn/condorports/buildport.sh Sun Jul 21 22:20:44 2013 (r253534) @@ -14,8 +14,8 @@ scratchdir=$(pwd) set -e sudo /pkgscripts/prepbuildjail $_CONDOR_SLOT $release -cat $tarballname | sudo jexec $_CONDOR_SLOT tar xvzf - -sudo jexec $_CONDOR_SLOT make -C $port install USE_PACKAGE_DEPENDS=true BATCH=true DISABLE_MAKE_JOBS=true +cat $tarballname | sudo jexec $_CONDOR_SLOT tar xvf - +sudo jexec $_CONDOR_SLOT make -C $port install USE_PACKAGE_DEPENDS=true BATCH=true DISABLE_MAKE_JOBS=true PKGREPOSITORY=/packages PACKAGES=/packages DISTDIR=/distfiles sudo jexec $_CONDOR_SLOT make -C $port package PKGREPOSITORY=/packages BATCH=true NO_LATEST_LINK=true ls -lh /scratch/$_CONDOR_SLOT/packages cp /scratch/$_CONDOR_SLOT/packages/$pkg $scratchdir/$pkg Modified: user/nwhitehorn/condorports/buildportsdag.sh ============================================================================== --- user/nwhitehorn/condorports/buildportsdag.sh Sun Jul 21 21:10:53 2013 (r253533) +++ user/nwhitehorn/condorports/buildportsdag.sh Sun Jul 21 22:20:44 2013 (r253534) @@ -1,13 +1,12 @@ #!/bin/sh : ${PORTSDIR=/usr/ports} -: ${PKGSDIR=/usr/ports/packages} +: ${PKGSDIR=$PORTSDIR/packages} +: ${DISTHORDE=$PORTSDIR/distfiles} : ${ARCH=`uname -p`} : ${RELEASE=$ARCH/`uname -r`} : ${STAGEDIR=$(pwd)/stage-$ARCH/$PKGSDIR} -#(cd $PORTSDIR && make index) -ports="$(cut -f 2 -d '|' $PORTSDIR/INDEX-`uname -r | cut -f 1 -d .`)" -ports="/usr/ports/games/sl /usr/ports/ports-mgmt/pkg /usr/ports/japanese/nkf" +#ports="/usr/ports/games/sl /usr/ports/ports-mgmt/pkg /usr/ports/japanese/nkf" mkdir -p $STAGEDIR @@ -15,7 +14,9 @@ dagjobs=$(pwd)/ports.dagjobs dagdeps=$(pwd)/ports.dagdeps dag=$(pwd)/ports.dag rm -f $dagjobs $dagdeps -for job in $ports; do +for category in $(make -C $PORTSDIR -V SUBDIR); do for port in $(make -C $PORTSDIR/$category -V SUBDIR); do + echo $category/$port + job=$PORTSDIR/$category/$port cd $job echo JOB $job package.sub >> $dagjobs # One fetch for all archs with dummy fetch job? @@ -23,7 +24,7 @@ for job in $ports; do export UNAME_p=$ARCH PKGNAME=$(make package-name) TARBALL=$STAGEDIR/$(make -V UNIQUENAME).tgz - echo SCRIPT PRE $job stagebuildfiles.sh $job $PKGSDIR $ARCH $TARBALL >> $dagjobs + echo SCRIPT PRE $job stagebuildfiles.sh $job $PKGSDIR $ARCH $TARBALL $DISTHORDE >> $dagjobs echo SCRIPT POST $job postbuild.sh $TARBALL \$RETURN >> $dagjobs echo VARS $job BuildArch=\"$ARCH\" >> $dagjobs echo VARS $job port=\"$job\" >> $dagjobs @@ -31,11 +32,11 @@ for job in $ports; do echo VARS $job pkgdir=\"$PKGSDIR/All\" >> $dagjobs echo VARS $job stagetarball=\"$TARBALL\" >> $dagjobs echo VARS $job release=\"$RELEASE\" >> $dagjobs - deps=$(make all-depends-list) + deps=$(make -V _DEPEND_DIRS) if [ ! -z "$deps" ]; then echo PARENT $deps CHILD $job >> $dagdeps fi -done +done; done cat $dagjobs $dagdeps > $dag rm -f $dagjobs $dagdeps Modified: user/nwhitehorn/condorports/stagebuildfiles.sh ============================================================================== --- user/nwhitehorn/condorports/stagebuildfiles.sh Sun Jul 21 21:10:53 2013 (r253533) +++ user/nwhitehorn/condorports/stagebuildfiles.sh Sun Jul 21 22:20:44 2013 (r253534) @@ -5,10 +5,11 @@ set -e export UNAME_p=$3 +DISTHORDE=$5 # Fetch distfiles if unfetched cd $1 -make fetch +make checksum DISTDIR=$DISTHORDE make package-links PACKAGES=$2 mkdir -p $2/All @@ -18,9 +19,10 @@ mkdir -p $2/All # Need POST script to clean up TARBALL=$4 +mkdir -p $(dirname $TARBALL) # Find distfiles -DIST_SUBDIR=$(make -V DISTDIR)/$(make -V DIST_SUBDIR) +DIST_SUBDIR=$DISTHORDE/$(make -V DIST_SUBDIR) ALLFILES=$(for f in `make -V ALLFILES`; do echo $DIST_SUBDIR/$f; done) # Find packages @@ -30,8 +32,19 @@ PKGLIST=$(for x in $DEPENDS; do make -C # Take an interesting subset of the ports tree # TODO: how on Earth to do this robustly? PORTSDIR=$(make -V PORTSDIR) -PORTS_SUBSET="$PORTSDIR/Mk $PORTSDIR/Templates $PORTSDIR/Tools $PORTSDIR/Keywords $PORTSDIR/UIDs $PORTSDIR/GIDs $PORTSDIR/*/Makefile.inc $DEPENDS" # Other important bits? +PORTS_SUBSET="$PORTSDIR/Mk $PORTSDIR/Templates $PORTSDIR/Tools $PORTSDIR/Keywords $PORTSDIR/UIDs $PORTSDIR/GIDs $PORTSDIR/*/Makefile.inc $DEPENDS $(make -V DIR_DEPENDS) $(make -V MASTERDIR)" # Other important bits? -tar cvzf $TARBALL $ALLFILES $PKGLIST $PORTS_SUBSET $1 +tar cvf $TARBALL.tmp $ALLFILES $PKGLIST $PORTS_SUBSET $1 +rm -rf $TARBALL-dir +mkdir -p $TARBALL-dir/packages $TARBALL-dir/distfiles +if [ -n "$ALLFILES" ]; then + ln -s $ALLFILES $TARBALL-dir/distfiles +fi +if [ -n "$PKGLIST" ]; then + ln -s $PKGLIST $TARBALL-dir/packages +fi +tar -r -C $TARBALL-dir -v -L -f $TARBALL.tmp distfiles packages +tar cvzf $TARBALL @$TARBALL.tmp +rm -rf $TARBALL.tmp $TARBALL-dir set +e