Date: Wed, 11 May 2011 08:42:31 +0000 (UTC) From: Florent Thoumie <flz@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r221762 - in projects/portbuild: conf qmanager scripts Message-ID: <201105110842.p4B8gVkP081578@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: flz Date: Wed May 11 08:42:31 2011 New Revision: 221762 URL: http://svn.freebsd.org/changeset/base/221762 Log: portbuild: don't hardcode package suffix. This isn't yet perfect, but setting pkg_sufx in portbuild.conf will now be mandatory, and will set PKG_SUFX during package creation. Modified: projects/portbuild/conf/common.conf projects/portbuild/qmanager/packagebuild projects/portbuild/scripts/buildscript projects/portbuild/scripts/chopindex projects/portbuild/scripts/claim-chroot projects/portbuild/scripts/dopackages projects/portbuild/scripts/dopackagestats projects/portbuild/scripts/pdispatch projects/portbuild/scripts/portbuild projects/portbuild/scripts/prunefailure projects/portbuild/scripts/prunepkgs projects/portbuild/scripts/stats Modified: projects/portbuild/conf/common.conf ============================================================================== --- projects/portbuild/conf/common.conf Wed May 11 08:42:29 2011 (r221761) +++ projects/portbuild/conf/common.conf Wed May 11 08:42:31 2011 (r221762) @@ -14,7 +14,6 @@ # LOCALBASE=/usr/local -PKGSUFFIX=.tbz ARCHS_REQUIRING_AOUT_COMPAT="i386" ARCHS_REQUIRING_LINPROCFS="amd64 i386" Modified: projects/portbuild/qmanager/packagebuild ============================================================================== --- projects/portbuild/qmanager/packagebuild Wed May 11 08:42:29 2011 (r221761) +++ projects/portbuild/qmanager/packagebuild Wed May 11 08:42:31 2011 (r221762) @@ -70,6 +70,8 @@ DEBUG = False categories = {} ports = {} +pkg_sufx = None + # When a build fails we requeue it with a lower priority such that it # will never preempt a phase 1 build but will build when spare # capacity is available. @@ -164,6 +166,7 @@ class Port(object): self.descr = descr self.maintainer = maintainer self.www = www + self.sufx = pkg_sufx # Populated later self.bdep = [] @@ -325,8 +328,8 @@ they still need to know about us as depe def packagename(self, arch, branch, buildid): """ Return the path where a package may be found""" - return "%s/%s/%s/builds/%s/packages/All/%s.tbz" \ - % (pbd, arch, branch, buildid, self.name) + return "%s/%s/%s/builds/%s/packages/All/%s%s" \ + % (pbd, arch, branch, buildid, self.name, self.sufx) def is_stale(self, arch, branch, buildid): """ Does a package need to be (re)-built? @@ -377,8 +380,8 @@ def gettargets(targets): if i.endswith("-all"): cat = i.rpartition("-")[0] plist.update(p.name for p in categories[cat].ports) - elif i.rstrip(".tbz") in ports: - plist.update([ports[i.rstrip(".tbz")].name]) + elif i.rstrip(pkg_sufx) in ports: + plist.update([ports[i.rstrip(pkg_sufx)].name]) else: raise KeyError, i @@ -425,18 +428,18 @@ class worker(threading.Thread): try: runenv={'HOME':"/root", 'PATH':'/sbin:/bin:/usr/sbin:/usr/bin:/usr/games:/usr/local/sbin:/usr/local/bin:%s/scripts' + pbc, - 'FD':" ".join(["%s.tbz" % p.name for p in pkg.fdep]), - 'ED':" ".join(["%s.tbz" % p.name for p in pkg.edep]), - 'PD':" ".join(["%s.tbz" % p.name for p in pkg.pdep]), - 'BD':" ".join(["%s.tbz" % p.name for p in pkg.bdep]), - 'RD':" ".join(["%s.tbz" % p.name for p in pkg.rdep])} + 'FD':" ".join([p.name + p.sufx for p in pkg.fdep]), + 'ED':" ".join([p.name + p.sufx for p in pkg.edep]), + 'PD':" ".join([p.name + p.sufx for p in pkg.pdep]), + 'BD':" ".join([p.name + p.sufx for p in pkg.bdep]), + 'RD':" ".join([p.name + p.sufx for p in pkg.rdep])} for var in ["NOCLEAN", "NO_RESTRICTED", "NOPLISTCHECK", "NO_DISTFILES", "FETCH_ORIGINAL", "TRYBROKEN", "PORTBUILD_CHECKOUT", "PORTBUILD_DATA" ]: if var in os.environ: runenv[var] = os.environ.get(var) build = subprocess.Popen( ["/bin/sh", "%s/scripts/pdispatch" % pbc, self.arch, self.branch, self.buildid, self.machine, - "/tmp/%s/scripts/portbuild" % self.buildid, "%s.tbz" % pkg.name, + "/tmp/%s/scripts/portbuild" % self.buildid, pkg.name + pkg.sufx, pkg.path], env=runenv, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, bufsize=0) @@ -506,7 +509,7 @@ class worker(threading.Thread): wrk.start() def main(arch, branch, buildid, args): - global index + global index, pkg_sufx basedir=os.path.realpath(pbd+"/"+arch+"/"+branch+"/builds/"+buildid) buildid=basedir.split("/")[-1] @@ -518,6 +521,18 @@ def main(arch, branch, buildid, args): branchbase = branchbase.split(".")[ 0 ] indexfile=portsdir+"/INDEX-"+branchbase + archconfig = getConfig(pbd, arch, "portbuild.conf") + try: + branchconfig = getConfig(pbd, "%s/%s" % (arch, branch), "portbuild.conf") + archconfig.merge(branchconfig) + except: + pass + + pkg_sufx = archconfig.get('pkg_sufx') + if not pkg_sufx: + print "error: pkg_sufx not defined in portbuild.conf" + sys.exit(1) + print "[MASTER] parseindex..." index = Index(indexfile) index.parse() Modified: projects/portbuild/scripts/buildscript ============================================================================== --- projects/portbuild/scripts/buildscript Wed May 11 08:42:29 2011 (r221761) +++ projects/portbuild/scripts/buildscript Wed May 11 08:42:31 2011 (r221762) @@ -51,8 +51,7 @@ add_pkg() { echo "adding dependencies" for i in $pkgs; do echo "pkg_add $i" - base=$(basename $i .tgz) - base=$(basename $base .tbz) + base=$(basename $i ${pkg_sufx}) if pkg_info -q -e $base; then echo "skipping $base, already added" else @@ -78,8 +77,7 @@ del_pkg() { unset delpkg nextpkg recursion=0 for i in $pkgs; do - base=$(basename $i .tgz) - base=$(basename $base .tbz) + base=$(basename $i ${pkg_sufx}) if [ -s /var/db/pkg/${base}/+REQUIRED_BY ]; then recursion=1 nextpkg="${base} ${nextpkg}" @@ -127,6 +125,9 @@ Z=`ident ${dir}/Makefile | grep 'FreeBSD cd $dir || exit 1 restr=$(make -V RESTRICTED) +# Inherit from environment set by portbuild. +pkg_sufx=${PKG_SUFX} + # Keep restricted distfiles in a subdirectory for extra protection # against leakage if [ ! -z "$restr" ]; then Modified: projects/portbuild/scripts/chopindex ============================================================================== --- projects/portbuild/scripts/chopindex Wed May 11 08:42:29 2011 (r221761) +++ projects/portbuild/scripts/chopindex Wed May 11 08:42:31 2011 (r221762) @@ -1,6 +1,7 @@ #!/usr/bin/env python import os, sys +import re if len(sys.argv) != 3: print "%s: <index> <pkgdir>" % sys.argv[0] @@ -12,7 +13,7 @@ pkgdir = sys.argv[2] if not pkgdir.endswith("/All"): pkgdir = pkgdir + "/All" -packages = [pkg for (pkg, ext) in map(os.path.splitext, os.listdir(pkgdir)) if ext == ".tbz"] +packages = [pkg for (pkg, ext) in map(os.path.splitext, os.listdir(pkgdir)) if re.match('[.]t[bgx]z', ext)] index=[] pkgs=[] Modified: projects/portbuild/scripts/claim-chroot ============================================================================== --- projects/portbuild/scripts/claim-chroot Wed May 11 08:42:29 2011 (r221761) +++ projects/portbuild/scripts/claim-chroot Wed May 11 08:42:31 2011 (r221762) @@ -73,7 +73,7 @@ fi . ${pbd}/${arch}/portbuild.$(hostname) buildroot=${scratchdir} -pkgname=${pkgname%.${PKGSUFFIX}} +pkgname=${pkgname%.${pkg_sufx}} chrootdir=${buildroot}/${branch}/${buildid}/chroot Modified: projects/portbuild/scripts/dopackages ============================================================================== --- projects/portbuild/scripts/dopackages Wed May 11 08:42:29 2011 (r221761) +++ projects/portbuild/scripts/dopackages Wed May 11 08:42:31 2011 (r221762) @@ -226,7 +226,7 @@ generatemd5 () { echo "started generating CHECKSUM.MD5 at $(date)" cd ${builddir}/packages/All - find . -name '*.tbz' | sort | sed -e 's/^..//' | xargs md5 > CHECKSUM.MD5 + find . -name "*${pkg_sufx}" | sort | sed -e 's/^..//' | xargs md5 > CHECKSUM.MD5 echo "ended generating CHECKSUM.MD5 at $(date)" } @@ -250,7 +250,7 @@ dobuild() { echo "ended at $(date)" end=$(date +%s) echo "Build took $(date -u -j -r $((end - start)) | awk '{print $4}')" - echo $(ls -1 ${builddir}/packages/All | grep tbz | wc -l) "packages built" + echo $(ls -1 ${builddir}/packages/All | grep ${pkg_sufx} | wc -l) "packages built" echo $(wc -l ${PORTSDIR}/${INDEXFILE} | awk '{print $1}') "lines in INDEX" @@ -648,7 +648,7 @@ if [ "$skipstart" = 0 ]; then rm ${INDEXFILE}.old1 ${INDEXFILE}.1 cd ${PACKAGES}/All - sed "s,$,${PKGSUFFIX}," ${builddir}/.oldports | xargs rm -f + sed "s,$,${pkg_sufx}," ${builddir}/.oldports | xargs rm -f # XXX MCL takes an unknown period of time. # XXX MCL return value not checked. ${pbc}/scripts/prunepkgs ${PORTSDIR}/${INDEXFILE} ${PACKAGES} @@ -731,7 +731,7 @@ fi #rm -rf ${builddir}/bad #mkdir -p ${builddir}/bad #echo "checking packages" -#for i in *${PKGSUFFIX}; do +#for i in *${pkg_sufx}; do # if ! ${PKGZIPCMD} -t $i; then # echo "Warning: package $i is bad, moving to ${builddir}/bad" # # the latest link will be left behind... Modified: projects/portbuild/scripts/dopackagestats ============================================================================== --- projects/portbuild/scripts/dopackagestats Wed May 11 08:42:29 2011 (r221761) +++ projects/portbuild/scripts/dopackagestats Wed May 11 08:42:31 2011 (r221762) @@ -123,7 +123,7 @@ write_row () { n_packages=0 if [ -d $directory/packages/All ]; then # MCL removed 20090808 -- this takes way too long - # n_packages=`find $directory/packages/All -name \*.tbz -or -name \*.tgz |wc -l` + # n_packages=`find $directory/packages/All -name '*.t[bgx]z' | wc -l` n_packages=`ls $directory/packages/All | grep -v CHECKSUM.MD5 | wc -l` have_packages="yes" fi Modified: projects/portbuild/scripts/pdispatch ============================================================================== --- projects/portbuild/scripts/pdispatch Wed May 11 08:42:29 2011 (r221761) +++ projects/portbuild/scripts/pdispatch Wed May 11 08:42:31 2011 (r221762) @@ -2,7 +2,7 @@ # $FreeBSD: ports/Tools/portbuild/scripts/pdispatch,v 1.40 2011/01/26 10:41:53 linimon Exp $ # -# pdispatch <arch> <branch> <buildid> <host> <command> <package.tbz> [<args> ...] +# pdispatch <arch> <branch> <buildid> <host> <command> <package> [<args> ...] # # server-side script to dispatch the job to a host via the ptimeout script. @@ -50,7 +50,7 @@ if [ -z "${scp_cmd}" ]; then scp_cmd=scp fi -pkgname=$(basename $1 ${PKGSUFFIX}) +pkgname=$(basename $1 ${pkg_sufx}) if [ -z "${pkgname}" ]; then echo "null packagename" @@ -150,7 +150,7 @@ error=$? # Pull in the results of the build from the client ${scp_cmd} ${client_user}@${host}:${chroot}/tmp/${pkgname}.log ${builddir}/logs/${pkgname}.log -(${ssh_cmd} -a -n ${client_user}@${host} test -f ${chroot}/tmp/work.tbz ) && ${scp_cmd} ${client_user}@${host}:${chroot}/tmp/work.tbz ${builddir}/wrkdirs/${pkgname}.tbz +(${ssh_cmd} -a -n ${client_user}@${host} test -f ${chroot}/tmp/work.tbz ) && ${scp_cmd} ${client_user}@${host}:${chroot}/tmp/work.tbz ${builddir}/wrkdirs/${pkgname}${pkg_sufx} # XXX Set dirty flag if any of the scp's fail @@ -164,8 +164,8 @@ if [ "${error}" = 0 ]; then tar --unlink -C ${builddir} -xvf - # XXX why is this needed? - test -f ${builddir}/packages/All/${pkgname}${PKGSUFFIX} && \ - touch ${builddir}/packages/All/${pkgname}${PKGSUFFIX} + test -f ${builddir}/packages/All/${pkgname}${pkg_sufx} && \ + touch ${builddir}/packages/All/${pkgname}${pkg_sufx} if [ -f ${builddir}/errors/${pkgname}.log ]; then rm -f ${builddir}/errors/${pkgname}.log Modified: projects/portbuild/scripts/portbuild ============================================================================== --- projects/portbuild/scripts/portbuild Wed May 11 08:42:29 2011 (r221761) +++ projects/portbuild/scripts/portbuild Wed May 11 08:42:31 2011 (r221762) @@ -130,7 +130,7 @@ unset __MAKE_CONF # set overrides for make.conf export BACKUP_FTP_SITE=${CLIENT_BACKUP_FTP_SITE} -pkgname=$(basename $6 ${PKGSUFFIX}) +pkgname=$(basename $6 ${pkg_sufx}) dirname=$7 shift 2 @@ -143,6 +143,7 @@ export DISTDIR=${CLIENT_DISTDIR} export LOCALBASE=${LOCALBASE} export PACKAGES=${CLIENT_PACKAGES_LOCATION} export SRC_BASE=${CLIENT_SRCBASE} +export PKG_SUFX=${pkg_sufx} # to catch missing dependencies #export DEPENDS_TARGET=/usr/bin/true Modified: projects/portbuild/scripts/prunefailure ============================================================================== --- projects/portbuild/scripts/prunefailure Wed May 11 08:42:29 2011 (r221761) +++ projects/portbuild/scripts/prunefailure Wed May 11 08:42:31 2011 (r221762) @@ -74,7 +74,7 @@ while read dir name ver olddate date cou newver=$(echo $entry | awk '{print $1}') - if [ -e "${builddir}/packages/All/$newver${PKGSUFFIX}" ]; then + if [ -e "${builddir}/packages/All/$newver${pkg_sufx}" ]; then echo "$newver package exists, should not still be here!" rm -f ${pbd}/${arch}/${branch}/latest/${dir} continue Modified: projects/portbuild/scripts/prunepkgs ============================================================================== --- projects/portbuild/scripts/prunepkgs Wed May 11 08:42:29 2011 (r221761) +++ projects/portbuild/scripts/prunepkgs Wed May 11 08:42:31 2011 (r221762) @@ -30,7 +30,7 @@ trap "rm -rf $tmpdir; exit 1" 1 2 3 5 10 # Check for non-package files -extras=$(find ${pkgdir} -type f \! \( -name INDEX -o -name CHECKSUM.MD5 -o -name \*.tgz -o -name \*.tbz \) ) +extras=$(find ${pkgdir} -type f \! \( -name INDEX -o -name CHECKSUM.MD5 -o -name '*.t[bgx]z' \) ) echo "==> Removing extra files" echo $extras if [ "x${extras}" != "x" ]; then @@ -42,14 +42,14 @@ fi # Check for files not present in INDEX echo "==> Removing extra package files" -find $pkgdir/All -type f -name \*.tgz -o -name \*.tbz | sed -e "s,${pkgdir}/All/,," -e 's,\.tbz$,,' -e 's,\.tgz$,,' |sort > ${tmpdir}/files +find $pkgdir/All -type f -name '*.t[bgx]z' | sed -e "s,${pkgdir}/All/,," -e 's,\.t[bgx]z$,,' |sort > ${tmpdir}/files cut -f 1 -d '|' ${index} |sort > ${tmpdir}/packages extras=$(comm -2 -3 ${tmpdir}/files ${tmpdir}/packages) echo $extras if [ "${dummy}" = "0" ]; then for i in $extras; do - rm -f $pkgdir/All/${i}.tgz $pkgdir/All/${i}.tbz + rm -f $pkgdir/All/${i}.t[bgx]z done fi Modified: projects/portbuild/scripts/stats ============================================================================== --- projects/portbuild/scripts/stats Wed May 11 08:42:29 2011 (r221761) +++ projects/portbuild/scripts/stats Wed May 11 08:42:31 2011 (r221762) @@ -15,7 +15,7 @@ branch=$1 for i in ${SUPPORTED_ARCHS}; do all=${pbd}/$i/${branch}/builds/latest/packages/All if [ -d ${all} ]; then - count=$(find ${all} -name \*.tbz -o -name \*.tgz |wc -l) + count=$(find ${all} -name '*.t[bgx]z' | wc -l) echo -n "$i: ${count} " fi done
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201105110842.p4B8gVkP081578>