Date: Thu, 28 Apr 2011 11:06:15 +0000 (UTC) From: Florent Thoumie <flz@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r221159 - in projects/portbuild: qmanager scripts Message-ID: <201104281106.p3SB6Fwc020072@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: flz Date: Thu Apr 28 11:06:14 2011 New Revision: 221159 URL: http://svn.freebsd.org/changeset/base/221159 Log: portbuild: remove hardcoded /var/portbuild. - Don't hardcode /var/portbuild. - Upload per-build scripts/ and sources/ directories. - Scripts which aren't tied to a specific build are uploaded first. This change makes it effectively possible to checkout portbuild anywhere and test changes without affecting any other builds. Modified: projects/portbuild/qmanager/dumpdb.py projects/portbuild/qmanager/packagebuild projects/portbuild/qmanager/qmanager projects/portbuild/qmanager/qmanager.py projects/portbuild/scripts/allgohans projects/portbuild/scripts/build projects/portbuild/scripts/buildenv projects/portbuild/scripts/buildfailure projects/portbuild/scripts/buildproxy projects/portbuild/scripts/buildproxy-client projects/portbuild/scripts/buildsuccess projects/portbuild/scripts/claim-chroot projects/portbuild/scripts/clean-chroot projects/portbuild/scripts/cleanup-chroots projects/portbuild/scripts/client-metrics projects/portbuild/scripts/cpdistfiles projects/portbuild/scripts/cppackages projects/portbuild/scripts/do-cleanup-chroots projects/portbuild/scripts/docppackages projects/portbuild/scripts/dodistfiles projects/portbuild/scripts/dologs projects/portbuild/scripts/dopackages projects/portbuild/scripts/dopackages.wrapper projects/portbuild/scripts/dopackagestats projects/portbuild/scripts/dosetupnode projects/portbuild/scripts/flushsquid projects/portbuild/scripts/keeprestr projects/portbuild/scripts/makeduds projects/portbuild/scripts/makeindex projects/portbuild/scripts/makerestr projects/portbuild/scripts/makeworld projects/portbuild/scripts/mkbindist projects/portbuild/scripts/pdispatch projects/portbuild/scripts/pollmachine projects/portbuild/scripts/portbuild projects/portbuild/scripts/processfail projects/portbuild/scripts/processlogs projects/portbuild/scripts/prunefailure projects/portbuild/scripts/reportload projects/portbuild/scripts/setupnode projects/portbuild/scripts/stats projects/portbuild/scripts/updatesnap projects/portbuild/scripts/updatesnap.ports Modified: projects/portbuild/qmanager/dumpdb.py ============================================================================== --- projects/portbuild/qmanager/dumpdb.py Thu Apr 28 09:09:34 2011 (r221158) +++ projects/portbuild/qmanager/dumpdb.py Thu Apr 28 11:06:14 2011 (r221159) @@ -4,17 +4,21 @@ import sys, os, threading, socket, Queue +pbc = os.getenv('PORTBUILD_CHECKOUT') \ + if os.getenv('PORTBUILD_CHECKOUT') else "/var/portbuild" +pbd = os.getenv('PORTBUILD_DATA') \ + if os.getenv('PORTBUILD_DATA') else "/var/portbuild" + +sys.path.insert(0, '%s/lib/python' % pbc) + from signal import * from sys import exc_info from itertools import chain -sys.path.insert(0, '/var/portbuild/lib/python') - from freebsd_config import * from qmanagerobj import * -CONFIG_DIR="/var/portbuild" CONFIG_SUBDIR="conf" CONFIG_FILENAME="server.conf" @@ -127,7 +131,7 @@ def show_machines_for_arch( engine, arch if __name__ == '__main__': print "acquiring engine and session" - config = getConfig( CONFIG_DIR, CONFIG_SUBDIR, CONFIG_FILENAME ) + config = getConfig( pbc, CONFIG_SUBDIR, CONFIG_FILENAME ) QMANAGER_PATH = config.get( 'QMANAGER_PATH' ) QMANAGER_DATABASE_FILE = config.get( 'QMANAGER_DATABASE_FILE' ) (engine, session) = obj_startup( \ Modified: projects/portbuild/qmanager/packagebuild ============================================================================== --- projects/portbuild/qmanager/packagebuild Thu Apr 28 09:09:34 2011 (r221158) +++ projects/portbuild/qmanager/packagebuild Thu Apr 28 11:06:14 2011 (r221159) @@ -32,31 +32,33 @@ # * check mtime for package staleness (cf make) # * option to skip phase 2 +import os import sys -sys.path.insert(0, '/var/portbuild/lib/python') +pbc = os.getenv('PORTBUILD_CHECKOUT') \ + if os.getenv('PORTBUILD_CHECKOUT') else "/var/portbuild" +pbd = os.getenv('PORTBUILD_DATA') \ + if os.getenv('PORTBUILD_DATA') else "/var/portbuild" -from qmanagerclient import * +sys.path.insert(0, '%s/lib/python' % pbc) +from qmanagerclient import * from freebsd_config import * -import os, string, threading, time, subprocess -#import random +import string, threading, time, subprocess from itertools import chain -#import gc from stat import * from Queue import Queue from heapq import * -CONFIG_DIR="/var/portbuild" CONFIG_SUBDIR="conf" CONFIG_FILENAME="server.conf" -config = getConfig( CONFIG_DIR, CONFIG_SUBDIR, CONFIG_FILENAME ) +config = getConfig( pbc, CONFIG_SUBDIR, CONFIG_FILENAME ) QMANAGER_MAX_JOB_ATTEMPTS = int( \ config.get( 'QMANAGER_MAX_JOB_ATTEMPTS' ) ) -QMANAGER_PRIORITY_PACKAGES = string.split( \ +QMANAGER_PRIORITY_PACKAGES = string.split( \ config.get( 'QMANAGER_PRIORITY_PACKAGES' ) ) QMANAGER_RUNAWAY_PERCENTAGE = float( \ config.get( 'QMANAGER_RUNAWAY_PERCENTAGE' ) ) @@ -323,8 +325,8 @@ they still need to know about us as depe def packagename(self, arch, branch, buildid): """ Return the path where a package may be found""" - return "/var/portbuild/%s/%s/builds/%s/packages/All/%s.tbz" \ - % (arch, branch, buildid, self.name) + return "%s/%s/%s/builds/%s/packages/All/%s.tbz" \ + % (pbd, arch, branch, buildid, self.name) def is_stale(self, arch, branch, buildid): """ Does a package need to be (re)-built? @@ -409,6 +411,7 @@ class worker(threading.Thread): self.branch = branch self.buildid = buildid self.queue = queue + self.basedir = pbd+"/"+arch+"/"+branch+"/builds/"+buildid self.setDaemon(True) @@ -422,19 +425,19 @@ class worker(threading.Thread): print try: runenv={'HOME':"/root", - 'PATH':'/sbin:/bin:/usr/sbin:/usr/bin:/usr/games:/usr/local/sbin:/usr/local/bin:/var/portbuild/scripts', + 'PATH':'/sbin:/bin:/usr/sbin:/usr/bin:/usr/games:/usr/local/sbin:/usr/local/bin:%s/scripts' + pbc, 'FD':" ".join(["%s.tbz" % p.name for p in pkg.fdep]), 'ED':" ".join(["%s.tbz" % p.name for p in pkg.edep]), 'PD':" ".join(["%s.tbz" % p.name for p in pkg.pdep]), 'BD':" ".join(["%s.tbz" % p.name for p in pkg.bdep]), 'RD':" ".join(["%s.tbz" % p.name for p in pkg.rdep])} - for var in ["NOCLEAN", "NO_RESTRICTED", "NOPLISTCHECK", "NO_DISTFILES", "FETCH_ORIGINAL", "TRYBROKEN" ]: + for var in ["NOCLEAN", "NO_RESTRICTED", "NOPLISTCHECK", "NO_DISTFILES", "FETCH_ORIGINAL", "TRYBROKEN", "PORTBUILD_CHECKOUT", "PORTBUILD_DATA" ]: if var in os.environ: runenv[var] = os.environ.get(var) build = subprocess.Popen( - ["/bin/sh", "/var/portbuild/scripts/pdispatch", + ["/bin/sh", "%s/scripts/pdispatch" % pbc, self.arch, self.branch, self.buildid, self.machine, - "/var/portbuild/scripts/portbuild", "%s.tbz" % pkg.name, + "%s/scripts/portbuild" % self.basedir, "%s.tbz" % pkg.name, pkg.path], env=runenv, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, bufsize=0) @@ -506,7 +509,7 @@ class worker(threading.Thread): def main(arch, branch, buildid, args): global index - basedir="/var/portbuild/"+arch+"/"+branch+"/builds/"+buildid + basedir=pbd+"/"+arch+"/"+branch+"/builds/"+buildid portsdir=basedir+"/ports" # get the major branch number. Modified: projects/portbuild/qmanager/qmanager ============================================================================== --- projects/portbuild/qmanager/qmanager Thu Apr 28 09:09:34 2011 (r221158) +++ projects/portbuild/qmanager/qmanager Thu Apr 28 11:06:14 2011 (r221159) @@ -49,11 +49,15 @@ # - OR, NOT job description entries # - query jobs.machine properties +import os import sys -sys.path.insert(0, '/var/portbuild/lib/python') +pbc = os.getenv('PORTBUILD_CHECKOUT') \ + if os.getenv('PORTBUILD_CHECKOUT') else "/var/portbuild" +pbd = os.getenv('PORTBUILD_DATA') \ + if os.getenv('PORTBUILD_DATA') else "/var/portbuild" -from freebsd_config import * +sys.path.insert(0, '%s/lib/python' % pbc) import os, threading, socket, Queue @@ -61,12 +65,12 @@ from signal import * from itertools import chain from qmanagerobj import * +from freebsd_config import * -CONFIG_DIR="/var/portbuild" CONFIG_SUBDIR="conf" CONFIG_FILENAME="server.conf" -config = getConfig( CONFIG_DIR, CONFIG_SUBDIR, CONFIG_FILENAME ) +config = getConfig( pbc, CONFIG_SUBDIR, CONFIG_FILENAME ) QMANAGER_SOCKET_FILE = config.get( 'QMANAGER_SOCKET_FILE' ) class Worker(object): Modified: projects/portbuild/qmanager/qmanager.py ============================================================================== --- projects/portbuild/qmanager/qmanager.py Thu Apr 28 09:09:34 2011 (r221158) +++ projects/portbuild/qmanager/qmanager.py Thu Apr 28 11:06:14 2011 (r221159) @@ -49,24 +49,28 @@ # - OR, NOT job description entries # - query jobs.machine properties +import os import sys -sys.path.insert(0, '/var/portbuild/lib/python') +pbc = os.getenv('PORTBUILD_CHECKOUT') \ + if os.getenv('PORTBUILD_CHECKOUT') else "/var/portbuild" +pbd = os.getenv('PORTBUILD_DATA') \ + if os.getenv('PORTBUILD_DATA') else "/var/portbuild" -from freebsd_config import * +sys.path.insert(0, '%s/lib/python' % pbc) -import os, socket, threading, time, Queue +import socket, threading, time, Queue from signal import * from itertools import chain from qmanagerobj import * +from freebsd_config import * -CONFIG_DIR="/var/portbuild" CONFIG_SUBDIR="conf" CONFIG_FILENAME="server.conf" -config = getConfig( CONFIG_DIR, CONFIG_SUBDIR, CONFIG_FILENAME ) +config = getConfig( pbc, CONFIG_SUBDIR, CONFIG_FILENAME ) QMANAGER_SOCKET_FILE = config.get( 'QMANAGER_SOCKET_FILE' ) DEBUG = True Modified: projects/portbuild/scripts/allgohans ============================================================================== --- projects/portbuild/scripts/allgohans Thu Apr 28 09:09:34 2011 (r221158) +++ projects/portbuild/scripts/allgohans Thu Apr 28 11:06:14 2011 (r221159) @@ -1,30 +1,46 @@ #!/bin/sh +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} +pbd=${PORTBUILD_DATA:-/var/portbuild} + +. ${pbc}/conf/server.conf + doarch() { arch=$1 shift - if [ -f "${pb}/${arch}/portbuild.conf" ]; then - . ${pb}/${arch}/portbuild.conf + if [ -f "${pbd}/${arch}/portbuild.conf" ]; then + . ${pbd}/${arch}/portbuild.conf else echo "Invalid arch" exit 1 fi - machines=$(awk '{print $1}' ${pb}/${arch}/mlist) + machines=$(awk '{print $1}' ${pbd}/${arch}/mlist) for i in ${machines}; do - . ${pb}/${arch}/portbuild.conf - if [ -f "${pb}/${arch}/portbuild.${i}" ]; then - . ${pb}/${arch}/portbuild.${i} + . ${pbd}/${arch}/portbuild.conf + if [ -f "${pbd}/${arch}/portbuild.${i}" ]; then + . ${pbd}/${arch}/portbuild.${i} fi if [ ${quiet} -eq 0 ]; then echo "[$i]" fi - lockf -t 60 ${pb}/${arch}/lockfiles/lock.$i su ports-${arch} -c "ssh ${client_user}@$i ${sudo_cmd} $@" + cmdpath=${cmd} + if [ ${scpfirst} -ne 0 ]; then + # If we need to scp first, the command given in the parameters is the local + # path, which we need to upload first. The command will return the remote + # temporary file, which we can subsequently execute. So this isn't really scp. + cmdpath=$(su ports-${arch} -c "cat ${cmd} | ssh ${client_user}@$i 't=\$(mktemp -t ${cmd##*/}); cat >\$t; echo \$t; chmod 755 \$t'") + case ${cmdpath} in /tmp/*) ;; *) echo "Failed to scp ${cmd} to $i."; return 1;; esac + fi + lockf -t 60 ${pbd}/${arch}/lockfiles/lock.$i su ports-${arch} -c "ssh ${client_user}@$i ${sudo_cmd} ${cmdpath} $@" result=$? if [ $result -ne 0 ]; then - echo "could not execute command $@ on $i: $result" + echo "could not execute command ${cmdpath} $@ on $i: $result" + fi + if [ ${scpfirst} -ne 0 ]; then + su ports-${arch} -c "ssh ${client_user}@$i 'rm -f ${cmdpath}'" fi done } @@ -39,6 +55,7 @@ else root=0 fi +# XXX - Have to convert this to getopt(). if [ "$1" = "-q" ]; then quiet=1 shift @@ -46,11 +63,18 @@ else quiet=0 fi -pb=/var/portbuild -. ${pb}/conf/server.conf +if [ "$1" = "-scp" ]; then + scpfirst=1 + shift +else + scpfirst=0 +fi + +cmd=$1 +shift if [ "${arch}" = "all" ]; then - arches=$(find ${pb}/*/portbuild.conf) + arches=$(find ${pbd}/*/portbuild.conf) for i in ${arches}; do arch=$(basename $(dirname $i)) doarch $arch "$@" Modified: projects/portbuild/scripts/build ============================================================================== --- projects/portbuild/scripts/build Thu Apr 28 09:09:34 2011 (r221158) +++ projects/portbuild/scripts/build Thu Apr 28 11:06:14 2011 (r221159) @@ -4,7 +4,8 @@ # server-side script to handle various commands common to builds # configurable variables -pb=/var/portbuild +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} +pbd=${PORTBUILD_DATA:-/var/portbuild} # subdirectories to process. yes, this is a hack, but it saves code # duplication. @@ -25,7 +26,7 @@ do_list() { arch=$1 branch=$2 - buildpar=/var/portbuild/${arch}/${branch}/builds + buildpar=${pbd}/${arch}/${branch}/builds if [ -d ${buildpar} ]; then snaps=$(cd ${buildpar}; ls -1d 2* 2> /dev/null) @@ -51,7 +52,7 @@ do_create() { buildid=$3 shift 3 - archivedir=${pb}/${arch}/archive + archivedir=${pbd}/${arch}/archive # create directory for all build logs buildlogsdir=${archivedir}/buildlogs @@ -93,8 +94,7 @@ do_create() { ln -sf ${builddir} ${pbab}/builds/latest # create the links for the webserver - # XXX MCL hardcoding - errorlogs=/var/portbuild/errorlogs + errorlogs=${pbd}/errorlogs ln -s ${buildsdir}/latest/bak/errors ${errorlogs}/${arch}-${branch}-previous ln -s ${buildsdir}/latest/bak/logs ${errorlogs}/${arch}-${branch}-previous-logs @@ -148,7 +148,7 @@ do_clone() { if [ -d ${newbuilddir} ]; then if [ ! -f ${pbab}/builds/previous/.keep ]; then - /var/portbuild/scripts/build destroy ${arch} ${branch} previous + ${pbc}/scripts/build destroy ${arch} ${branch} previous fi rm -f ${pbab}/builds/previous mv ${pbab}/builds/latest ${pbab}/builds/previous @@ -251,10 +251,10 @@ cleanup_client() { echo "Started cleaning up ${arch}/${branch} build ID ${buildid} on ${mach}" - test -f ${pb}/${arch}/portbuild.${mach} && . ${pb}/${arch}/portbuild.${mach} + test -f ${pbd}/${arch}/portbuild.${mach} && . ${pbd}/${arch}/portbuild.${mach} # Kill off builds and clean up chroot - ${pb}/scripts/dosetupnode ${arch} ${branch} ${buildid} ${mach} -nocopy -queue -full + ${pbc}/scripts/dosetupnode ${arch} ${branch} ${buildid} ${mach} -nocopy -queue -full echo "Finished cleaning up ${arch}/${branch} build ID ${buildid} on ${mach}" } @@ -267,7 +267,7 @@ do_cleanup() { arg=$5 shift 5 - for i in `cat ${pb}/${arch}/mlist`; do + for i in `cat ${pbd}/${arch}/mlist`; do cleanup_client ${arch} ${branch} ${buildid} ${i} ${arg} & done wait @@ -394,7 +394,7 @@ do_destroy() { shift 4 echo - buildid=$(resolve ${pb} ${arch} ${branch} ${buildid}) + buildid=$(resolve ${pbd} ${arch} ${branch} ${buildid}) if [ -z "${buildid}" ]; then echo "Invalid build ID ${buildid}" exit 1 @@ -435,7 +435,7 @@ proxy_root() { id=$(id -u) if [ ${id} != "0" ]; then - /var/portbuild/scripts/buildproxy-client "build ${cmd} ${arch} ${branch} ${buildid} ${args}" + ${pbc}/scripts/buildproxy-client "build ${cmd} ${arch} ${branch} ${buildid} ${args}" error=$? if [ ${error} -eq 254 ]; then echo "Proxy error" @@ -463,7 +463,7 @@ proxy_user() { eval "do_${cmd} ${arch} ${branch} ${buildid} \"${builddir}\" ${args}" error=$? else - su ports-${arch} -c "/var/portbuild/scripts/build ${cmd} ${arch} ${branch} ${buildid} \"${builddir}\" ${args}" + su ports-${arch} -c "${pbc}/scripts/build ${cmd} ${arch} ${branch} ${buildid} \"${builddir}\" ${args}" error=$? fi @@ -486,11 +486,11 @@ arch=$2 branch=$3 shift 3 -. ${pb}/conf/server.conf -. ${pb}/${arch}/portbuild.conf -. ${pb}/scripts/buildenv +. ${pbc}/conf/server.conf +. ${pbd}/${arch}/portbuild.conf +. ${pbc}/scripts/buildenv -pbab=${pb}/${arch}/${branch} +pbab=${pbd}/${arch}/${branch} validate_env ${arch} ${branch} || exit 1 @@ -505,7 +505,7 @@ if [ $# -ge 1 ]; then case "$cmd" in cleanup) # Resolve symlinks but don't bail if the build doesn't exist. - newbuildid=$(resolve ${pb} ${arch} ${branch} ${buildid}) + newbuildid=$(resolve ${pbd} ${arch} ${branch} ${buildid}) if [ ! -z "${newbuildid}" -a "${newbuildid}" != "${buildid}" ]; then echo "Resolved ${buildid} to ${newbuildid}" buildid=${newbuildid} @@ -527,7 +527,7 @@ if [ $# -ge 1 ]; then # We can't rely on buildenv for this code path ;; *) - newbuildid=$(resolve ${pb} ${arch} ${branch} ${buildid}) + newbuildid=$(resolve ${pbd} ${arch} ${branch} ${buildid}) if [ -z "${newbuildid}" ]; then echo "Build ID ${buildid} does not exist" exit 1 @@ -539,7 +539,7 @@ if [ $# -ge 1 ]; then builddir=$(realpath ${pbab}/builds/${buildid}/) - buildenv ${pb} ${arch} ${branch} ${builddir} + buildenv ${pbd} ${arch} ${branch} ${builddir} ;; esac fi Modified: projects/portbuild/scripts/buildenv ============================================================================== --- projects/portbuild/scripts/buildenv Thu Apr 28 09:09:34 2011 (r221158) +++ projects/portbuild/scripts/buildenv Thu Apr 28 11:06:14 2011 (r221159) @@ -40,7 +40,7 @@ validate_env() { # only used on server side resolve() { - pb=$1 + pbd=$1 arch=$2 branch=$3 buildid=$4 @@ -48,7 +48,7 @@ resolve() { # Resolve a possibly symlinked buildid (e.g. "latest") to the # underlying physical directory - pbab=${pb}/${arch}/${branch} + pbab=${pbd}/${arch}/${branch} builddir=${pbab}/builds/${buildid}/ if [ ! -d ${builddir} ]; then @@ -83,7 +83,7 @@ export_src_metadata() { # establish commonly-used environment variables (server-side) # buildenv () { - pb=$1 + pbd=$1 arch=$2 branch=$3 builddir=$4 @@ -139,7 +139,7 @@ buildenv () { fi # probably only used in mkbindist - export __MAKE_CONF=${pb}/${arch}/make.conf + export __MAKE_CONF=${pbd}/${arch}/make.conf } # Modified: projects/portbuild/scripts/buildfailure ============================================================================== --- projects/portbuild/scripts/buildfailure Thu Apr 28 09:09:34 2011 (r221158) +++ projects/portbuild/scripts/buildfailure Thu Apr 28 11:06:14 2011 (r221159) @@ -9,7 +9,8 @@ cleanup() { } # configurable variables -pb=/var/portbuild +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} +pbd=${PORTBUILD_DATA:-/var/portbuild} usage () { echo "usage: buildfailure arch branch buildid pkgname" @@ -26,14 +27,14 @@ buildid=$3 pkgname=$4 shift 4 -builddir=${pb}/${arch}/${branch}/builds/${buildid} +builddir=${pbd}/${arch}/${branch}/builds/${buildid} -. ${pb}/conf/server.conf -. ${pb}/conf/common.conf -. ${pb}/${arch}/portbuild.conf -. ${pb}/scripts/buildenv +. ${pbc}/conf/server.conf +. ${pbc}/conf/common.conf +. ${pbd}/${arch}/portbuild.conf +. ${pbc}/scripts/buildenv -buildenv ${pb} ${arch} ${branch} ${builddir} +buildenv ${pbd} ${arch} ${branch} ${builddir} # Don't pick up installed packages from the host export LOCALBASE=/nonexistentlocal @@ -43,7 +44,7 @@ index=${PORTSDIR}/${INDEXFILE} portloc=$(grep "^$pkgname|" ${index} | cut -f 2 -d \| | sed s,/usr/ports/,,) pkgbase=$(cd ${PORTSDIR}/${portloc}/ && make -V PKGBASE) -cd ${pb}/${arch}/${branch} +cd ${pbd}/${arch}/${branch} entry=$(grep "^${portloc}|" failure) date=$(date +%s) IFS='|' @@ -58,7 +59,7 @@ else (echo "${portloc}|${pkgbase}|${pkgname}|${date}|${date}|1" >> failure) || cleanup fi -link=${pb}/${arch}/${branch}/latest/${portloc} +link=${pbd}/${arch}/${branch}/latest/${portloc} mkdir -p $(dirname ${link}) errorloc=$(realpath ${builddir}/errors/${pkgname}.log) ln -sf ${errorloc} ${link} Modified: projects/portbuild/scripts/buildproxy ============================================================================== --- projects/portbuild/scripts/buildproxy Thu Apr 28 09:09:34 2011 (r221158) +++ projects/portbuild/scripts/buildproxy Thu Apr 28 11:06:14 2011 (r221159) @@ -6,12 +6,16 @@ import sys, socket, os, commands -sys.path.insert(0, '/var/portbuild/lib/python') +pbc = os.getenv('PORTBUILD_CHECKOUT') \ + if os.getenv('PORTBUILD_CHECKOUT') else "/var/portbuild" +pbd = os.getenv('PORTBUILD_DATA') \ + if os.getenv('PORTBUILD_DATA') else "/var/portbuild" + +sys.path.insert(0, '%s/lib/python' % pbc) from freebsd import * from freebsd_config import * -CONFIG_DIR="/var/portbuild" CONFIG_SUBDIR="conf" CONFIG_FILENAME="server.conf" @@ -48,11 +52,11 @@ def process(cmd, sockfile): if not j.isalnum() and not j in "-_.": return (254, "Illegal characters in input") - (status, out) = commands.getstatusoutput("/var/portbuild/scripts/build %s" % " ".join(cmd[1:])) + (status, out) = commands.getstatusoutput("%s/scripts/build %s" % (pbc, " ".join(cmd[1:]))) return (status, out) -config = getConfig( CONFIG_DIR, CONFIG_SUBDIR, CONFIG_FILENAME ) +config = getConfig( pbc, CONFIG_SUBDIR, CONFIG_FILENAME ) BUILDPROXY_SOCKET_FILE = config.get( 'BUILDPROXY_SOCKET_FILE' ) if os.path.exists(BUILDPROXY_SOCKET_FILE): Modified: projects/portbuild/scripts/buildproxy-client ============================================================================== --- projects/portbuild/scripts/buildproxy-client Thu Apr 28 09:09:34 2011 (r221158) +++ projects/portbuild/scripts/buildproxy-client Thu Apr 28 11:06:14 2011 (r221159) @@ -5,16 +5,20 @@ import sys, socket, os, commands -sys.path.insert(0, '/var/portbuild/lib/python') +pbc = os.getenv('PORTBUILD_CHECKOUT') \ + if os.getenv('PORTBUILD_CHECKOUT') else "/var/portbuild" +pbd = os.getenv('PORTBUILD_DATA') \ + if os.getenv('PORTBUILD_DATA') else "/var/portbuild" + +sys.path.insert(0, '%s/lib/python' % pbc) from freebsd import * from freebsd_config import * -CONFIG_DIR="/var/portbuild" CONFIG_SUBDIR="conf" CONFIG_FILENAME="server.conf" -config = getConfig( CONFIG_DIR, CONFIG_SUBDIR, CONFIG_FILENAME ) +config = getConfig( pbc, CONFIG_SUBDIR, CONFIG_FILENAME ) BUILDPROXY_SOCKET_FILE = config.get( 'BUILDPROXY_SOCKET_FILE' ) try: Modified: projects/portbuild/scripts/buildsuccess ============================================================================== --- projects/portbuild/scripts/buildsuccess Thu Apr 28 09:09:34 2011 (r221158) +++ projects/portbuild/scripts/buildsuccess Thu Apr 28 11:06:14 2011 (r221159) @@ -3,7 +3,8 @@ # buildsuccess <arch> <branch> <buildid> <pkgname> # configurable variables -pb=/var/portbuild +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} +pbd=${PORTBUILD_DATA:-/var/portbuild} usage () { echo "usage: buildsuccess arch branch buildid pkgname" @@ -20,14 +21,14 @@ buildid=$3 pkgname=$4 shift 4 -builddir=${pb}/${arch}/${branch}/builds/${buildid} +builddir=${pbd}/${arch}/${branch}/builds/${buildid} -. ${pb}/conf/server.conf -. ${pb}/conf/common.conf -. ${pb}/${arch}/portbuild.conf -. ${pb}/scripts/buildenv +. ${pbc}/conf/server.conf +. ${pbc}/conf/common.conf +. ${pbd}/${arch}/portbuild.conf +. ${pbc}/scripts/buildenv -buildenv ${pb} ${arch} ${branch} ${builddir} +buildenv ${pbd} ${arch} ${branch} ${builddir} # Don't pick up installed packages from the host export LOCALBASE=/nonexistentlocal @@ -36,13 +37,13 @@ index=${PORTSDIR}/${INDEXFILE} portloc=$(grep "^$pkgname|" ${index} | cut -f 2 -d \| | sed s,/usr/ports/,,) -cd ${pb}/${arch}/${branch} +cd ${pbd}/${arch}/${branch} if grep -q "^${portloc}|" failure; then echo | mail -s "${pkgname} now builds on ${arch} ${branch}" ${mailto} grep -v "^${portloc}|" failure > failure.new mv failure.new failure fi -if [ -L ${pb}/${arch}/${branch}/latest/${portloc} ]; then - rm -f ${pb}/${arch}/${branch}/latest/${portloc} +if [ -L ${pbd}/${arch}/${branch}/latest/${portloc} ]; then + rm -f ${pbd}/${arch}/${branch}/latest/${portloc} fi Modified: projects/portbuild/scripts/claim-chroot ============================================================================== --- projects/portbuild/scripts/claim-chroot Thu Apr 28 09:09:34 2011 (r221158) +++ projects/portbuild/scripts/claim-chroot Thu Apr 28 11:06:14 2011 (r221159) @@ -18,7 +18,7 @@ # could use a lockf lock, and be able to tell if the setup process was # still running or died prematurely -pb=/var/portbuild +pbd=${PORTBUILD_DATA:-/var/portbuild} usage () { echo "usage: claim-chroot arch branch buildid" @@ -48,7 +48,7 @@ fi # provide mutual exclusion against other callers, since we only want # one of them to perform setup -builddir=${pb}/${arch}/${branch}/builds/${buildid} +builddir=${pbd}/${arch}/${branch}/builds/${buildid} # Is the build environment populated? Again we only want a single # instance to gain setup rights if not. @@ -64,10 +64,10 @@ if [ ! -f ${builddir}/.ready ]; then exit 1 fi -. ${pb}/${arch}/client.conf -. ${pb}/${arch}/common.conf -. ${pb}/${arch}/portbuild.conf -. ${pb}/${arch}/portbuild.$(hostname) +. ${pbd}/${arch}/client.conf +. ${pbd}/${arch}/common.conf +. ${pbd}/${arch}/portbuild.conf +. ${pbd}/${arch}/portbuild.$(hostname) buildroot=${scratchdir} pkgname=${pkgname%.${PKGSUFFIX}} @@ -109,7 +109,7 @@ for dir in ${chrootdir}/*; do mkdir ${dir}/used 2>/dev/null || continue touch ${dir}/used/${pkgname} if [ -f ${dir}/.dirty ]; then - ${pb}/scripts/clean-chroot ${arch} ${branch} ${buildid} ${dir} 2 >/dev/null 2>/dev/null & + ${builddir}/scripts/clean-chroot ${arch} ${branch} ${buildid} ${dir} 2 >/dev/null 2>/dev/null & continue fi found=1 Modified: projects/portbuild/scripts/clean-chroot ============================================================================== --- projects/portbuild/scripts/clean-chroot Thu Apr 28 09:09:34 2011 (r221158) +++ projects/portbuild/scripts/clean-chroot Thu Apr 28 11:06:14 2011 (r221159) @@ -42,12 +42,12 @@ buildid=$3 chroot=$4 clean=$5 -pb=/var/portbuild +pbd=${PORTBUILD_DATA:-/var/portbuild} -. ${pb}/${arch}/client.conf -. ${pb}/${arch}/common.conf -. ${pb}/${arch}/portbuild.conf -. ${pb}/${arch}/portbuild.$(hostname) +. ${pbd}/${arch}/client.conf +. ${pbd}/${arch}/common.conf +. ${pbd}/${arch}/portbuild.conf +. ${pbd}/${arch}/portbuild.$(hostname) # directories to clean cleandirs="${LOCALBASE} /compat /var/db/pkg" Modified: projects/portbuild/scripts/cleanup-chroots ============================================================================== --- projects/portbuild/scripts/cleanup-chroots Thu Apr 28 09:09:34 2011 (r221158) +++ projects/portbuild/scripts/cleanup-chroots Thu Apr 28 11:06:14 2011 (r221159) @@ -7,7 +7,7 @@ # port builds that have timed out or shut down uncleanly) and prunes # them to reclaim space. -pb=/var/portbuild +pbd=${PORTBUILD_DATA:-/var/portbuild} kill_procs() { @@ -44,9 +44,9 @@ cleanup_mount() { # note: uname is not being overridden (should not need client.conf here) arch=$(uname -m) -. ${pb}/${arch}/client.conf -. ${pb}/${arch}/portbuild.conf -. ${pb}/${arch}/portbuild.$(hostname) +. ${pbd}/${arch}/client.conf +. ${pbd}/${arch}/portbuild.conf +. ${pbd}/${arch}/portbuild.$(hostname) if [ "${use_zfs}" = "1" ]; then old=$(find ${scratchdir}/*/*/* -prune -mmin +60 2> /dev/null) Modified: projects/portbuild/scripts/client-metrics ============================================================================== --- projects/portbuild/scripts/client-metrics Thu Apr 28 09:09:34 2011 (r221158) +++ projects/portbuild/scripts/client-metrics Thu Apr 28 11:06:14 2011 (r221159) @@ -6,7 +6,7 @@ # - max vnodes # - number of packages built in the past hour -pb=/var/portbuild +pbd=${PORTBUILD_DATA:-/var/portbuild} arch=$(uname -m) me=$(hostname) @@ -16,10 +16,10 @@ export PATH=/sbin:/bin:/usr/sbin:/usr/bi gmetric --name="maxvnodes" --value=`sysctl -n kern.maxvnodes` --tmax=120 --dmax=0 --type=uint32 --units="# vnodes" gmetric --name="vnodes" --value=`sysctl -n vfs.numvnodes` --tmax=120 --dmax=0 --type=uint32 --units="# vnodes" -if [ -f ${pb}/${arch}/portbuild.conf -a -f ${pb}/${arch}/portbuild.${me} ]; then - . ${pb}/${arch}/client.conf - . ${pb}/${arch}/portbuild.conf - . ${pb}/${arch}/portbuild.${me} +if [ -f ${pbd}/${arch}/portbuild.conf -a -f ${pbd}/${arch}/portbuild.${me} ]; then + . ${pbd}/${arch}/client.conf + . ${pbd}/${arch}/portbuild.conf + . ${pbd}/${arch}/portbuild.${me} else exit 1 fi Modified: projects/portbuild/scripts/cpdistfiles ============================================================================== --- projects/portbuild/scripts/cpdistfiles Thu Apr 28 09:09:34 2011 (r221158) +++ projects/portbuild/scripts/cpdistfiles Thu Apr 28 11:06:14 2011 (r221159) @@ -6,14 +6,16 @@ if [ $# -lt 3 ]; then fi # configurable variables -pb=/var/portbuild -. ${pb}/conf/server.conf +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} +pbd=${PORTBUILD_DATA:-/var/portbuild} + +. ${pbc}/conf/server.conf arch=$1 branch=$2 buildid=$3 -builddir=${pb}/${arch}/${branch}/builds/${buildid} +builddir=${pbd}/${arch}/${branch}/builds/${buildid} yesreally=0 dryrun=-n Modified: projects/portbuild/scripts/cppackages ============================================================================== --- projects/portbuild/scripts/cppackages Thu Apr 28 09:09:34 2011 (r221158) +++ projects/portbuild/scripts/cppackages Thu Apr 28 11:06:14 2011 (r221159) @@ -1,11 +1,11 @@ #!/bin/sh # configurable variables -pb=/var/portbuild +pbd=${PORTBUILD_DATA:-/var/portbuild} -. ${pb}/portbuild.conf +. ${pbd}/portbuild.conf -lock=${pb}/cppackages.lock +lock=${pbd}/cppackages.lock unset DISPLAY Modified: projects/portbuild/scripts/do-cleanup-chroots ============================================================================== --- projects/portbuild/scripts/do-cleanup-chroots Thu Apr 28 09:09:34 2011 (r221158) +++ projects/portbuild/scripts/do-cleanup-chroots Thu Apr 28 11:06:14 2011 (r221159) @@ -2,4 +2,7 @@ # # clean up stale chroots on all client machines. MCL 20081216. # -/var/portbuild/scripts/allgohans all -q /var/portbuild/scripts/cleanup-chroots + +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} + +${pbc}/scripts/allgohans all -q -scp ${pbc}/scripts/cleanup-chroots Modified: projects/portbuild/scripts/docppackages ============================================================================== --- projects/portbuild/scripts/docppackages Thu Apr 28 09:09:34 2011 (r221158) +++ projects/portbuild/scripts/docppackages Thu Apr 28 11:06:14 2011 (r221159) @@ -1,7 +1,11 @@ #!/bin/sh + if [ $# != 1 ]; then echo "usage: $0 branch" exit 1 fi -echo /var/portbuild/scripts/cppackages $1 | at + 1 minute + +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} + +echo ${pbc}/scripts/cppackages $1 | at + 1 minute atq Modified: projects/portbuild/scripts/dodistfiles ============================================================================== --- projects/portbuild/scripts/dodistfiles Thu Apr 28 09:09:34 2011 (r221158) +++ projects/portbuild/scripts/dodistfiles Thu Apr 28 11:06:14 2011 (r221159) @@ -30,8 +30,9 @@ arch=$1 branch=$2 buildid=$3 -pb=/var/portbuild -builddir=${pb}/${arch}/${branch}/builds/${buildid} +pbd=${PORTBUILD_DATA:-/var/portbuild} + +builddir=${pbd}/${arch}/${branch}/builds/${buildid} distdir=${builddir}/distfiles Modified: projects/portbuild/scripts/dologs ============================================================================== --- projects/portbuild/scripts/dologs Thu Apr 28 09:09:34 2011 (r221158) +++ projects/portbuild/scripts/dologs Thu Apr 28 11:06:14 2011 (r221159) @@ -4,21 +4,22 @@ arch=$1 versions="latest full" -pb=/var/portbuild +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} +pbd=${PORTBUILD_DATA:-/var/portbuild} -. ${pb}/conf/server.conf -. ${pb}/${arch}/portbuild.conf +. ${pbc}/conf/server.conf +. ${pbd}/${arch}/portbuild.conf -home=${pb}/errorlogs -scripts=${pb}/scripts +home=${pbd}/errorlogs +scripts=${pbc}/scripts -branches=`cd ${pb}/${arch} && ls -d [1-9]* 2> /dev/null` +branches=`cd ${pbd}/${arch} && ls -d [1-9]* 2> /dev/null` for version in ${versions}; do for branch in ${branches}; do dir=$home/$arch-$branch-$version test -d $dir && cd $dir && ${scripts}/processlogs ${arch} - dir=${pb}/${arch}/${branch} + dir=${pbd}/${arch}/${branch} test -d $dir && cd $dir && ${scripts}/processfail ${arch} ${branch} done for branch in ${branches}; do Modified: projects/portbuild/scripts/dopackages ============================================================================== --- projects/portbuild/scripts/dopackages Thu Apr 28 09:09:34 2011 (r221158) +++ projects/portbuild/scripts/dopackages Thu Apr 28 11:06:14 2011 (r221159) @@ -4,9 +4,10 @@ # main server-side script to run a package build # configurable variables -pb=/var/portbuild +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} +pbd=${PORTBUILD_DATA:-/var/portbuild} -PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:${pb}/scripts +PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:${pbc}/scripts # writable by portmgr umask 002 @@ -52,26 +53,26 @@ buildid=$3 datestamp=$4 shift 4 -. ${pb}/conf/server.conf -. ${pb}/conf/common.conf -. ${pb}/scripts/buildenv +. ${pbc}/conf/server.conf +. ${pbc}/conf/common.conf +. ${pbc}/scripts/buildenv validate_env ${arch} ${branch} || usage # XXX MCL too early to do this here. -buildid=$(resolve ${pb} ${arch} ${branch} ${buildid}) +buildid=$(resolve ${pbd} ${arch} ${branch} ${buildid}) if [ -z "${buildid}" ]; then echo "Invalid build ID ${buildid}" exit 1 fi -if [ -f ${pb}/${arch}/portbuild.conf ]; then - . ${pb}/conf/server.conf - . ${pb}/${arch}/portbuild.conf +if [ -f ${pbd}/${arch}/portbuild.conf ]; then + . ${pbc}/conf/server.conf + . ${pbd}/${arch}/portbuild.conf else usage fi -pbab=${pb}/${arch}/${branch} +pbab=${pbd}/${arch}/${branch} trap "exit 1" 1 2 3 9 10 11 15 @@ -97,7 +98,7 @@ portstar() { # usage: makeindex pb arch branch builddir [target] # note: can take ~24 minutes! makeindex () { - pb=$1 + pbc=$1 arch=$2 branch=$3 buildid=$4 @@ -109,7 +110,7 @@ makeindex () { echo "generating index" echo "================================================" echo "index generation started at $(date)" - ${pb}/scripts/makeindex ${arch} ${branch} ${buildid} ${target} || return 1 + ${pbc}/scripts/makeindex ${arch} ${branch} ${buildid} ${target} || return 1 echo "index generation ended at $(date)" echo $(wc -l ${INDEXFILE} | awk '{print $1}') "lines in INDEX" @@ -142,7 +143,7 @@ checkindex () { # usage: makeduds pb arch branch builddir [target] # note: can take ~21 minutes! makeduds () { - pb=$1 + pbc=$1 arch=$2 branch=$3 buildid=$4 @@ -157,7 +158,7 @@ makeduds () { if [ -e ${builddir}/duds ]; then cp -p ${builddir}/duds ${builddir}/duds.old fi - if ! ${pb}/scripts/makeduds ${arch} ${branch} ${buildid} ${target}; then + if ! ${pbc}/scripts/makeduds ${arch} ${branch} ${buildid} ${target}; then echo "error(s) detected, exiting script at $(date). Failed duds list was:" cat ${builddir}/duds mailexit 1 @@ -176,7 +177,7 @@ makeduds () { # usage: restrictedlist pb arch branch builddir [target] # note: can take ~25 minutes! restrictedlist () { - pb=$1 + pbc=$1 arch=$2 branch=$3 buildid=$4 @@ -188,7 +189,7 @@ restrictedlist () { echo "creating restricted list" echo "================================================" echo "restricted list generation started at $(date)" - ${pb}/scripts/makerestr ${arch} ${branch} ${buildid} ${target} || return 1 + ${pbc}/scripts/makerestr ${arch} ${branch} ${buildid} ${target} || return 1 echo "restricted list generation ended at $(date)" echo $(grep -c '^#' ${builddir}/restricted.sh) "ports in ${builddir}/restricted.sh" } @@ -196,7 +197,7 @@ restrictedlist () { # usage: cdromlist pb arch branch builddir # note: can take ~48 minutes! cdromlist () { - pb=$1 + pbc=$1 arch=$2 branch=$3 builddir=$4 @@ -218,7 +219,7 @@ cdromlist () { # XXX Should generate these as the packages are copied in, instead of all at once at the end # usage: generatemd5 pb arch branch builddir generatemd5 () { - pb=$1 + pbc=$1 arch=$2 branch=$3 builddir=$4 @@ -231,7 +232,7 @@ generatemd5 () { dobuild() { - pb=$1 + pbc=$1 arch=$2 branch=$3 builddir=$4 @@ -241,7 +242,7 @@ dobuild() { echo "================================================" echo "started at $(date)" start=$(date +%s) - ${QMANAGER_PATH}/packagebuild ${arch} ${branch} ${buildid} > ${builddir}/${journalname} 2>&1 < /dev/null + ${pbc}/qmanager/packagebuild ${arch} ${branch} ${buildid} > ${builddir}/${journalname} 2>&1 < /dev/null result=$? if [ $result -ne 0 ]; then echo "ERROR: packagebuild ${arch} ${branch} ${buildid} failed: see ${builddir}/${journalname} for details" @@ -422,7 +423,7 @@ if [ ! -d ${builddir} ]; then fi # Set up our environment variables *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201104281106.p3SB6Fwc020072>