From owner-svn-src-projects@FreeBSD.ORG Fri Apr 22 19:03:33 2011 Return-Path: Delivered-To: svn-src-projects@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id AB155106566C; Fri, 22 Apr 2011 19:03:33 +0000 (UTC) (envelope-from flz@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 98FA68FC16; Fri, 22 Apr 2011 19:03:33 +0000 (UTC) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id p3MJ3XL0054440; Fri, 22 Apr 2011 19:03:33 GMT (envelope-from flz@svn.freebsd.org) Received: (from flz@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id p3MJ3XDF054432; Fri, 22 Apr 2011 19:03:33 GMT (envelope-from flz@svn.freebsd.org) Message-Id: <201104221903.p3MJ3XDF054432@svn.freebsd.org> From: Florent Thoumie Date: Fri, 22 Apr 2011 19:03:33 +0000 (UTC) To: src-committers@freebsd.org, svn-src-projects@freebsd.org X-SVN-Group: projects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r220956 - in projects/portbuild: . conf errorlogs scripts X-BeenThere: svn-src-projects@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the src " projects" tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 22 Apr 2011 19:03:33 -0000 Author: flz Date: Fri Apr 22 19:03:33 2011 New Revision: 220956 URL: http://svn.freebsd.org/changeset/base/220956 Log: Import portbuild from pcvs. Discussed with: portmgr (linimon, pav, self) Added: projects/portbuild/ projects/portbuild/conf/ projects/portbuild/conf/README.dotunnel projects/portbuild/conf/client.conf projects/portbuild/conf/common.conf projects/portbuild/conf/make.conf projects/portbuild/conf/server.conf projects/portbuild/errorlogs/ projects/portbuild/errorlogs/index.shtml projects/portbuild/scripts/ projects/portbuild/scripts/allgohans (contents, props changed) projects/portbuild/scripts/allgohans.safe (contents, props changed) projects/portbuild/scripts/bothlogs (contents, props changed) projects/portbuild/scripts/build (contents, props changed) projects/portbuild/scripts/buildenv projects/portbuild/scripts/buildfailure (contents, props changed) projects/portbuild/scripts/buildproxy (contents, props changed) projects/portbuild/scripts/buildproxy-client (contents, props changed) projects/portbuild/scripts/buildscript (contents, props changed) projects/portbuild/scripts/buildsuccess (contents, props changed) projects/portbuild/scripts/checkmachines (contents, props changed) projects/portbuild/scripts/checkmachines.sh (contents, props changed) projects/portbuild/scripts/chopindex (contents, props changed) projects/portbuild/scripts/claim-chroot (contents, props changed) projects/portbuild/scripts/clean-chroot (contents, props changed) projects/portbuild/scripts/cleanup-chroots (contents, props changed) projects/portbuild/scripts/client-metrics (contents, props changed) projects/portbuild/scripts/comparelogs (contents, props changed) projects/portbuild/scripts/cpdistfiles (contents, props changed) projects/portbuild/scripts/cppackages (contents, props changed) projects/portbuild/scripts/do-cleanup-chroots (contents, props changed) projects/portbuild/scripts/docppackages (contents, props changed) projects/portbuild/scripts/dodistfiles (contents, props changed) projects/portbuild/scripts/dologs (contents, props changed) projects/portbuild/scripts/dopackages (contents, props changed) projects/portbuild/scripts/dopackages.wrapper (contents, props changed) projects/portbuild/scripts/dopackages2 (contents, props changed) projects/portbuild/scripts/dopackagestats (contents, props changed) projects/portbuild/scripts/dosetupnode (contents, props changed) projects/portbuild/scripts/flushsquid (contents, props changed) projects/portbuild/scripts/keeprestr (contents, props changed) projects/portbuild/scripts/makeduds (contents, props changed) projects/portbuild/scripts/makeindex (contents, props changed) projects/portbuild/scripts/makerestr (contents, props changed) projects/portbuild/scripts/makeworld (contents, props changed) projects/portbuild/scripts/mkbindist (contents, props changed) projects/portbuild/scripts/nukesquid (contents, props changed) projects/portbuild/scripts/packagebuild (contents, props changed) projects/portbuild/scripts/pdispatch (contents, props changed) projects/portbuild/scripts/pnohang.c projects/portbuild/scripts/pollmachine (contents, props changed) projects/portbuild/scripts/portbuild (contents, props changed) projects/portbuild/scripts/processfail (contents, props changed) projects/portbuild/scripts/processlogs (contents, props changed) projects/portbuild/scripts/processlogs2 (contents, props changed) projects/portbuild/scripts/processonelog (contents, props changed) projects/portbuild/scripts/prunebad projects/portbuild/scripts/prunefailure (contents, props changed) projects/portbuild/scripts/prunepkgs (contents, props changed) projects/portbuild/scripts/ptimeout.c projects/portbuild/scripts/releasemachine (contents, props changed) projects/portbuild/scripts/reportload (contents, props changed) projects/portbuild/scripts/reportload.sh (contents, props changed) projects/portbuild/scripts/retcodes projects/portbuild/scripts/setupnode (contents, props changed) projects/portbuild/scripts/showrunning (contents, props changed) projects/portbuild/scripts/stats (contents, props changed) projects/portbuild/scripts/straslivy.py (contents, props changed) projects/portbuild/scripts/updatesnap (contents, props changed) projects/portbuild/scripts/updatesnap.ports (contents, props changed) projects/portbuild/scripts/zbackup (contents, props changed) projects/portbuild/scripts/zclient (contents, props changed) projects/portbuild/scripts/zexpire Added: projects/portbuild/conf/README.dotunnel ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ projects/portbuild/conf/README.dotunnel Fri Apr 22 19:03:33 2011 (r220956) @@ -0,0 +1,32 @@ +Various package build nodes require us to set up TCP tunnels to talk +to them. (Some systems don't pass certain ports; some systems have +firewalls; some systems have multiple nodes on one IP address.) + +These have always been hardcoded in crontab lines of the form "while true; +do ; done". Other than the magic hardcoding, +there's a problem with this. When the tunnel command exits, such as if +the host suddenly becoming unreachable, it doesn't send mail -- instead +it just accumulates a huge file in /var/spool/clientmqueue which never +gets sent. To add insult to injury, /var is on the root partition on +pointyhat. + +To cure these problems, we now have + + /var/portbuild/conf//dotunnel.XXX + +where XXX corresponds to one line in the old crontab. Each script sets +up one tunnel, sends mail to the user(s) in + + /var/portbuild//portbuild.conf + +once the command exits, and then sleeps. + +Why not put it in /var/portbuild/ you ask? That directory is +propogated to all nodes for that arch. This would be a security leak. +The intention is that none of the dotunnel files will be checked into +CVS. + +Final note: each script figures out which arch it is for by fiddling +with its $0, so invoke it with its full pathname. + +mcl Added: projects/portbuild/conf/client.conf ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ projects/portbuild/conf/client.conf Fri Apr 22 19:03:33 2011 (r220956) @@ -0,0 +1,41 @@ +# +# client-side definitions (used in /var/portbuild/scripts/portbuild) +# +# $FreeBSD: ports/Tools/portbuild/conf/client.conf,v 1.4 2011/01/23 02:34:58 linimon Exp $ +# + +# +# items to be customized per each package build master +# + +# for nodes where disconnected=0, the NFS host they should mount ports/ +# and src/ from +CLIENT_NFS_MASTER=pointyhat.FreeBSD.org + +# fallback for distfiles (see make.conf in bindist-*.tar) +CLIENT_BACKUP_FTP_SITE=pointyhat.FreeBSD.org + +# where completed packages get uploaded +CLIENT_UPLOAD_HOST=pointyhat.FreeBSD.org + +# +# items that will most likely be common to all package build masters +# + +CLIENT_DISTDIR=/tmp/distfiles +# XXX renaming this from PACKAGES +CLIENT_PACKAGES_LOCATION=/tmp/packages +CLIENT_SRCBASE=/usr/src +CLIENT_WRKDIRPREFIX=/work + +# wait 2 hours before killing build with no output +CLIENT_BUILD_TIMEOUT=7200 +CLIENT_FTP_TIMEOUT=900 +CLIENT_HTTP_TIMEOUT=900 + +# to prevent runaway processes -- 400 meg file size limit, 2 hours CPU limit +CLIENT_ULIMIT_F=819200 +CLIENT_ULIMIT_T=7200 + +# debugging definitions +CLIENT_MALLOC_OPTIONS=AJ Added: projects/portbuild/conf/common.conf ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ projects/portbuild/conf/common.conf Fri Apr 22 19:03:33 2011 (r220956) @@ -0,0 +1,21 @@ +# +# package building configuration file containing things that are common +# both to the server-side (pointyhat instance) and the client side +# (individual build clients). +# +# original author: linimon +# +# $FreeBSD: ports/Tools/portbuild/conf/common.conf,v 1.1 2010/12/01 02:35:20 linimon Exp $ +# + +# +# top-level package building things. These will probably be common +# to all package build masters. +# + +LOCALBASE=/usr/local +PKGSUFFIX=.tbz + +ARCHS_REQUIRING_AOUT_COMPAT="i386" +ARCHS_REQUIRING_LINPROCFS="amd64 i386" +ARCHS_SUPPORTING_COMPAT_IA32="amd64 i386 ia64" Added: projects/portbuild/conf/make.conf ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ projects/portbuild/conf/make.conf Fri Apr 22 19:03:33 2011 (r220956) @@ -0,0 +1,16 @@ +# DON'T SET PORT VARIABLES UNCONDITIONALLY - THEY NEED TO BE +# OVERRIDABLE BY THE SCRIPTS +USA_RESIDENT?=YES + +# +MASTER_SITE_BACKUP= \ + ftp://ftp-master.freebsd.org/pub/FreeBSD/ports/distfiles/${DIST_SUBDIR}/ +MASTER_SITE_OVERRIDE?= ${MASTER_SITE_BACKUP} +MASTER_SITE_LOCAL= \ + ftp://ftp-master.FreeBSD.org/pub/FreeBSD/ports/local-distfiles/%SUBDIR%/ + +NO_PROFILE=true +MAKE_KERBEROS5= yes +SENDMAIL_CF= freefall.cf +BOOT_PXELDR_PROBE_KEYBOARD= true +ENABLE_SUID_K5SU=yes Added: projects/portbuild/conf/server.conf ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ projects/portbuild/conf/server.conf Fri Apr 22 19:03:33 2011 (r220956) @@ -0,0 +1,105 @@ +# +# package building configuration file (server-side). Specific to each +# pointyhat instance. +# +# note: readable by both Python and /bin/sh files. HOWEVER, there is no +# code yet to do the {}-style shell expansions in the Python scripts. +# Beware! +# +# original author: linimon +# +# $FreeBSD: ports/Tools/portbuild/conf/server.conf,v 1.7 2011/04/22 18:32:13 linimon Exp $ +# + +# +# top-level package building things +# + +SUPPORTED_ARCHS="amd64 i386 ia64 powerpc sparc64" + +SRC_BRANCHES="7 8 9" +SRC_BRANCHES_PATTERN="^[0-9]*" +SRC_BRANCH_7_TAG=RELENG_7_3 +SRC_BRANCH_8_TAG=RELENG_8_1 +SRC_BRANCH_9_TAG=. + +DEFAULT_LINUX_OSRELEASE="2.6.16" + +# +# directory management definitions +# + +ZFS_VOLUME=a +ZFS_MOUNTPOINT=/a + +SNAP_DIRECTORY=snap +SNAP_PORTS_DIRECTORY=${SNAP_DIRECTORY}/ports-head +SNAP_SRC_DIRECTORY_PREFIX=${SNAP_DIRECTORY}/src- + +SUPFILE_DIRECTORY=/home/portmgr/sup +PORTS_MASTER_SUPFILE=${SUPFILE_DIRECTORY}/ports-master-supfile +SRC_MASTER_SUPFILE=${SUPFILE_DIRECTORY}/src-master-supfile +PORTS_SUPFILE=${SUPFILE_DIRECTORY}/ports-supfile +SRC_SUPFILE=${SUPFILE_DIRECTORY}/src-supfile + +WORLDDIR=${ZFS_MOUNTPOINT}/chroot/ + +# XXX TODO (note: Python script, so avoid {}) +#zbackup a/nfs a/local a/portbuild/* /dumpster +#zexpire a/nfs a/local a/portbuild/* a/snap/* + +# +# buildproxy definitions (note: Python script, so avoid {}) +# + +BUILDPROXY_SOCKET_FILE=/tmp/.build + +# +# pdispatch definitions +# + +# reflect hardwiring in 'buildscript' phase 1 and also 'processonelog' and +# 'processlogs2'. You probably do not want to change this! +PDISPATCH_HDRLENGTH=6 + +# number of lines of log to email +PDISPATCH_LOGLENGTH=1000 + +# wait 100 hours maximum +PDISPATCH_TIMEOUT=360000 + +# +# qmanager definitions (note: Python script, so avoid {}) +# + +QMANAGER_PATH=/var/portbuild/evil/qmanager +QMANAGER_DATABASE_FILE=qdb.sl3 +QMANAGER_SOCKET_FILE=/tmp/.qmgr + +QMANAGER_PRIORITY_PACKAGES="openoffice kde-3" + +# maximum number of times to build an individual job +QMANAGER_MAX_JOB_ATTEMPTS=5 + +# attempt to limit the amount time (and email) on botched runs +QMANAGER_RUNAWAY_PERCENTAGE=0.75 +QMANAGER_RUNAWAY_THRESHOLD=100 + +# +# upload definitions (see 'cpdistfiles') +# + +UPLOAD_DIRECTORY="w/ports/distfiles/" +UPLOAD_TARGET="ftp-master.FreeBSD.org" +UPLOAD_USER="portmgr" + +# +# user-visible things +# + +MASTER_URL="pointyhat.FreeBSD.org" + +# +# www definitions (see processfail) +# +WWW_DIRECTORY=/usr/local/www/data/ Added: projects/portbuild/errorlogs/index.shtml ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ projects/portbuild/errorlogs/index.shtml Fri Apr 22 19:03:33 2011 (r220956) @@ -0,0 +1,984 @@ + + + + +FreeBSD Package building logs and errors + + +

FreeBSD Package building logs and errors

+ +

FreeBSD maintains a build farm (the "pointyhat cluster") that is used +to build all packages across all supported architectures and major releases. +This page contains the build logs and errors for all the ports built by +the cluster. See the notes for additional information, +or the types of package errors detected.

+ +

+Port cross-reference summaries +

+

These live on +portsmon.FreeBSD.org.

+
+ +
* Build errors +by portname; +by portname vs. build environment; +by error type vs. build environment
+
* Problem Reports +for existing ports; +for new ports; +for the ports framework; +for repocopies requested; +for unknown
+
* Build Errors and +Problem Reports +by portname; +for one maintainer; +for broken ports; +for deprecated ports; +for forbidden ports
+
* Everything about +one port
+
* The +complete list of all reports
+
+ +

+New build failures +

+

Check here to find the most recent error log from your port.

+ + + +
+
+* New build +failures on 7.x-stable: +amd64 +i386 +ia64 +sparc64 +
+* New build +failures on 8.x-stable: +amd64 +i386 +ia64 +powerpc +sparc64 +
+* New build +failures on 9.x-current: +amd64 +i386 +ia64 +powerpc +sparc64 +
+
+* New build failures on 7.x-stable with experimental port patches: +amd64 +i386 +
+* New build failures on 8.x-stable with experimental port patches: +amd64 +i386 +
+* New build failures on 9.x-current with experimental port patches: +amd64 +i386 +
+
+
+ +

+Error logs +

+ +
+
+* Previous run +on 7.x-stable: +amd64 +i386 +ia64 +sparc64 +
+* Current run +on 7.x-stable: +amd64 +i386 +ia64 +sparc64 +
+* Previous run +on 8.x-stable: +amd64 +i386 +ia64 +powerpc +sparc64 +
+* Current run +on 8.x-stable: +amd64 +i386 +ia64 +powerpc +sparc64 +
+* Previous run +on 9.x-current: +amd64 +i386 +ia64 +powerpc +sparc64 +
+* Current run +on 9.x-current: +amd64 +i386 +ia64 +powerpc +sparc64 +
+
+* Previous run +on 7.x-stable with experimental port patches: +amd64 +i386 +
+* Current run +on 7.x-stable with experimental port patches: +amd64 +i386 +
+* Previous run +on 8.x-stable with experimental port patches: +amd64 +i386 +
+* Current run +on 8.x-stable with experimental port patches: +amd64 +i386 +
+* Previous run +on 9.x-current with experimental port patches: +amd64 +i386 +
+* Current run +on 9.x-current with experimental port patches: +amd64 +i386 +
+
+ +

+Build logs (errors and otherwise) +

+ +
+
+*Previous run +on 7.x-stable: +amd64 +i386 +ia64 +sparc64 +
+*Current run on +7.x-stable: +amd64 +i386 +ia64 +sparc64 +
+*Previous run +on 8.x-stable: +amd64 +i386 +ia64 +powerpc +sparc64 +
+*Current run on +8.x-stable: +amd64 +i386 +ia64 +powerpc +sparc64 +
+*Previous run +on 9.x-current: +amd64 +i386 +ia64 +powerpc +sparc64 +
+*Current run on +9.x-current: +amd64 +i386 +ia64 +powerpc +sparc64 +
+
+*Previous run +on 7.x-stable with experimental port patches: +amd64 +i386 +
+*Current run on +7.x-stable with experimental port patches: +amd64 +i386 +
+*Previous run +on 8.x-stable with experimental port patches: +amd64 +i386 +
+*Current run on +8.x-stable with experimental port patches: +amd64 +i386 +
+*Previous run +on 9.x-current with experimental port patches: +amd64 +i386 +
+*Current run on +9.x-current with experimental port patches: +amd64 +i386 +
+
+
+ +

+Packages +

+ +
+
+* Packages from +latest run on 6-stable: +amd64 +i386 +sparc64 +
+* Packages from +latest run on 7-stable: +amd64 +i386 +ia64 +sparc64 +
+* Packages from +latest run on 8-stable: +amd64 +i386 +ia64 +powerpc +sparc64 +
+* Packages from +latest run on 9-current: +amd64 +i386 +ia64 +powerpc +sparc64 +
+
+* Package +building statistics (current state of all package builds) +
+
+ +

+Archive +

+ +
+
* All error +logs: +amd64 +i386 +ia64 +powerpc +sparc64 +
+* All portbuild +logs: +amd64 +i386 +ia64 +powerpc +sparc64 +
+
+ + +

Additional information

+ + +

All of the "Current run" links are possibly in progress and may be +partial, so keep that in mind if there appear to be some missing.

+ +

None of the ports marked IS_INTERACTIVE or +NO_PACKAGE are built any more -- if you have ports that +fall into those categories, assume their packages or distfiles will +never show up in ftp sites or CDROMs. RESTRICTED packages +are built and deleted (using "make +clean-restricted-list"). NO_CDROM packages are built +but deleted (using "make clean-cdrom-list") before being +put on a CDROM.

+ +

See also the types of errors detected.

+ +

Notes on the building process:

+ +
    +
  • Every port is built in its own chroot environment + +, starting with an +empty /usr/local and /usr/X11R6. The dependencies are installed as packages just before +the build. You can see the list of dependencies on the third line +of the log -- the "foo.tgz bar.tgz" stuff are the dependencies. To +make sure that these actually work, DEPENDS_TARGET is set +to "/usr/bin/true"; if you see "/usr/bin/true is up to date" or +some such, that means there is something wrong with the dependency +lines or the packages this port is depending on.
    • + +
  • The build is done on a shared (read-only) /usr/ports +with +WRKDIRPREFIX set to /work. If your +WRKSRC looks funny, that's probably why.
    • + +
  • MASTER_SITE_OVERRIDE is pointing to pointyhat's distfile +dir, and MASTER_SITE_BACKUP is pointing to +ftp-master.freebsd.org or a local mirror. +Please pay attention to + +Emanuel Haupt's distfile checker +for that.
    • + +
  • The ports are built on machines that are mostly running + 9-CURRENT, with some 8.x-STABLE machines. If your port depends on + the result of uname(3) or sysctl to determine the running version of + FreeBSD, change it to use uname(1) instead (the builds use a dummy + uname(1) that reports the target version of FreeBSD), or change it to + use the value of the OSVERSION variable that can be passed in from + the port makefile.
+ + +

Types of package errors detected

+ +

Here is the alphabetical list of current errors +detected by the AI script. Note that this is all just a rough guess -- +it is merely for your aid.

+ +

Key: +

+
(common)The most common + errors.
+
(uncommon)Some less common + errors.
+
(transient)Transient + errors. These may not be your fault.
+
+

+ +

+

+
(common)arch
+ +
The port does not build on a particular architecture, due to +assembler or linker errors. In some easy cases this is due to +not picking up the various ARCH configuration variables +in the Makefile; you'll see this via, e.g., a Sparc make +failing while looking for an i386 subdirectory. For the 64-bit +architectures, a common problem is the assumption many programmers +make that pointers may be cast to and from 32-bit ints. In other cases +the problems run much deeper, in which case ONLY_FOR_ARCHS +may be needed.
+ +
(uncommon)autoconf
+ +
Your port depends on autoconf, but the Makefile +either doesn't have USE_AUTOCONF, or does not use +USE_AUTOCONF_VER correctly.
+ +
(uncommon)autoheader
+ +
Your port depends on autoheader, but the Makefile +cannot find it; set USE_AUTOHEADER.
+ +
(uncommon)automake
+ +
Your port depends on automake, but the Makefile +either doesn't have USE_AUTOMAKE, or does not use +USE_AUTOMAKE_VER correctly.
+ +
(common)bad C++ code
+ +
There is a compiler error which is caused by something specific +to C++.
+ +
(common)compiler error
+ +
There is a C compiler error which is caused by something other +than e.g. "new compiler error".
+ +
(uncommon)CATEGORIES
+ +
The CATEGORIES line in Makefile includes an +invalid category.
+ +
(common)checksum
+ +
The checksum of one or more of the files is incorrect.
+ +
(common)clang
+ +
Your code does not run with the the experimental clang compiler. See +(TBA) +for further information.
+ +
(uncommon)clang bug
+ +
You have tickled a bug in clang itself. See +(TBA) +for further information.
+ +
(transient)cluster
+ +
There was some kind of transient error on the build cluster. It is not your +fault.
+ +
(uncommon)compat6x
+ +
This port needs to depend on a port misc/compat6x.
+ +
(common)configure error
+ +
The port's configure script produced some kind of +error. +(Note: using clang as the ports compiler can also trigger this message.)
+ +
(common)coredump
+ +
Some process in the build chain dropped core. While your port may indeed +be faulty, the process that dropped core should also be fixed.
+ +
(uncommon)cpusetsize
+ +
This port needs to catch up with the cpusetsize sysctl change in 9-CURRENT.
+ +
(common)depend object
+ +
The port is trying to reinstall a dependency that already +exists. This is usually caused by the first field of a +*_DEPENDS line (the obj of +obj:dir[:target]) indicating a file that is not installed +by the dependency, causing it to be rebuilt even though it has +already been added from a package.
+ +
(common)depend package
+ +
There was an error during adding dependencies +from packages. It is the fault of the package being added, not +this port.
+ +
(uncommon)dirent
+ +
The port has not caught up with the change to scandir(3) +as committed in FreeBSD version 800501.
+ +
(transient)disk full
+ +
The disk filled up on the build system. It is not your +fault.
+ +
(uncommon)DISPLAY
+ +
This port requires an X display to build. There is nothing you +can do about it unless you can somehow make it not require an X +connection.
+ +
(common)distinfo update
+ +
The contents of distinfo does not match the list of +distfiles or patchfiles.
+ +
(common)fetch
+ +
One or more of the files could not be fetched.
+ +
(uncommon)fetch timeout
+ +
Your fetch process was killed because it took too long. (More +accurately, it did not produce any output for a long time.) Please +put sites with better connectivity near the beginning of +MASTER_SITES.
+ +
(uncommon)forbidden
+ +
Someone has marked this port as "forbidden", almost always due +to security concerns. See the logfile for more information.
+ +
(uncommon)gcc bug
+ +
You have tickled a bug in gcc itself. See the +GNU bug report documentation +for further information.
+ +
(common)gcc4
+ +
Your code does not run with the latest gcc version +See the wiki page +for further information.
+ +
(common)gmake
+ +
Your code does not run with the latest, incompatible, gmake version +(3.82.)
+ +
(common)install error
+ +
There was an error during installation.
+ +
(common)LIB_DEPENDS
+ +
The LIB_DEPENDS line specifies a library name +incorrectly. This often happens when a port is upgraded and the +shared library version number changes.
+ +
(common)linker error
+ +
There is a linker error which is caused by something other than +those flagged elsewhere. +(Note: using clang as the ports compiler can also trigger this message.)
+ +
(common)makefile
+ +
There is an error in the Makefile, possibly in the default +targets.
+ +
(uncommon)manpage
+ +
There is a manpage listed in a MAN? macro that does not +exist or is not installed in the right place.
+ +
(common)missing header
+ +
There is a missing header file. This is usually caused by +either (1) a missing dependency, or (2) specifying an incorrect +location with -I in the compiler command line.
+ +
(common)mtree
+ +
The port leaves ${PREFIX} in a state that is not +consistent with the mtree definition after pkg_delete. This +usually means some files are missing from PLIST. It could +also mean that your installation scripts create files or +directories not properly deleted by the deinstallation scripts. +Another possibility is that your port is deleting some directories +it is not supposed to, or incorrectly modifying some directory's +permission.
+ +
(uncommon)nested_declaration
+ +
There is a nested declaration in the source code.
+ +
(common)new compiler error
+ +
The newest version of gcc in the base does not like the source code. This is +usually due to stricter C++ type checking or changes in register +allocation policy.
+ +
(transient)NFS
+ +
There was either a temporary NFS error on the build system +(which is not your fault), or the WRKSRC is invalid +(which is your fault).
+ +
(common)patch
+ +
One or more of the patches failed.
+ +
(uncommon)perl
+ +
perl is no longer included by default in the base +system, but your port's configuration process depends on it. While +this change helps avoid having a stale version of perl +in the base system, it also means that many ports now need to include +USE_PERL5.
+ +
(common)PLIST
+ +
There is a missing item in the PLIST. Note that this is +often caused by an earlier error that went undetected. In this case, +you should fix the error and also the build process so it will fail +upon an error instead of continuing, since that makes debugging +that much harder.
+ +
(uncommon)process failed
+ +
The make process terminated unexpectedly, due to +something like a signal 11 or bus error.
+ +
(uncommon)runaway process
+ +
Your make package process was killed because it took +too long. (More accurately, it did not produce any output for a long +time.) It is probably because there is a process spinning in an infinite +loop. Please check the log to determine the exact cause of the +problem.
+ +
(uncommon)segfault
+ +
Some process in the build chain got a segmentation fault.
+ +
(uncommon)sem_wait
+ +
This port needs to catch up with semaphore changes in 9-CURRENT.
+ +
(uncommon)termios
+ +
This port needs to catch up with the termios.h changes in src.
+ +
(uncommon)threads
+ +
There is a linker error which is caused by failing to find one of +the thread libraries.
+ +
(transient)truncated_distfile
+ +
A package node encountered an error during pkg_add. It is not your +fault. Linimon is trying to figure out this problem.
+ +
(uncommon)utmp_x
+ +
This port needs to catch up with the utmp_x.h changes in src.
+ +
(uncommon)WRKDIR
+ +
The port is attempting to change something outside +${WRKDIR}. See handbook +for details.
+ +
(common)??? (unknown)
+ +
The automated script cannot even guess what is wrong with your +port. portmgr tries to keep the processonelog script +reasonably efficient while covering as many errors as possible, but many +errors are not common enough to try to catch.
+
+ +

Here is an alphabetical list of obsolete errors +that used to be detected by the AI script, but are now uncommon enough to +be skipped:

+ +
+
(uncommon)alignment
+ +
You've managed to confuse the assembler with a misaligned +structure.
+ +
(uncommon)apxs
+ *** DIFF OUTPUT TRUNCATED AT 1000 LINES *** From owner-svn-src-projects@FreeBSD.ORG Sat Apr 23 21:02:25 2011 Return-Path: Delivered-To: svn-src-projects@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id E427C106566B; Sat, 23 Apr 2011 21:02:25 +0000 (UTC) (envelope-from flz@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id CDD5E8FC19; Sat, 23 Apr 2011 21:02:25 +0000 (UTC) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.4/8.14.4) with ESMTP id p3NL2Pr3003592; Sat, 23 Apr 2011 21:02:25 GMT (envelope-from flz@svn.freebsd.org) Received: (from flz@localhost) by svn.freebsd.org (8.14.4/8.14.4/Submit) id p3NL2PpT003585; Sat, 23 Apr 2011 21:02:25 GMT (envelope-from flz@svn.freebsd.org) Message-Id: <201104232102.p3NL2PpT003585@svn.freebsd.org> From: Florent Thoumie Date: Sat, 23 Apr 2011 21:02:25 +0000 (UTC) To: src-committers@freebsd.org, svn-src-projects@freebsd.org X-SVN-Group: projects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r220976 - in projects/portbuild: conf qmanager scripts X-BeenThere: svn-src-projects@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the src " projects" tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 23 Apr 2011 21:02:26 -0000 Author: flz Date: Sat Apr 23 21:02:25 2011 New Revision: 220976 URL: http://svn.freebsd.org/changeset/base/220976 Log: Import qmanager into portbuild. Added: projects/portbuild/qmanager/ projects/portbuild/qmanager/acl.py projects/portbuild/qmanager/dumpdb.py (contents, props changed) projects/portbuild/qmanager/packagebuild (contents, props changed) projects/portbuild/qmanager/qclient (contents, props changed) projects/portbuild/qmanager/qmanager (contents, props changed) projects/portbuild/qmanager/qmanager.py (contents, props changed) projects/portbuild/qmanager/qmanagerclient.py projects/portbuild/qmanager/qmanagerhandler.py projects/portbuild/qmanager/qmanagerobj.py projects/portbuild/qmanager/schema.sql Modified: projects/portbuild/conf/server.conf projects/portbuild/scripts/dopackages Modified: projects/portbuild/conf/server.conf ============================================================================== --- projects/portbuild/conf/server.conf Sat Apr 23 20:59:58 2011 (r220975) +++ projects/portbuild/conf/server.conf Sat Apr 23 21:02:25 2011 (r220976) @@ -72,7 +72,6 @@ PDISPATCH_TIMEOUT=360000 # qmanager definitions (note: Python script, so avoid {}) # -QMANAGER_PATH=/var/portbuild/evil/qmanager QMANAGER_DATABASE_FILE=qdb.sl3 QMANAGER_SOCKET_FILE=/tmp/.qmgr Added: projects/portbuild/qmanager/acl.py ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ projects/portbuild/qmanager/acl.py Sat Apr 23 21:02:25 2011 (r220976) @@ -0,0 +1,156 @@ +# Validate a (uid, (gids)) tuple against an ACL. + +import pwd, grp + +def getuidbyname(username): + if str(username).isdigit(): + return int(username) + return pwd.getpwnam(username)[2] + +def getgidbyname(grname): + if str(grname).isdigit(): + return int(grname) + return grp.getgrnam(grname)[2] + +class ACLElement(object): + """ Component of an ACL. """ + + def __init__(self, name, uidlist, gidlist, sense): + self.name = name + self.uidlist = [getuidbyname(uid) for uid in uidlist] + self.gidlist = [getgidbyname(gid) for gid in gidlist] + self.sense = bool(sense) + + def validate(self, uid, gids): + """ Validate an ACL Element. In order to match, the following must + hold: + + * uid is a subset of self.uidlist, or self.uidlist is empty + * one of the gids must be present in self.gidlist, or + self.gidlist is empty + + If both conditions hold, then the validation returns self.sense + + Returns: True/False if Element matches + None if Element fails to match + """ + + if (len(self.uidlist) == 0 or uid in self.uidlist) and \ + (len(self.gidlist) == 0 or set(gids).intersection(self.gidlist)): + return self.sense + return None + +class ACL(object): + """ List of ACLElements that form an ACL """ + + def __init__(self, acllist): + self.acls = acllist + + def validate(self, uid, gids): + uid=getuidbyname(uid) + gids=set(getgidbyname(gid) for gid in gids) + + for acl in self.acls: + res=acl.validate(uid, gids) + if res is not None: + return res + return False + +if __name__ == "__main__": + + from sys import exit + + assert getuidbyname(123) == 123 + assert getuidbyname('123') == 123 + + try: + ACLElement("test", ["foobar"], [""], True) + except KeyError: + pass + + try: + ACLElement("test", [123, "foobar"], [""], True) + except KeyError: + pass + + assert ACLElement("test", [123], [], True) != None + assert ACLElement("test", ["123"], [], True) != None + + acl = ACL([ACLElement("el 1", ["kris"], [], True), + ACLElement("el 2", [], ["wheel"], True), + ACLElement("el 3", [], [], False)]) + + assert acl.validate(getuidbyname('kris'), []) == True + assert acl.validate(getuidbyname('simon'), []) == False + assert acl.validate(getuidbyname('simon'), [getgidbyname('devel'), getgidbyname('wheel')]) == True + assert acl.validate(getuidbyname('root'), [pwd.getpwnam('root')[3]]) == True + + acl = ACL([ACLElement("el 1", ["kris"], ["distcc"], True), + ACLElement("el 2", [], ["wheel"], True), + ACLElement("el 3", [], [], False)]) + assert acl.validate("kris", ["wheel"]) == True + assert acl.validate("kris", ["staff"]) == False + + acl = ACL([ACLElement("", ('kris',), (), True), + ACLElement("", (), ('wheel', 'devel'), True), + ACLElement("", (), (), False)]) + + assert acl.validate(getuidbyname('simon'), [getgidbyname('devel'), getgidbyname('wheel')]) == True + assert acl.validate(getuidbyname('simon'), [getgidbyname('staff'), getgidbyname('wheel')]) == True + assert acl.validate(getuidbyname('simon'), [getgidbyname('staff')]) == False + + acl = ACL([ACLElement("", ('kris',), (), True), + ACLElement("", (), ('devel',), True), + ACLElement("", (), ('wheel',), True), + ACLElement("", (), (), False)]) + + assert acl.validate(getuidbyname('simon'), [getgidbyname('devel'), getgidbyname('wheel')]) == True + assert acl.validate(getuidbyname('simon'), [getgidbyname('staff'), getgidbyname('wheel')]) == True + assert acl.validate(getuidbyname('simon'), [getgidbyname('staff')]) == False + + acl = ACL([ACLElement("", ('kris',), (), True), + ACLElement("", (), ('devel',), False), + ACLElement("", (), ('wheel',), True), + ACLElement("", (), (), False)]) + + assert acl.validate(getuidbyname('simon'), [getgidbyname('devel'), getgidbyname('wheel')]) == False + assert acl.validate(getuidbyname('simon'), [getgidbyname('staff'), getgidbyname('wheel')]) == True + assert acl.validate(getuidbyname('simon'), [getgidbyname('staff')]) == False + + + acl = ACL([ACLElement("", ('kris',), (), True), + ACLElement("", (), ('devel',), True), + ACLElement("", (), ('wheel',), False), + ACLElement("", (), (), False)]) + + assert acl.validate(getuidbyname('simon'), [getgidbyname('devel'), getgidbyname('wheel')]) == True + assert acl.validate(getuidbyname('simon'), [getgidbyname('staff'), getgidbyname('wheel')]) == False + assert acl.validate(getuidbyname('simon'), [getgidbyname('staff')]) == False + + + acl = ACL([ACLElement("", ('kris',), (), True), + ACLElement("", (), ('devel',), True), + ACLElement("", (), ('wheel',), False), + ACLElement("", (), (), True)]) + + assert acl.validate(getuidbyname('simon'), []) == True + assert acl.validate(getuidbyname('simon'), [getgidbyname('devel'), getgidbyname('wheel')]) == True + assert acl.validate(getuidbyname('simon'), [getgidbyname('staff'), getgidbyname('wheel')]) == False + assert acl.validate(getuidbyname('simon'), [getgidbyname('staff')]) == True + + acl = ACL([ACLElement("", ('kris',), (), False), + ACLElement("", (), ('devel',), True), + ACLElement("", (), ('wheel',), False), + ACLElement("", (), (), True)]) + + assert acl.validate(getuidbyname('simon'), []) == True + assert acl.validate(getuidbyname('kris'), []) == False + assert acl.validate(getuidbyname('simon'), [getgidbyname('devel'), getgidbyname('wheel')]) == True + assert acl.validate(getuidbyname('simon'), [getgidbyname('staff'), getgidbyname('wheel')]) == False + assert acl.validate(getuidbyname('simon'), [getgidbyname('staff')]) == True + + acl = ACL([ACLElement("", (4206,), set([]), True), + ACLElement("", (), set([]), False)]) + + assert acl.validate(4206, (4206, 31337)) == True + assert acl.validate(4201, (4201, 31337)) == False Added: projects/portbuild/qmanager/dumpdb.py ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ projects/portbuild/qmanager/dumpdb.py Sat Apr 23 21:02:25 2011 (r220976) @@ -0,0 +1,140 @@ +# +# try doing some SQL reads as a test +# +from freebsd_config import * + +import os, threading, socket, Queue + +from signal import * +from sys import exc_info +from itertools import chain + +from qmanagerobj import * + +CONFIG_DIR="/var/portbuild" +CONFIG_SUBDIR="conf" +CONFIG_FILENAME="server.conf" + +# pieces of qmanagerobj.startup +def obj_startup(filename): + + engine = create_engine('sqlite:///' + filename, echo=True) + Session = sessionmaker(bind=engine) + session = Session() + + Base.metadata.create_all(engine) + + return (engine, session) + + +def show_acl( session ): + + acls = session.query(QManagerACL) + acls = acls.order_by('name') + + print + print 'starting dump of acl table:' + print + + for acl in acls: + + print + print "name: %s" % acl.name + # list + print "uidlist: " + str( acl.uidlist ) + # list + print "gidlist: " + str( acl.gidlist ) + print "sense: " + str( acl.sense ) + + +def show_jobs( session ): + + jobs = session.query(Job) + jobs = jobs.order_by('id') + + print + print 'starting dump of Job table:' + print + + for job in jobs: + + print + # job ID + print "job id: " + `job.id` + # Name of job + print "name: " + job.name + # priority of request + print "priority: " + `job.priority` + # job type + print "type: " + job.type + # uid of job owner + print "owner: " + `job.owner` + # gids of job owner (tuple) + #print str( type( job.gids ) ) + print "gids: " + str( job.gids ) + # machines that satisfied initial query (list) + #print str( type( job.machines ) ) + print "machines: " + str( job.machines ) + # Time job started/blocked (must not be modified when job is + # blocked or it will corrupt the heapq) + print "startttime: " + `job.starttime` + # initial machine description in case we have to revalidate (list) + # print str( type( job.mdl ) ) + print "mdl: " + str( job.mdl ) + # True --> job is running; False --> job is blocked + print "running: " + str( job.running ) + + +def show_machines( session ): + + machines = session.query(Machine) + machines = machines.order_by('name') + + print + print 'starting dump of Machines table:' + print + + for machine in machines: + + print + print "name: %s" % machine.name + # list + print "acl: " + str( machine.acl ) + # boolean + print "haszfs: " + str( machine.haszfs ) + # boolean + print "online: " + str( machine.online ) + + +def show_machines_for_arch( engine, arch ): + + mdl = ["arch = %s" % arch] + + q = SQL.construct(Machine, mdl) + res = engine.execute(Machine.__table__.select(q)) + result = [SQL.to_dict(Machine, i) for i in res] + + print + for machine in result: + print "machine for %s : %s " % ( arch, machine[ 'name' ] ) + + +# main + +if __name__ == '__main__': + + print "acquiring engine and session" + config = getConfig( CONFIG_DIR, CONFIG_SUBDIR, CONFIG_FILENAME ) + QMANAGER_PATH = config.get( 'QMANAGER_PATH' ) + QMANAGER_DATABASE_FILE = config.get( 'QMANAGER_DATABASE_FILE' ) + (engine, session) = obj_startup( \ + os.path.join( QMANAGER_PATH, QMANAGER_DATABASE_FILE ) ) + print "acquired engine and session" + # print "engine = '" + str( engine ) + "', session = '" + str( session ) + "'" + + show_acl( session ) + show_machines( session ) + show_jobs( session ) + + show_machines_for_arch( engine, 'i386' ) + show_machines_for_arch( engine, 'amd64' ) Added: projects/portbuild/qmanager/packagebuild ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ projects/portbuild/qmanager/packagebuild Sat Apr 23 21:02:25 2011 (r220976) @@ -0,0 +1,649 @@ +#!/usr/bin/env python + +# Improved build dispatcher. Invoked on server-side from dopackages. + +# We try to build leaf packages (those +# which can be built immediately without requiring additional +# dependencies to be built) in the order such that the ones required +# by the longest dependency chains are built first. +# +# This has the effect of favouring deep parts of the package tree and +# evening out the depth over time, hopefully avoiding the situation +# where the entire cluster waits for a deep part of the tree to +# build on a small number of machines +# +# We can dynamically respond to changes in build machine availability, +# since the queue manager will block jobs that cannot be immediately +# satisfied and will unblock us when a job slot becomes available. +# +# When a package build fails, it is requeued with a lower priority +# such that it will rebuild again as soon as no "phase 1" packages +# are available to build. This prevents the cluster staying idle +# until the last phase 1 package builds. +# +# Other advantages are that this system is easily customizable and in +# the future will let us customize things like the matching policy of +# jobs to machines. For example, we could avoid dispatching multiple +# openoffice builds to the same system. +# +# TODO: +# * Combine build prep stages? +# - initial check for file up-to-date +# * check mtime for package staleness (cf make) +# * option to skip phase 2 + +from qmanagerclient import * + +from freebsd_config import * + +import os, string, sys, threading, time, subprocess +#import random +from itertools import chain +#import gc +from stat import * + +from Queue import Queue +from heapq import * + +CONFIG_DIR="/var/portbuild" +CONFIG_SUBDIR="conf" +CONFIG_FILENAME="server.conf" + +config = getConfig( CONFIG_DIR, CONFIG_SUBDIR, CONFIG_FILENAME ) +QMANAGER_MAX_JOB_ATTEMPTS = int( \ + config.get( 'QMANAGER_MAX_JOB_ATTEMPTS' ) ) +QMANAGER_PRIORITY_PACKAGES = string.split( \ + config.get( 'QMANAGER_PRIORITY_PACKAGES' ) ) +QMANAGER_RUNAWAY_PERCENTAGE = float( \ + config.get( 'QMANAGER_RUNAWAY_PERCENTAGE' ) ) +QMANAGER_RUNAWAY_THRESHOLD = int( \ + config.get( 'QMANAGER_RUNAWAY_THRESHOLD' ) ) + +DEBUG = False + +categories = {} +ports = {} + +# When a build fails we requeue it with a lower priority such that it +# will never preempt a phase 1 build but will build when spare +# capacity is available. +PHASE2_BASE_PRIO=1000 + +# Process success quickly so other jobs are started +SUCCESS_PRIO = -1000 + +# Failure should be a less common event :) +FAILURE_PRIO = -900 + +# Port status codes +PENDING = 1 # Yet to build +PHASE2 = 2 # Failed once + +class PriorityQueue(Queue): + """Variant of Queue that retrieves open entries in + priority order (lowest first). + Entries are typically tuples of the form: (priority number, + data) + This class can be found at: Python-2.6a3/Lib/Queue.py + """ + maxsize = 0 + + def _init(self, maxsize): + self.queue = [] + + def _qsize(self, len=len): + return len(self.queue) + + def _put(self, item, heappush=heappush): + heappush(self.queue, item) + + def _get(self, heappop=heappop): + return heappop(self.queue) + +class Index(object): + + def __init__(self, indexfile): + self.indexfile = indexfile + + def parse(self, targets = None): + + print "[MASTER] Read index" + f = file(self.indexfile) + index = f.readlines() + f.close() + f = None + del f + + lines=[] + print "[MASTER] Phase 1" + for i in index: + (name, path, prefix, comment, descr, maintainer, categories, bdep, + rdep, www, edep, pdep, fdep) = i.rstrip().split("|") + + if targets is None or name in targets: + lines.append((name, bdep, rdep, edep, pdep, fdep)) + + Port(name, path, "", "", "", "", + categories, "") + index = None + del index + + print "[MASTER] Phase 2" + for (name, bdep, rdep, edep, pdep, fdep) in lines: + ports[name].setdeps(bdep, rdep, edep, pdep, fdep) + + lines = None + del lines + print "[MASTER] Done" + +def depthindex(targets): + """ Initial population of depth tree """ + + for i in targets: + i.depth_recursive() + +class Port(object): + + def __init__(self, name, path, prefix, comment, descr, maintainer, + cats, www): + + __slots__ = ["name", "path", "prefix", "comment", "descr", + "maintainer", "www", "bdep", "rdep", "edep", "pdep", + "fdep", "alldep", "parents", "depth", "categories"] + + self.name = name + self.path = path + self.prefix = prefix + self.comment = comment + self.descr = descr + self.maintainer = maintainer + self.www = www + + # Populated later + self.bdep = [] + self.rdep = [] + self.edep = [] + self.pdep = [] + self.fdep = [] + + self.alldep = [] + self.parents = [] + self.id = None # XXX + + self.status = PENDING + self.attempts = 0 + + # Whether the package build has completed and is hanging around + # to resolve dependencies for others XXX use status + self.done = False + + # Depth is the maximum length of the dependency chain of this port + self.depth = None + + self.categories=[] + scats = cats.split() + if len(scats) != len(set(scats)): + print "[MASTER] Warning: port %s includes duplicated categories: %s" % (name, cats) + + for c in set(scats): + try: + cat = categories[c] + except KeyError: + cat = Category(c) + + self.categories.append(cat) + cat.add(self) + + ports[name] = self + + def remove(self): + """ Clean ourselves up but don't touch references in other objects; +they still need to know about us as dependencies etc """ + + self.fdep = None + self.edep = None + self.pdep = None + self.bdep = None + self.rdep = None + self.alldep = None + self.parents = None + + for cat in self.categories: + cat.remove(self) + + ports[self.name] = None + del ports[self.name] + del self + + def destroy(self): + """ Remove a package and all references to it """ + + for pkg in self.alldep: + if pkg.parents is not None: + # Already removed but not destroyed + try: + pkg.parents.remove(self) + except ValueError: + continue + + for pkg in self.parents: + try: + pkg.fdep.remove(self) + except ValueError: + pass + try: + pkg.edep.remove(self) + except ValueError: + pass + try: + pkg.pdep.remove(self) + except ValueError: + pass + try: + pkg.bdep.remove(self) + except ValueError: + pass + try: + pkg.rdep.remove(self) + except ValueError: + pass + pkg.alldep.remove(self) + + sys.exc_clear() + + self.remove() + + def setdeps(self, bdep, rdep, edep, pdep, fdep): + self.fdep = [ports[p] for p in fdep.split()] + self.edep = [ports[p] for p in edep.split()] + self.pdep = [ports[p] for p in pdep.split()] + self.bdep = [ports[p] for p in bdep.split()] + self.rdep = [ports[p] for p in rdep.split()] + + self.alldep = list(set(chain(self.fdep, self.edep, self.pdep, + self.bdep, self.rdep))) + + for p in self.alldep: + p.parents.append(self) + + def depth_recursive(self): + + """ + Recursively populate the depth tree up from a given package + through dependencies, assuming empty values on entries not yet + visited + """ + + if self.depth is None: + if len(self.parents) > 0: + max = 0 + for i in self.parents: + w = i.depth_recursive() + if w > max: + max = w + self.depth = max + 1 + else: + self.depth = 1 + for port in QMANAGER_PRIORITY_PACKAGES: + if self.name.startswith(port): + # Artificial boost to try and get it building earlier + self.depth = 100 + return self.depth + + def destroy_recursive(self): + """ Remove a port and everything that depends on it """ + + parents=set([self]) + + while len(parents) > 0: + pkg = parents.pop() + assert pkg.depth is not None + parents.update(pkg.parents) + pkg.destroy() + + def success(self): + """ Build succeeded and possibly uncovered some new leaves """ + + parents = self.parents[:] + self.done = True + self.remove() + + newleafs = [p for p in parents if all(c.done for c in p.alldep)] + return newleafs + + def failure(self): + """ Build failed """ + + self.destroy_recursive() + + def packagename(self, arch, branch, buildid): + """ Return the path where a package may be found""" + + return "/var/portbuild/%s/%s/builds/%s/packages/All/%s.tbz" \ + % (arch, branch, buildid, self.name) + + def is_stale(self, arch, branch, buildid): + """ Does a package need to be (re)-built? + + Returns: False: if it exists and has newer mtime than all of + its dependencies. + True: otherwise + """ + + my_pkgname = self.packagename(arch, branch, buildid) + pkg_exists = os.path.exists(my_pkgname) + + if pkg_exists: + my_mtime = os.stat(my_pkgname)[ST_MTIME] + + dep_packages = [pkg.packagename(arch, branch, buildid) + for pkg in self.alldep] + deps_exist = all(os.path.exists(pkg) for pkg in dep_packages) + return not (pkg_exists and deps_exist and + all(os.stat(pkg)[ST_MTIME] <= my_mtime + for pkg in dep_packages)) + +class Category(object): + def __init__(self, name): + self.name = name + self.ports = {} + categories[name] = self + + def add(self, port): + self.ports[port] = port + + def remove(self, port): + self.ports[port]=None + del self.ports[port] + +def gettargets(targets): + """ split command line arguments into list of packages to build. + Returns set or iterable of all ports that will be built including + dependencies """ + + plist = set() + if len(targets) == 0: + targets = ["all"] + for i in targets: + if i == "all": + return ports.itervalues() + + if i.endswith("-all"): + cat = i.rpartition("-")[0] + plist.update(p.name for p in categories[cat].ports) + elif i.rstrip(".tbz") in ports: + plist.update([ports[i.rstrip(".tbz")].name]) + else: + raise KeyError, i + + # Compute transitive closure of all dependencies + pleft=plist.copy() + while len(pleft) > 0: + pkg = pleft.pop() + new = [p.name for p in ports[pkg].alldep] + plist.update(new) + pleft.update(new) + + for p in set(ports.keys()).difference(plist): + ports[p].destroy() + + return [ports[p] for p in plist] + +class worker(threading.Thread): + + # Protects threads + lock = threading.Lock() + + # Running threads, used for collecting status + threads = {} + + def __init__(self, mach, job, arch, branch, buildid, queue): + threading.Thread.__init__(self) + self.machine = mach + self.job = job + self.arch = arch + self.branch = branch + self.buildid = buildid + self.queue = queue + + self.setDaemon(True) + + def run(self): + pkg = self.job + + print "[MASTER] Running job %s" % (pkg.name), + if pkg.status == PHASE2: + print " (phase 2)" + else: + print + try: + runenv={'HOME':"/root", + 'PATH':'/sbin:/bin:/usr/sbin:/usr/bin:/usr/games:/usr/local/sbin:/usr/local/bin:/var/portbuild/scripts', + 'FD':" ".join(["%s.tbz" % p.name for p in pkg.fdep]), + 'ED':" ".join(["%s.tbz" % p.name for p in pkg.edep]), + 'PD':" ".join(["%s.tbz" % p.name for p in pkg.pdep]), + 'BD':" ".join(["%s.tbz" % p.name for p in pkg.bdep]), + 'RD':" ".join(["%s.tbz" % p.name for p in pkg.rdep])} + for var in ["NOCLEAN", "NO_RESTRICTED", "NOPLISTCHECK", "NO_DISTFILES", "FETCH_ORIGINAL", "TRYBROKEN" ]: + if var in os.environ: + runenv[var] = os.environ.get(var) + build = subprocess.Popen( + ["/bin/sh", "/var/portbuild/scripts/pdispatch", + self.arch, self.branch, self.buildid, self.machine, + "/var/portbuild/scripts/portbuild", "%s.tbz" % pkg.name, + pkg.path], + env=runenv, + stderr=subprocess.STDOUT, stdout=subprocess.PIPE, bufsize=0) + except OSError, e: + print >>sys.stderr, "[%s:%s]: Execution failed: %s" % \ + (pkg.id, pkg.name, e) + while True: + try: + line = build.stdout.readline() + except: + print "[%s:%s]: Failed reading from build script" % \ + (pkg.id, pkg.name) + break + if line == "": + break + print "[%s:%s] %s" % (pkg.id, pkg.name, line.rstrip()) + + retcode = build.wait() + +# time.sleep(random.randint(0,60)) +# +# r = random.random() +# if r < 0.1: +# retcode = 1 +# elif r < 0.15: +# retcode = 254 +# else: +# retcode = 0 + + conn = QManagerClientConn(stderr = sys.stderr) + timeout = 1 + try: + (code, vars) = conn.command("release", {'id':pkg.id}) + except RequestError, e: + print "[MASTER] Error releasing job %s (%s): %s" % (pkg.name, pkg.id, e.value) + + if DEBUG: + print "[MASTER] got retcode %d from pkg %s" % (retcode, pkg.name) + if retcode == 254: + # Requeue soft failure at original priority + # XXX exponential backoff? + time.sleep(60) +# print "Requeueing %s" % pkg.id + self.queue.put((-pkg.depth, pkg)) + elif retcode == 253: + # setting up a machine, we should immediately retry + self.queue.put((-pkg.depth, pkg)) + elif retcode == 0: + self.queue.put((SUCCESS_PRIO, pkg)) + else: + self.queue.put((FAILURE_PRIO, pkg)) + + # Clean up + worker.lock.acquire() + worker.threads[self]=None + del worker.threads[self] + worker.lock.release() + + @staticmethod + def dispatch(mach, job, arch, branch, buildid, queue): + wrk = worker(mach, job, arch, branch, buildid, queue) + + worker.lock.acquire() + worker.threads[wrk] = wrk + worker.lock.release() + + wrk.start() + +def main(arch, branch, buildid, args): + global index + + basedir="/var/portbuild/"+arch+"/"+branch+"/builds/"+buildid + portsdir=basedir+"/ports" + + # get the major branch number. + branchbase = branch.split("-")[ 0 ] + # XXX ERWLA - Ugly hack + branchbase = branchbase.split(".")[ 0 ] + indexfile=portsdir+"/INDEX-"+branchbase + + print "[MASTER] parseindex..." + index = Index(indexfile) + index.parse() + print "[MASTER] length = %s" % len(ports) + + print "[MASTER] Finding targets..." + targets = gettargets(args) + + print "[MASTER] Calculating depth..." + depthindex(targets) + + print "[MASTER] Pruning duds..." + dudsfile=basedir+"/duds" + for line in file(dudsfile): + try: + dud = ports[line.rstrip()] + except KeyError: + continue + print "[MASTER] Skipping %s (duds)" % dud.name + dud.destroy_recursive() + + queue = PriorityQueue() + # XXX can do this while parsing index if we prune targets/duds + # first + for pkg in ports.itervalues(): + if len(pkg.alldep) == 0: + queue.put((-pkg.depth, pkg)) + + # XXX check osversion, pool + mdl=["arch = %s" % arch] + + # Main work loop + completed_jobs = 0 + failed_jobs = 0 + while len(ports) > 0: + print "[MASTER] Ports remaining=%s, Queue length=%s" % (len(ports), queue.qsize()) + + if len(ports) < 10: + print "[MASTER] Remaining ports: %s" % ports.keys() + + (prio, job) = queue.get() + if DEBUG: + print "[MASTER] Job %s pulled from queue with prio %d" % ( job.name, prio ) + if prio == SUCCESS_PRIO: + print "[MASTER] Job %s succeeded" % job.name + for new in job.success(): + queue.put((-new.depth, new)) + completed_jobs = completed_jobs + 1 + continue + elif prio == FAILURE_PRIO: + if job.status == PHASE2: + print "[MASTER] Job %s failed" % job.name + job.failure() + continue + else: + # XXX MCL 20110421 + completed_jobs = completed_jobs + 1 + failed_jobs = failed_jobs + 1 + if DEBUG: + print "[MASTER] jobs: %d failed jobs out of %d:" % \ + ( failed_jobs, completed_jobs ) + if completed_jobs > QMANAGER_RUNAWAY_THRESHOLD and \ + float( failed_jobs ) / completed_jobs > QMANAGER_RUNAWAY_PERCENTAGE: + print "[MASTER] ERROR: runaway build detected: %d failed jobs out of %d:" % \ + ( failed_jobs, completed_jobs ) + print "[MASTER] RUN TERMINATED." + break + + job.attempts = job.attempts + 1 + # XXX MCL in theory, if all this code worked correctly, + # this condition would never trigger. In practice, + # however, it does, so bomb out before filling portmgr's + # mbox. + # XXX MCL 20110422 perhaps this code has been fixed now; + # XXX it did not use to work: + if job.attempts > QMANAGER_MAX_JOB_ATTEMPTS: + print "[MASTER] Job %s failed %d times; RUN TERMINATED." % ( job.name, job.attempts ) + break + else: + # Requeue at low priority + print "[MASTER] Job %s failed (requeued for phase 2)" % job.name + job.status = PHASE2 + queue.put((PHASE2_BASE_PRIO-job.depth, job)) + continue + elif job.status == PHASE2: + depth = -(prio - PHASE2_BASE_PRIO) + else: + depth = -prio + + print "[MASTER] Working on job %s, depth %d" % (job.name, depth) + if job.is_stale(arch, branch, buildid): + conn = QManagerClientConn(stderr = sys.stderr) + (code, vars) = conn.command("acquire", + {"name":job.name, + "type":"%s/%s/%s package" % \ + (arch, branch, buildid), + "priority":10, "mdl":mdl}) + + if code[0] == "2": + machine=vars['machine'] + job.id=vars['id'] +# print "Got ID %s" % job.id + + worker.dispatch(machine, job, arch, branch, buildid, queue) + else: + print "[MASTER] Error acquiring job %s: %s" % (pkg.name, code) + else: + print "[MASTER] Skipping %s since it already exists" % job.name + for new in job.success(): + queue.put((-new.depth, new)) + + print "[MASTER] Waiting for threads" + threads = worker.threads.copy() + + for t in threads: + print "[MASTER] Outstanding thread: %s" % t.job.name + + for t in threads: + print "[MASTER] Waiting for thread %s" % t.job.name + t.join() + + print "[MASTER] Finished" + +if __name__ == "__main__": + + try: + main(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4:]) + sys.exit( 0 ) + except Exception, e: + # XXX MCL TODO move this above + print "packagebuild: Exception:" + try: + print str( e ) + except: + pass + sys.exit( 1 ) Added: projects/portbuild/qmanager/qclient ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ projects/portbuild/qmanager/qclient Sat Apr 23 21:02:25 2011 (r220976) @@ -0,0 +1,242 @@ +#!/usr/bin/env python + +# queue manager client + +# TODO: +# * pretty-print command output + +import socket, os, sys + +from optparse import OptionParser +from qmanagerclient import * + +def error(msg): + print >>sys.stderr, "%s: %s" % (sys.argv[0], msg.rstrip()) + sys.exit(1) + +def buildquery(option, opt, values, parser): + """ + Turn command line options into a query description + + Modifies: + query global + + Raises: + ValueError if bogus arguments to numeric operators + + """ + global query + + numopt = False + if opt in ("-m", "--machine"): + key="name" + *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***