Date: Mon, 4 Mar 2013 01:56:00 +0000 (UTC) From: Colin Percival <cperciva@FreeBSD.org> To: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: svn commit: r247750 - in user/cperciva: . portsnap-build portsnap-build/s Message-ID: <201303040156.r241u0pZ075652@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: cperciva Date: Mon Mar 4 01:56:00 2013 New Revision: 247750 URL: http://svnweb.freebsd.org/changeset/base/247750 Log: Add Portsnap build code. To set up portsnap builds: 1. Edit build.conf, 2. Run setup.sh and follow the instructions it prints, 3. Run keygen.sh and follow the instructions it prints. After that, # build.sh snap does a portsnap build and generates a snapshot tarball (these are used to provide a faster bootstrap than downloading 23000+ individual files), # build.sh update does a portsnap build without a snapshot tarball, and # upload.sh uploads files over SSH to the target host. Alternatively, # loop.sh & will do a continuous loop of builds and uploads with one snapshot per day. Added: user/cperciva/ user/cperciva/portsnap-build/ user/cperciva/portsnap-build/build.conf (contents, props changed) user/cperciva/portsnap-build/build.sh (contents, props changed) user/cperciva/portsnap-build/keygen.sh (contents, props changed) user/cperciva/portsnap-build/loop.sh (contents, props changed) user/cperciva/portsnap-build/releasesnap.sh (contents, props changed) user/cperciva/portsnap-build/s/ user/cperciva/portsnap-build/s/alias-all.sh (contents, props changed) user/cperciva/portsnap-build/s/alias-index.sh (contents, props changed) user/cperciva/portsnap-build/s/describes-err.sh (contents, props changed) user/cperciva/portsnap-build/s/describes-fallback.sh (contents, props changed) user/cperciva/portsnap-build/s/describes-icbm.sh (contents, props changed) user/cperciva/portsnap-build/s/describes-run.sh (contents, props changed) user/cperciva/portsnap-build/s/describes-warn.sh (contents, props changed) user/cperciva/portsnap-build/s/fileset-addfiles.sh (contents, props changed) user/cperciva/portsnap-build/s/fileset-findnew-cleanup.sh (contents, props changed) user/cperciva/portsnap-build/s/fileset-findnew.sh (contents, props changed) user/cperciva/portsnap-build/s/fileset-mergedb.sh (contents, props changed) user/cperciva/portsnap-build/s/fileset-mirrorlists.sh (contents, props changed) user/cperciva/portsnap-build/s/fileset-mkpatches.sh (contents, props changed) user/cperciva/portsnap-build/s/fileset-prune.sh (contents, props changed) user/cperciva/portsnap-build/s/fileset-publish-tindex.sh (contents, props changed) user/cperciva/portsnap-build/s/fileset-publish.sh (contents, props changed) user/cperciva/portsnap-build/s/fileset-snap.sh (contents, props changed) user/cperciva/portsnap-build/s/svn-getrev.sh (contents, props changed) user/cperciva/portsnap-build/s/treesnap-build.sh (contents, props changed) user/cperciva/portsnap-build/s/treesnap-index.sh (contents, props changed) user/cperciva/portsnap-build/s/treesnap-mktars-all.sh (contents, props changed) user/cperciva/portsnap-build/s/treesnap-mktars.sh (contents, props changed) user/cperciva/portsnap-build/s/treesnap-publishsigs.sh (contents, props changed) user/cperciva/portsnap-build/s/treesnap-sign.sh (contents, props changed) user/cperciva/portsnap-build/setup.sh (contents, props changed) user/cperciva/portsnap-build/upload.sh (contents, props changed) Added: user/cperciva/portsnap-build/build.conf ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/cperciva/portsnap-build/build.conf Mon Mar 4 01:56:00 2013 (r247750) @@ -0,0 +1,68 @@ +# SVN repository +export REPO=svn://svn.freebsd.org/ports + +# tar(1) utility for building tarballs +export TAR=`which tar` + +# sizes of memory disks +export PORTSMDSIZE=500M +export SNAPMDSIZE=300M +export JAILMDSIZE=100M +export TMPMDSIZE=100M + +# number of jobs to run in parallel +export JNUM=16 + +# world tarball to use for INDEX building +export WORLDTAR=`pwd`/world.tar + +# file listing "aliases" for tarballs to allow widespread changes (e.g., +# switching from cvs to svn, or between versions of tar) to be reverted. +# NOTE: This is used by freebsd.org portsnap builds but is unlikely to +# be needed by anyone else. +# export ALIASFILE=`pwd`/ALIAS + +# internal work directory +export STATEDIR=`pwd`/state + +# From: and To: for INDEX-breakage notifications +export INDEXMAIL_FROM="your@email.address" +export INDEXMAIL_TO="your@email.address" + +# From: and To: for build status emails +export BUILDMAIL_FROM="your@email.address" +export BUILDMAIL_TO="your@email.address" + +# Duration to keep port tarballs and generate binary patches +export MAXAGE_DATA=2592000 + +# Duration to keep metadata files on mirrors and generate patches +export MAXAGE_META=691200 + +# Duration to keep "extra" files (tags and snapshot tarballs) around +export MAXAGE_EXTRA=172800 + +# Describes files to build (on HEAD) +export DESCRIBES_BUILD="7 8 9" + +# Describes files to publish (on HEAD) +export DESCRIBES_PUBLISH="4 5 6 7 8 9" + +# Keys to sign builds with (get this from keygen.sh) +export SIGNKEYS="XXXX" + +# Key to generate backwards-compatible signature with (until new +# multi-signature functionality is added to portsnap in mid-2013). +export SIGNKEYS_PRIMARY="XXXX" + +# SSH key for uploading bits +UPLOAD_KEY=/root/id_dsa_upload + +# Account to ssh into for uploading bits +UPLOAD_ACCT=ssh-user@ssh-host + +# Directory (relative to the home directory) to extract uploaded bits into +UPLOAD_DIR=htmldocs + +# keep this here +export PORTSNAP_BUILD_CONF_READ=1 Added: user/cperciva/portsnap-build/build.sh ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/cperciva/portsnap-build/build.sh Mon Mar 4 01:56:00 2013 (r247750) @@ -0,0 +1,98 @@ +#!/bin/sh -e +set -e + +# usage: sh -e build.sh MODE +MODE=$1 + +# Sanity-check input +if ! [ "${MODE}" = "update" ] && ! [ "${MODE}" = "snap" ]; then + echo "usage: sh -e build.sh (update|snap)" + exit 1 +fi + +# Load configuration +. build.conf + +# Create working direcories +WORKDIR=${STATEDIR}/work +SNAPDIR=${WORKDIR}/snap +TMPDIR=${WORKDIR}/tmp +SIGDIR=${WORKDIR}/sigs +mkdir ${WORKDIR} ${SNAPDIR} ${TMPDIR} ${SIGDIR} + +# Record when we're starting +SNAPDATE=`date "+%s"` + +# Get the latest revision # on the tree +NEWREV=`sh -e s/svn-getrev.sh head` + +# Create a memory disk for holding the snapshot files. +SNAPMD=`mdconfig -a -t swap -s ${SNAPMDSIZE} -n` +newfs -O 1 -n /dev/md${SNAPMD} >/dev/null +mount /dev/md${SNAPMD} ${SNAPDIR} + +# Build a snapshot +sh -e s/treesnap-build.sh head@${NEWREV} "${DESCRIBES_BUILD}" \ + ${TMPDIR} ${SNAPDIR} + +# Replace tarballs with "aliased" tarballs +if ! [ -z ${ALIASFILE} ]; then + if [ ${MODE} = "snap" ]; then + sh -e s/alias-all.sh ${SNAPDIR} ${STATEDIR}/fileset/oldfiles \ + ${ALIASFILE} ${WORKDIR} + else + sh -e s/alias-index.sh ${SNAPDIR} ${STATEDIR}/fileset/oldfiles \ + ${ALIASFILE} ${WORKDIR} + fi +fi + +# Send emails if INDEX was broken or fixed +sh -e s/describes-warn.sh ${SNAPDIR} ${NEWREV} ${STATEDIR}/describes \ + "${DESCRIBES_BUILD}" + +# Use old DESCRIBE files if the latest ones didn't build +sh -e s/describes-fallback.sh ${SNAPDIR} ${STATEDIR}/describes \ + "${DESCRIBES_PUBLISH}" + +# Collect metadata +sh -e s/treesnap-index.sh ${SNAPDIR} ${WORKDIR}/INDEX ${WORKDIR}/tINDEX + +# Add these files to our (overlapping) set of snapshots +sh -e s/fileset-snap.sh ${SNAPDATE} ${STATEDIR}/fileset ${SNAPDIR} \ + ${WORKDIR}/INDEX ${WORKDIR}/tINDEX ${STATEDIR}/stage ${TMPDIR} + +# Sign the tree +sh -e s/treesnap-sign.sh ${SNAPDATE} ${WORKDIR}/tINDEX ${SIGDIR} + +# Publish signatures as the "latest" build +sh -e s/treesnap-publishsigs.sh ${SIGDIR} ${STATEDIR}/stage latest + +# Build a snapshot tarball if necessary +if [ ${MODE} = "snap" ]; then + echo "`date`: Building snapshot tarball" + SNAPSHOTHASH=`sha256 -q ${WORKDIR}/tINDEX` + tar -czf ${STATEDIR}/stage/s/${SNAPSHOTHASH}.tgz -C ${WORKDIR} snap + echo "${SNAPDATE}|s/${SNAPSHOTHASH}.tgz" \ + >> ${STATEDIR}/fileset/extradb + sh -e s/treesnap-publishsigs.sh ${SIGDIR} ${STATEDIR}/stage snapshot +fi + +# Delete signatures +rm ${SIGDIR}/*.ssl + +# Unmount and delete the snapshot disk +umount /dev/md${SNAPMD} +mdconfig -d -u ${SNAPMD} + +# Delete indexes +rm ${WORKDIR}/INDEX ${WORKDIR}/tINDEX + +# Delete temporary directories +rmdir ${SIGDIR} ${TMPDIR} ${SNAPDIR} ${WORKDIR} + +# Publish file lists for mirroring +sh -e s/fileset-mirrorlists.sh ${STATEDIR}/fileset ${STATEDIR}/stage + +# Report sucess +echo "`date`: Finished" + Added: user/cperciva/portsnap-build/keygen.sh ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/cperciva/portsnap-build/keygen.sh Mon Mar 4 01:56:00 2013 (r247750) @@ -0,0 +1,37 @@ +#!/bin/sh -e +set -e + +# usage: sh -e keygen.sh + +# Load configuration +. build.conf + +# Create temporary working space +mkdir ${STATEDIR}/work + +# Create key +openssl genrsa -F4 2048 > ${STATEDIR}/work/priv.ssl +openssl rsa -in ${STATEDIR}/work/priv.ssl -pubout > ${STATEDIR}/work/pub.ssl + +# Compute key hash +KEYHASH=`sha256 -q ${STATEDIR}/work/pub.ssl` + +# Move files into their permanent location +mv ${STATEDIR}/work/priv.ssl ${STATEDIR}/keys/priv-${KEYHASH}.ssl +mv ${STATEDIR}/work/pub.ssl ${STATEDIR}/keys/pub-${KEYHASH}.ssl + +# Announce the public key hash +cat <<- EOF + + A key has been generated with hash: + ${KEYHASH} + +EOF +xargs -s 80 <<- EOF + Add this to the SIGNKEYS list in build.conf to start signing builds + with it; and set it as the KEYPRINT value in /etc/portsnap.conf on + systems which need to use these updates. +EOF + +# Remove our temporary working directory +rmdir ${STATEDIR}/work Added: user/cperciva/portsnap-build/loop.sh ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/cperciva/portsnap-build/loop.sh Mon Mar 4 01:56:00 2013 (r247750) @@ -0,0 +1,47 @@ +#!/bin/sh +set -e + +# Load configuration +. build.conf + +# Loop until we fail or the looping is stopped administratively +while ! [ -f failed ] && ! [ -f adminlock ]; do + # Figure out if this is the first build of a new day + touch -t `date "+%Y%m%d0000"` ${STATEDIR}/midnight + if ! [ ${STATEDIR}/lastsnap -nt ${STATEDIR}/midnight ]; then + BUILDTYPE=snap + touch ${STATEDIR}/lastsnap + else + BUILDTYPE=update + fi + rm ${STATEDIR}/midnight + + # Send an email + ( + echo "From: ${BUILDMAIL_FROM}" + for ADDR in ${BUILDMAIL_TO}; do + echo "To: ${ADDR}" + done + echo "Subject: `hostname` Portsnap build.sh ${BUILDTYPE}" + echo + + # Do the build + if ! sh -e build.sh ${BUILDTYPE} 2>&1; then + touch failed; + else + # Upload if the build succeeded + sh upload.sh 2>&1 || true + fi + + # Once a day, clean up portsnap-master + if [ ${BUILDTYPE} = "snap" ]; then + echo "`date`: Cleaning bits on portsnap-master" + ssh -i ${UPLOAD_KEY} ${UPLOAD_ACCT} \ + sh portsnap-clean.sh 2>&1 + fi + ) | sendmail -t +done + +# Send a warning if builds stop running +echo "Subject: `hostname` Portsnap builds no longer running!" | + sendmail ${BUILDMAIL_TO} Added: user/cperciva/portsnap-build/releasesnap.sh ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/cperciva/portsnap-build/releasesnap.sh Mon Mar 4 01:56:00 2013 (r247750) @@ -0,0 +1,69 @@ +#!/bin/sh -e +set -e + +# usage: sh -e releasesnap.sh TREE DESCRIBES TARBALL +# e.g., sh -e releasesnap.sh tags/RELEASE_9_1_0 "7 8 9" portsnap.tgz +TREE="$1" +DESCRIBES="$2" +TARBALL="$3" + +# Sanity-check input +if [ -z "${TREE}" ] || [ -z "${DESCRIBES}" ] || [ -z "${TARBALL}" ]; then + echo "usage: sh -e releasesnap.sh TREE DESCRIBES TARBALL" + exit 1 +fi + +# Load configuration +. build.conf + +# Get the latest revision # on the tree +NEWREV=`sh -e s/svn-getrev.sh ${TREE}` + +# Create a memory disk for holding everything which will end up in +# /var/db/portsnap. Note that for normal (head) builds we mount the disk +# on ${SNAPDIR}; we can't do that here because we want everything to be on +# a single filesystem so that hardlinks work. +SNAPMD=`mdconfig -a -t swap -s ${SNAPMDSIZE} -n` +newfs -O 1 -n /dev/md${SNAPMD} >/dev/null + +# Mount the memory disk +WORKDIR=${STATEDIR}/work +mkdir ${WORKDIR} +mount /dev/md${SNAPMD} ${WORKDIR} + +# Build a snapshot +SNAPDIR=${STATEDIR}/work/files +mkdir ${SNAPDIR} +sh -e s/treesnap-build.sh ${TREE}@${NEWREV} "${DESCRIBES}" ${WORKDIR} ${SNAPDIR} + +# Replace tarballs with "aliased" tarballs +if ! [ -z ${ALIASFILE} ]; then + sh -e s/alias-all.sh ${SNAPDIR} ${STATEDIR}/fileset/oldfiles \ + ${ALIASFILE} ${WORKDIR} +fi + +# Make sure we have the required describe files +sh -e s/describes-err.sh ${SNAPDIR} "${DESCRIBES}" + +# Collect metadata +sh -e s/treesnap-index.sh ${SNAPDIR} ${WORKDIR}/INDEX ${WORKDIR}/tINDEX + +# Hard-link compressed index file and remove the uncompressed file +ln ${SNAPDIR}/`sha256 -q ${WORKDIR}/INDEX`.gz ${WORKDIR}/INDEX.gz +rm ${WORKDIR}/INDEX + +# Create tag file +echo "portsnap|`date "+%s"`|`sha256 -q ${WORKDIR}/tINDEX`" > ${WORKDIR}/tag + +# Create tarball +tar -czf ${TARBALL} -C ${WORKDIR} tag tINDEX INDEX.gz files + +# Unmount and delete the snapshot disk +umount /dev/md${SNAPMD} +mdconfig -d -u ${SNAPMD} + +# Delete temporary directories +rmdir ${WORKDIR} + +# Report sucess +echo "`date`: Finished" Added: user/cperciva/portsnap-build/s/alias-all.sh ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/cperciva/portsnap-build/s/alias-all.sh Mon Mar 4 01:56:00 2013 (r247750) @@ -0,0 +1,40 @@ +#!/bin/sh -e + +# No user-serviceable parts +if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then + echo "Do not run $0 manually" + exit 1 +fi + +# usage: sh -e alias-all.sh SNAP OLDFILES ALIASFILE WORKDIR +SNAP="$1" +OLDFILES="$2" +ALIASFILE="$3" +WORKDIR="$4" + +# Report progress +echo "`date`: Reverting files to aliased versions" + +# Revert index +sort -k2 -t '|' ${SNAP}/INDEX | + join -1 2 -t '|' -o 1.1,1.2,2.2 - ${ALIASFILE} > ${WORKDIR}/INDEX.remap +cut -f 1,2 -d '|' ${WORKDIR}/INDEX.remap | + sort > ${WORKDIR}/INDEX.remap.delete +cut -f 1,3 -d '|' ${WORKDIR}/INDEX.remap | + sort > ${WORKDIR}/INDEX.remap.add +sort ${SNAP}/INDEX | + comm -23 - ${WORKDIR}/INDEX.remap.delete | + sort -k 1,1 -t '|' - ${WORKDIR}/INDEX.remap.add > ${SNAP}/INDEX.new +mv ${SNAP}/INDEX.new ${SNAP}/INDEX + +# Revert files +cut -f 2,3 -d '|' ${WORKDIR}/INDEX.remap | + tr '|' ' ' | while read OLD NEW; do + rm ${SNAP}/${OLD}.gz + cp ${OLDFILES}/${NEW}.gz ${SNAP}/ +done + +# Clean up +rm ${WORKDIR}/INDEX.remap +rm ${WORKDIR}/INDEX.remap.add +rm ${WORKDIR}/INDEX.remap.delete Added: user/cperciva/portsnap-build/s/alias-index.sh ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/cperciva/portsnap-build/s/alias-index.sh Mon Mar 4 01:56:00 2013 (r247750) @@ -0,0 +1,33 @@ +#!/bin/sh -e + +# No user-serviceable parts +if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then + echo "Do not run $0 manually" + exit 1 +fi + +# usage: sh -e alias-index.sh SNAP OLDFILES ALIASFILE WORKDIR +SNAP="$1" +OLDFILES="$2" +ALIASFILE="$3" +WORKDIR="$4" + +# Report progress +echo "`date`: Reverting files to aliased versions" + +# Revert index +sort -k2 -t '|' ${SNAP}/INDEX | + join -1 2 -t '|' -o 1.1,1.2,2.2 - ${ALIASFILE} > ${WORKDIR}/INDEX.remap +cut -f 1,2 -d '|' ${WORKDIR}/INDEX.remap | + sort > ${WORKDIR}/INDEX.remap.delete +cut -f 1,3 -d '|' ${WORKDIR}/INDEX.remap | + sort > ${WORKDIR}/INDEX.remap.add +sort ${SNAP}/INDEX | + comm -23 - ${WORKDIR}/INDEX.remap.delete | + sort -k 1,1 -t '|' - ${WORKDIR}/INDEX.remap.add > ${SNAP}/INDEX.new +mv ${SNAP}/INDEX.new ${SNAP}/INDEX + +# Clean up +rm ${WORKDIR}/INDEX.remap +rm ${WORKDIR}/INDEX.remap.add +rm ${WORKDIR}/INDEX.remap.delete Added: user/cperciva/portsnap-build/s/describes-err.sh ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/cperciva/portsnap-build/s/describes-err.sh Mon Mar 4 01:56:00 2013 (r247750) @@ -0,0 +1,19 @@ +#!/bin/sh -e + +# No user-serviceable parts +if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then + echo "Do not run $0 manually" + exit 1 +fi + +# usage: sh -e describes-err.sh SNAP DESCRIBES +SNAP="$1" +DESCRIBES="$2" + +# Check that the required describes files exist +for N in $DESCRIBES; do + if ! [ -f ${SNAP}/DESCRIBE.${N} ]; then + echo "DESCRIBE.${N} does not exist!" + exit 1 + fi +done Added: user/cperciva/portsnap-build/s/describes-fallback.sh ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/cperciva/portsnap-build/s/describes-fallback.sh Mon Mar 4 01:56:00 2013 (r247750) @@ -0,0 +1,24 @@ +#!/bin/sh -e + +# No user-serviceable parts +if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then + echo "Do not run $0 manually" + exit 1 +fi + +# usage: sh -e describes-fallback.sh SNAP DESCDIR DESCRIBES +SNAP="$1" +DESCDIR="$2" +DESCRIBES="$3" + +# For each DESCRIBE file... +for N in ${DESCRIBES}; do + # If a DESCRIBE failed... + if ! [ -f ${SNAP}/DESCRIBE.${N} ]; then + # ... use an old version ... + cp ${DESCDIR}/DESCRIBE.${N} ${SNAP}/DESCRIBE.${N} + else + # ... otherwise, store what we have for future reference. + cp ${SNAP}/DESCRIBE.${N} ${DESCDIR}/DESCRIBE.${N} + fi +done Added: user/cperciva/portsnap-build/s/describes-icbm.sh ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/cperciva/portsnap-build/s/describes-icbm.sh Mon Mar 4 01:56:00 2013 (r247750) @@ -0,0 +1,62 @@ +#!/bin/sh -e + +# No user-serviceable parts +if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then + echo "Do not run $0 manually" + exit 1 +fi + +# usage: sh -e describes-icbm.sh GOODREV BADREV ERRFILE +GOODREV="$1" +BADREV="$2" +ERRFILE="$3" + +# The first potentially faulty commit is GOODREV+1 +BADSTART=`expr "$GOODREV" + 1` + +# Standard From/To/Subject lines +cat <<EOF +From: ${INDEXMAIL_FROM} +To: ${INDEXMAIL_TO} +Subject: INDEX build breakage +EOF + +# CC people who might have broken the INDEX +jot - ${BADSTART} ${BADREV} | + while read REV; do + svn log -c ${REV} ${REPO} | + tail +2 | + head -1; +done | + cut -f 2 -d '|' | + sort -u | + tr -d ' ' | + lam -s 'CC: ' - -s '@freebsd.org' + +# Blank line and build failure output +echo +cat ${ERRFILE} + +# List potentially at-fault committers (again) and SVN history +echo +echo "Committers on the hook (CCed):" +jot - ${BADSTART} ${BADREV} | + while read REV; do + svn log -c ${REV} ${REPO} | + tail +2 | + head -1; +done | + cut -f 2 -d '|' | + sort -u | + tr -d ' ' +echo +echo "Latest SVN commits:" +svn log -r ${BADSTART}:${BADREV} ${REPO} + +# Final message about when emails are sent +cat <<EOF + +There may be different errors exposed by INDEX builds on other +branches, but no further emails will be sent until after the +INDEX next builds successfully on all branches. +EOF Added: user/cperciva/portsnap-build/s/describes-run.sh ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/cperciva/portsnap-build/s/describes-run.sh Mon Mar 4 01:56:00 2013 (r247750) @@ -0,0 +1,98 @@ +#!/bin/sh -e + +# No user-serviceable parts +if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then + echo "Do not run $0 manually" + exit 1 +fi + +# usage: sh -e describes-run.sh PORTSDISK WORLDTAR JAILDIR OSVERSION DESCFILE +PORTSDISK="$1" +WORLDTAR="$2" +JAILDIR="$3" +OSVERSION="$4" +DESCFILE="$5" + +# helper function +findruleset () { + jot 0 | + while read N; do + if ! devfs rule -s ${N} show | grep -q .; then + echo ${N} + break + fi + done +} + +# Create memory disks and format filesystems +JAILMD=`mdconfig -a -t swap -s ${JAILMDSIZE} -n` +TMPMD=`mdconfig -a -t swap -s ${TMPMDSIZE} -n` +newfs -O 1 -n /dev/md${JAILMD} >/dev/null +newfs -O 1 -n /dev/md${TMPMD} >/dev/null + +# Mount filesystems under jail mount point +mount -o noatime,nosuid /dev/md${JAILMD} ${JAILDIR} +mkdir ${JAILDIR}/tmp +mount -o noatime,nosuid,noexec /dev/md${TMPMD} ${JAILDIR}/tmp +chmod 1777 ${JAILDIR}/tmp +mkdir ${JAILDIR}/usr ${JAILDIR}/usr/ports +mount -o noatime,nosuid,noexec,ro ${PORTSDISK} ${JAILDIR}/usr/ports + +# Attach device filesystem with null, fd/*, and std* +mkdir ${JAILDIR}/dev +mount -t devfs devfs ${JAILDIR}/dev +RULESET=`findruleset` +devfs rule -s ${RULESET} add hide +devfs rule -s ${RULESET} add path null unhide +devfs rule -s ${RULESET} add path fd unhide +devfs rule -s ${RULESET} add path 'fd/*' unhide +devfs rule -s ${RULESET} add path 'std*' unhide +devfs -m ${JAILDIR}/dev ruleset ${RULESET} +devfs -m ${JAILDIR}/dev rule applyset + +# Extract world tarball +tar -xf ${WORLDTAR} -C ${JAILDIR} + +# Protect against naughtiness +mount -u -o noatime,nosuid,ro /dev/md${JAILMD} + +# Build the describes output +if env - PATH=${PATH} jail -c path=${JAILDIR} host.hostname=localhost \ + exec.jail_user=nobody exec.system_jail_user command=/bin/sh -e \ + > ${DESCFILE} <<- EOF + export __MAKE_CONF=/nonexistant + export OSVERSION=${OSVERSION} + export PORTOBJFORMAT=elf + export INDEX_TMPDIR=/tmp + export WRKDIRPREFIX=/tmp + export BUILDING_INDEX=1 + export LOCALBASE=/removeme/usr/local + export ECHO_MSG="echo >/dev/null" + cd /usr/ports && make describe -j ${JNUM} 1>&2 + cd /tmp && cat INDEX* | sed -e 's/ */ /g' -e 's/| */|/g' \ + -e 's/ *|/|/g' -e 's./removeme..g' | \ + sed -E -e ':x' -e 's|/[^/]+/\.\.||' -e 'tx' | \ + sort -k 1,1 -t '|' | tr -d '\r' +EOF +then + R=0 +else + R=1 +fi + +# Clean up +umount ${JAILDIR}/dev +umount ${JAILDIR}/tmp +umount ${JAILDIR}/usr/ports +umount ${JAILDIR} +mdconfig -d -u ${JAILMD} +mdconfig -d -u ${TMPMD} +devfs rule -s ${RULESET} delset + +# Test if make_index works +if [ ${R} = 0 ] && ! /usr/libexec/make_index ${DESCFILE} >/dev/null; then + R=1 +fi + +# Return success/failure +exit ${R} Added: user/cperciva/portsnap-build/s/describes-warn.sh ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/cperciva/portsnap-build/s/describes-warn.sh Mon Mar 4 01:56:00 2013 (r247750) @@ -0,0 +1,51 @@ +#!/bin/sh -e + +# No user-serviceable parts +if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then + echo "Do not run $0 manually" + exit 1 +fi + +# usage: sh -e describes-warn.sh SNAP NEWREV DESCDIR DESCRIBES +SNAP="$1" +NEWREV="$2" +DESCDIR="$3" +DESCRIBES="$4" + +# Haven't found any broken describes yet +BROKEN=0 + +# Check if any describes broke +for N in ${DESCRIBES}; do + # Did this one break? + if [ -f ${SNAP}/DESCRIBE.${N}.err ]; then + # Yep, remember this + BROKEN=1 + + # Launch an ICBM if this is new breakage + if [ -f ${DESCDIR}/indexok ]; then + sh -e s/describes-icbm.sh `cat ${DESCDIR}/indexok` \ + ${NEWREV} ${SNAP}/DESCRIBE.${N}.err | + sendmail -t + rm ${DESCDIR}/indexok + fi + + # Delete the error messages + rm ${SNAP}/DESCRIBE.${N}.err + fi +done + +# If the tree is non-broken but was previously broken, send an email +if [ ${BROKEN} = 0 ] && ! [ -f ${DESCDIR}/indexok ]; then + sendmail -t <<- EOF + From: ${INDEXMAIL_FROM} + To: ${INDEXMAIL_TO} + Subject: INDEX build fixed + + EOF +fi + +# If the tree is non-broken, record the new index-ok-at SVN # +if [ ${BROKEN} = 0 ]; then + echo ${NEWREV} > ${DESCDIR}/indexok +fi Added: user/cperciva/portsnap-build/s/fileset-addfiles.sh ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/cperciva/portsnap-build/s/fileset-addfiles.sh Mon Mar 4 01:56:00 2013 (r247750) @@ -0,0 +1,20 @@ +#!/bin/sh -e + +# No user-serviceable parts +if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then + echo "Do not run $0 manually" + exit 1 +fi + +# usage: sh -e fileset-addfiles.sh FSETDIR SNAP +FSETDIR="$1" +SNAP="$2" + +# Report progress +echo "`date`: Copying files into /oldfiles/" + +# Copy data files -- we don't need metadata files since we +# don't generate patches for those here +cut -f 2 -d '|' ${FSETDIR}/filedb.news | + lam -s "${SNAP}/" - -s '.gz' | + xargs -J % cp % ${FSETDIR}/oldfiles/ Added: user/cperciva/portsnap-build/s/fileset-findnew-cleanup.sh ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/cperciva/portsnap-build/s/fileset-findnew-cleanup.sh Mon Mar 4 01:56:00 2013 (r247750) @@ -0,0 +1,13 @@ +#!/bin/sh -e + +# No user-serviceable parts +if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then + echo "Do not run $0 manually" + exit 1 +fi + +# usage: sh -e fileset-findnew-cleanup.sh FSETDIR +FSETDIR="$1" + +# Clean up new-files lists generated be fileset-findnew.sh +rm ${FSETDIR}/filedb.news ${FSETDIR}/metadb.news Added: user/cperciva/portsnap-build/s/fileset-findnew.sh ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/cperciva/portsnap-build/s/fileset-findnew.sh Mon Mar 4 01:56:00 2013 (r247750) @@ -0,0 +1,36 @@ +#!/bin/sh -e + +# No user-serviceable parts +if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then + echo "Do not run $0 manually" + exit 1 +fi + +# usage: sh -e fileset-findnew.sh FSETDIR INDEX tINDEX TMP +FSETDIR="$1" +INDEX="$2" +tINDEX="$3" +TMP="$4" + +# Report progress +echo "`date`: Identifying new files" + +# Find new port tarballs +sort -k 3,3 -t '|' ${FSETDIR}/filedb > ${TMP}/filedb.sorted +sort -k 2,2 -t '|' ${INDEX} | + join -1 3 -2 2 -t '|' -v 2 ${TMP}/filedb.sorted - | + sort -k 1,1 -t '|' > ${FSETDIR}/filedb.news + +# Report new files +echo "New files:" +cut -f 1 -d '|' < ${FSETDIR}/filedb.news | + lam -s ' ' - + +# Find new metadata files +sort -k 3,3 -t '|' ${FSETDIR}/metadb > ${TMP}/metadb.sorted +sort -k 2,2 -t '|' ${tINDEX} | + join -1 3 -2 2 -t '|' -v 2 ${TMP}/metadb.sorted - | + sort -k 1,1 -t '|' > ${FSETDIR}/metadb.news + +# Clean up temporary files +rm ${TMP}/filedb.sorted ${TMP}/metadb.sorted Added: user/cperciva/portsnap-build/s/fileset-mergedb.sh ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/cperciva/portsnap-build/s/fileset-mergedb.sh Mon Mar 4 01:56:00 2013 (r247750) @@ -0,0 +1,45 @@ +#!/bin/sh -e + +# No user-serviceable parts +if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then + echo "Do not run $0 manually" + exit 1 +fi + +# usage: sh -e fileset-mergedb.sh FSETDIR SNAPDATE INDEX tINDEX TMP +FSETDIR="$1" +SNAPDATE="$2" +INDEX="$3" +tINDEX="$4" +TMP="$5" + +# Report progress +echo "`date`: Updating databases" + +# The filedb and metadb files contain lines of the form +# FILENAME|TIMESTAMP|HASH +# where TIMESTAMP is the *most recent* SNAPDATE for which the data +# associated with FILENAME is HASH.gz (and has SHA256 hash HASH). +# We use this to tell us which binary patches to generate, which +# also tells us which old data files to keep around; the mirrors +# use this to prune the data they publihsh and figure out which +# metadata patches they should generate. + +# Update filedb using INDEX +sed -e "s/|/|${SNAPDATE}|/" ${INDEX} | + sort -k 3,3 -t '|' > ${TMP}/new.dated +sort -k 3,3 -t '|' ${FSETDIR}/filedb | + join -1 3 -2 3 -t '|' -v 1 - ${TMP}/new.dated | + sort -k 1,1 -t '|' - ${TMP}/new.dated > ${FSETDIR}/filedb.tmp +mv ${FSETDIR}/filedb.tmp ${FSETDIR}/filedb + +# Update metadb using tINDEX +sed -e "s/|/|${SNAPDATE}|/" ${tINDEX} | + sort -k 3,3 -t '|' > ${TMP}/new.dated +sort -k 3,3 -t '|' ${FSETDIR}/metadb | + join -1 3 -2 3 -t '|' -v 1 - ${TMP}/new.dated | + sort -k 1,1 -t '|' - ${TMP}/new.dated > ${FSETDIR}/metadb.tmp +mv ${FSETDIR}/metadb.tmp ${FSETDIR}/metadb + +# Clean up +rm ${TMP}/new.dated Added: user/cperciva/portsnap-build/s/fileset-mirrorlists.sh ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/cperciva/portsnap-build/s/fileset-mirrorlists.sh Mon Mar 4 01:56:00 2013 (r247750) @@ -0,0 +1,19 @@ +#!/bin/sh -e + +# No user-serviceable parts +if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then + echo "Do not run $0 manually" + exit 1 +fi + +# usage: sh -e fileset-mirrorlists.sh FSETDIR STAGE +FSETDIR="$1" +STAGE="$2" + +# Publish file lists +echo "`date`: Copying file lists into staging area" +gzip -c < ${FSETDIR}/filedb > ${STAGE}/bl.gz +gzip -c < ${FSETDIR}/metadb > ${STAGE}/tl.gz +cut -f 2 -d '|' ${FSETDIR}/extradb | + sort -u | + gzip -c > ${STAGE}/el.gz Added: user/cperciva/portsnap-build/s/fileset-mkpatches.sh ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/cperciva/portsnap-build/s/fileset-mkpatches.sh Mon Mar 4 01:56:00 2013 (r247750) @@ -0,0 +1,32 @@ +#!/bin/sh -e + +# No user-serviceable parts +if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then + echo "Do not run $0 manually" + exit 1 +fi + +# usage: sh -e fileset-mkpatches.sh FSETDIR SNAP STAGE TMP +FSETDIR="$1" +SNAP="$2" +STAGE="$3" +TMP="$4" + +# Report progress +echo "`date`: Building patches" + +# Figure out which "OLD NEW" pairs we need +sort -k 1,1 -t '|' ${FSETDIR}/filedb | + join -t '|' -o 1.3,2.2 - ${FSETDIR}/filedb.news | + tr '|' ' ' > ${TMP}/patches + +# Build the patches +while read X Y; do + gunzip -c ${FSETDIR}/oldfiles/${X}.gz > ${TMP}/${X} + gunzip -c ${SNAP}/${Y}.gz > ${TMP}/${Y} + bsdiff ${TMP}/${X} ${TMP}/${Y} ${STAGE}/bp/${X}-${Y} + rm ${TMP}/${X} ${TMP}/${Y} +done < ${TMP}/patches + +# Clean up temporary file +rm ${TMP}/patches Added: user/cperciva/portsnap-build/s/fileset-prune.sh ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/cperciva/portsnap-build/s/fileset-prune.sh Mon Mar 4 01:56:00 2013 (r247750) @@ -0,0 +1,45 @@ +#!/bin/sh -e + +# No user-serviceable parts +if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then + echo "Do not run $0 manually" + exit 1 +fi + +# usage: sh -e fileset-prune.sh SNAPDATE FSETDIR TMP +SNAPDATE="$1" +FSETDIR="$2" +TMP="$3" + +# Report progress +echo "`date`: Removing old files and database entries" + +# Sort list of files so we can use comm(1) on it +sort ${FSETDIR}/filedb > ${TMP}/filedb.sorted + +# Find lines corresponding to files we don't want any more +awk -F \| -v cutoff=`expr ${SNAPDATE} - ${MAXAGE_DATA}` \ + '{ if ($2 < cutoff) { print } }' ${TMP}/filedb.sorted > ${TMP}/filedb.olds + +# Delete old files +cut -f 3 -d '|' ${TMP}/filedb.olds | + grep -E '^[0-9a-f]{64}$' | + lam -s "${FSETDIR}/oldfiles/" - -s '.gz' | + xargs rm -f + +# Construct a new old-files list +comm -23 ${TMP}/filedb.sorted ${TMP}/filedb.olds | + sort -k 1,1 -t '|' > ${FSETDIR}/filedb.tmp +mv ${FSETDIR}/filedb.tmp ${FSETDIR}/filedb + +# Remove old lines from metadb and extradb +echo "`date`: Removing old metadata and extra database entries" +awk -F \| -v cutoff=`expr ${SNAPDATE} - ${MAXAGE_META}` \ + '{ if ($2 >= cutoff) { print } }' ${FSETDIR}/metadb > ${FSETDIR}/metadb.tmp +mv ${FSETDIR}/metadb.tmp ${FSETDIR}/metadb +awk -F \| -v cutoff=`expr ${SNAPDATE} - ${MAXAGE_EXTRA}` \ + '{ if ($1 >= cutoff) { print } }' ${FSETDIR}/extradb > ${FSETDIR}/extradb.tmp +mv ${FSETDIR}/extradb.tmp ${FSETDIR}/extradb + +# Remove temporary files +rm ${TMP}/filedb.sorted ${TMP}/filedb.olds Added: user/cperciva/portsnap-build/s/fileset-publish-tindex.sh ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/cperciva/portsnap-build/s/fileset-publish-tindex.sh Mon Mar 4 01:56:00 2013 (r247750) @@ -0,0 +1,22 @@ +#!/bin/sh -e + +# No user-serviceable parts +if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then + echo "Do not run $0 manually" + exit 1 +fi + +# usage: sh -e fileset-publish-tindex.sh FSETDIR SNAPDATE tINDEX STAGE +FSETDIR="$1" +SNAPDATE="$2" +tINDEX="$3" +STAGE="$4" + +# Figure out the tINDEX hash +SNAPSHOTHASH=`sha256 -q ${tINDEX}` + +# Copy the tINDEX file into place in the staging directory +cp ${tINDEX} ${STAGE}/t/${SNAPSHOTHASH} + +# Add to our database +echo "${SNAPDATE}|t/${SNAPSHOTHASH}" >> ${FSETDIR}/extradb Added: user/cperciva/portsnap-build/s/fileset-publish.sh ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/cperciva/portsnap-build/s/fileset-publish.sh Mon Mar 4 01:56:00 2013 (r247750) @@ -0,0 +1,20 @@ +#!/bin/sh -e + +# No user-serviceable parts +if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then + echo "Do not run $0 manually" + exit 1 +fi + +# usage: sh -e fileset-publish.sh FSETDIR SNAP STAGE +FSETDIR="$1" +SNAP="$2" +STAGE="$3" + +# Report progress +echo "`date`: Copying files into staging area" + +# Copy new files +cut -f 2 -d '|' ${FSETDIR}/filedb.news ${FSETDIR}/metadb.news | + lam -s "${SNAP}/" - -s '.gz' | + xargs -J % cp % ${STAGE}/f/ Added: user/cperciva/portsnap-build/s/fileset-snap.sh ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/cperciva/portsnap-build/s/fileset-snap.sh Mon Mar 4 01:56:00 2013 (r247750) @@ -0,0 +1,47 @@ +#!/bin/sh -e + +# No user-serviceable parts +if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then + echo "Do not run $0 manually" + exit 1 +fi + +# usage: sh -e fileset-snap.sh SNAPDATE FSETDIR SNAP INDEX tINDEX STAGE WORKDIR +SNAPDATE="$1" +FSETDIR="$2" +SNAP="$3" +INDEX="$4" +tINDEX="$5" +STAGE="$6" +WORKDIR="$7" + +# Create temporary directory +TMPDIR=${WORKDIR}/tmp +mkdir ${TMPDIR} + +# Prune old files from the file set +sh -e s/fileset-prune.sh ${SNAPDATE} ${FSETDIR} ${TMP} + +# Figure out which files in this snapshot are new +sh -e s/fileset-findnew.sh ${FSETDIR} ${INDEX} ${tINDEX} ${TMP} + +# Build binary patches and place them into the staging directory +sh -e s/fileset-mkpatches.sh ${FSETDIR} ${SNAP} ${STAGE} ${TMP} + +# Make copies of old files so we can use them for future patch-building +sh -e s/fileset-addfiles.sh ${FSETDIR} ${SNAP} + +# Copy new files into the staging directory +sh -e s/fileset-publish.sh ${FSETDIR} ${SNAP} ${STAGE} + *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201303040156.r241u0pZ075652>