Date: Thu, 6 Mar 2014 02:48:55 +0000 (UTC) From: Glen Barber <gjb@FreeBSD.org> To: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: svn commit: r262820 - user/gjb/thermite Message-ID: <201403060248.s262mt1b042116@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: gjb Date: Thu Mar 6 02:48:54 2014 New Revision: 262820 URL: http://svnweb.freebsd.org/changeset/base/262820 Log: As result of a number of inquiries, add the release.sh wrapper script to my user directory for public consumption. Thermite is a pyrotechnic composition of shell script and zfs. When executed, it generates a significant amount of heat. Here's how it works: - Create the configuration files for each build to be run. The file name format is: {rev}-{arch}-{kernel}-{type}, where rev is the revision to build (i.e., 10, 9), arch is the TARGET_ARCH to be built, kernel is the KERNCONF, and type is the type of build ('snap' or 'release'). An example is included as 11-amd64-GENERIC-snap.conf. - Edit master.conf for the build environment, in particular, make sure zfs_root and zfs_mount are updated if the builder is a zfs-enabled machine. If the builder is zfs-enabled, a number of redundant steps are skipped, and 'zfs clone' is used to shorten overall times. - Run: # ./thermite.sh -c ./master.conf - Cross your fingers. The mk-vmimage.sh script uses qemu-img to create the VM images that are published on ftp.FreeBSD.org. It is also included, as parts of thermite.sh call it after successful builds. The master.conf file is effectively what is used for the weekly snapshot builds. Sponsored by: The FreeBSD Foundation Added: user/gjb/thermite/ user/gjb/thermite/11-amd64-GENERIC-snap.conf (contents, props changed) user/gjb/thermite/master.conf (contents, props changed) user/gjb/thermite/mk-vmimage.sh (contents, props changed) user/gjb/thermite/thermite.sh (contents, props changed) Added: user/gjb/thermite/11-amd64-GENERIC-snap.conf ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/gjb/thermite/11-amd64-GENERIC-snap.conf Thu Mar 6 02:48:54 2014 (r262820) @@ -0,0 +1,27 @@ +#!/bin/sh +# +# $FreeBSD$ +# + +# Sample configuration file for building a head/ snapshot using +# thermite.sh. + +CHROOTDIR="/scratch/11-amd64-GENERIC-snap" +CHROOTBUILD_SKIP=1 +SVNROOT="svn://svn.FreeBSD.org/" +SRCBRANCH="base/head@${_SVNREV}" +DOCBRANCH="doc/head@rHEAD" +PORTBRANCH="ports/head@rHEAD" +WORLD_FLAGS="-j10" +KERNEL_FLAGS="-j8" +KERNEL="GENERIC" +TARGET="amd64" +TARGET_ARCH="amd64" +NODOC=1 +NOPORTS=1 + +# svn revision to build. Do not include leading 'r'. +_SVNREV="262657" +# Date the build was started. +DATE="20140301" + Added: user/gjb/thermite/master.conf ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/gjb/thermite/master.conf Thu Mar 6 02:48:54 2014 (r262820) @@ -0,0 +1,35 @@ +#!/bin/sh +# +# $FreeBSD$ +# + +# Thermite is a pyrotechnic composition of shell script and zfs. +# When executed, it generates a significant amount of heat. +# This is its configuration file. + +use_zfs=1 +emailgoesto= +scriptdir="$(dirname $(realpath ${0}))" +mkdir -p ${scriptdir}/../release +srcdir="$(realpath ${scriptdir}/../release)" +mkdir -p ${scriptdir}/../logs +logdir="$(realpath ${scriptdir}/../logs)" +mkdir -p ${scriptdir}/../chroots +chroots="$(realpath ${scriptdir}/../chroots)" + +heads="11" +stables="10 9" + +revs="${heads} ${stables}" +archs="amd64 i386 ia64 powerpc powerpc64 sparc64 armv6" +types="snap" + +x86_kernels="GENERIC" +armv6_kernels="BEAGLEBONE RPI-B PANDABOARD WANDBOARD-QUAD" + +kernels="${x86_kernels} GENERIC64 ${armv6_kernels}" + +zfs_root="zroot" +zfs_mount="releng" +zfs_parent="${zfs_root}/${zfs_mount}" + Added: user/gjb/thermite/mk-vmimage.sh ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/gjb/thermite/mk-vmimage.sh Thu Mar 6 02:48:54 2014 (r262820) @@ -0,0 +1,157 @@ +#!/bin/sh +#- +# Copyright (c) 2013, 2014 The FreeBSD Foundation +# Copyright (c) 2012, 2013 Glen Barber +# All rights reserved. +# +# Portions of this software were developed by Glen Barber +# under sponsorship from the FreeBSD Foundation. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# Script to create the VM images available on ftp.FreeBSD.org. +# +# $FreeBSD$ +# + +PATH="/bin:/sbin:/usr/bin:/usr/sbin" +export PATH + +usage () { + echo "$(basename ${0}) -c /path/to/release.conf" + exit 1 +} + +while getopts c: opt; do + case ${opt} in + c) + VM_CONFIG="${OPTARG}" + if [ ! -e "${VM_CONFIG}" ]; then + echo -n "ERROR: Configuration file ${VM_CONFIG}" + echo " does not exist." + exit 1 + fi + . ${VM_CONFIG} + ;; + \?) + usage + ;; + esac +done + +# Force use of configuration file +if [ "X${VM_CONFIG}" = "X" ]; then + usage +fi + +check_prereq () { + rc=0 + if [ ! -x "/usr/local/bin/qemu-img" ]; then + rc=1 + fi + return ${rc} +} + +VM_TARGET_ARCH=$(echo ${__CONFIG_NAME} | cut -f 2 -d -) + +case ${VM_TARGET_ARCH} in + # For now, only create amd64 and i386 vm images. + i386|amd64) + # Ok to create vm image + ;; + *) + exit 0 + ;; +esac + +case ${KERNEL} in + GENERIC) + ;; + *) + exit 0 + ;; +esac + +check_prereq || exit 0 + +# Use the build environment to get the branch revision (i.e., 10.0) and branch +# (i.e., -CURRENT, -STABLE) for the vm image name. +VM_IMAGE_NAME=$(make -C ${CHROOTDIR}/usr/src/release -V REVISION -V BRANCH | tr '\n' '-') +# Prefix the image name with OS name, and suffix with the vm architecture. +VM_IMAGE_NAME="$(uname -s)-${VM_IMAGE_NAME}${VM_TARGET_ARCH}" + +mkdir -p ${CHROOTDIR}/vmimage ${CHROOTDIR}/vmimage/mnt + +# This should only ever happen if the script is being run again after failure. +if [ -e "${CHROOTDIR}/vmimage/${VM_IMAGE_NAME}.disk" ]; then + rm -f "${CHROOTDIR}/vmimage/${VM_IMAGE_NAME}.disk" +fi + +touch ${CHROOTDIR}/vmimage/${VM_IMAGE_NAME}.disk +truncate -s 20G ${CHROOTDIR}/vmimage/${VM_IMAGE_NAME}.disk +mddev=$(mdconfig -a -t vnode -f ${CHROOTDIR}/vmimage/${VM_IMAGE_NAME}.disk) +gpart create -s gpt /dev/${mddev} +gpart add -t freebsd-boot -s 512k -l bootfs /dev/${mddev} +gpart bootcode -b /boot/pmbr -p /boot/gptboot -i 1 /dev/${mddev} +gpart add -t freebsd-swap -s 1G -l swapfs /dev/${mddev} +gpart add -t freebsd-ufs -l rootfs /dev/${mddev} +newfs /dev/${mddev}p3 +mount /dev/${mddev}p3 ${CHROOTDIR}/vmimage/mnt + +# Errors here are ok, 'set -e' is re-enabled below again. +set +e +mount -t devfs devfs ${CHROOTDIR}/dev +chroot ${CHROOTDIR} make -s -C /usr/src DESTDIR=/vmimage/mnt \ + installworld installkernel distribution +echo "# Custom /etc/fstab for FreeBSD VM images" \ + > ${CHROOTDIR}/vmimage/mnt/etc/fstab +echo "/dev/gpt/rootfs / ufs rw 2 2" \ + >> ${CHROOTDIR}/vmimage/mnt/etc/fstab +echo "/dev/gpt/swapfs none swap sw 0 0" \ + >> ${CHROOTDIR}/vmimage/mnt/etc/fstab +# Make sure we wait until the md(4) is unmounted before destroying it. +while ! umount /dev/${mddev}p3; do + sleep 1 +done +mdconfig -d -u ${mddev} +while ! umount ${CHROOTDIR}/dev; do + sleep 1 +done +set -e +diskformats="vmdk vpc qcow2" +for f in ${diskformats}; do + _f=${f} + case ${_f} in + vpc) + _f=vhd + ;; + *) + ;; + esac + /usr/local/bin/qemu-img convert -O ${f} ${CHROOTDIR}/vmimage/${VM_IMAGE_NAME}.disk \ + ${CHROOTDIR}/vmimage/${VM_IMAGE_NAME}.${_f} + xz ${CHROOTDIR}/vmimage/${VM_IMAGE_NAME}.${_f} +done +cd ${CHROOTDIR}/vmimage +sha256 FreeBSD*.xz > CHECKSUM.SHA256 +md5 FreeBSD*.xz > CHECKSUM.MD5 + Added: user/gjb/thermite/thermite.sh ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/gjb/thermite/thermite.sh Thu Mar 6 02:48:54 2014 (r262820) @@ -0,0 +1,397 @@ +#!/bin/sh +#- +# Copyright (c) 2013, 2014 The FreeBSD Foundation +# Copyright (c) 2012, 2013 Glen Barber +# All rights reserved. +# +# Portions of this software were developed by Glen Barber +# under sponsorship from the FreeBSD Foundation. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# Thermite is a pyrotechnic composition of shell script and zfs. +# When executed, it generates a significant amount of heat. +# Wrapper script for release.sh to automate mass release builds. +# +# $FreeBSD$ +# + +usage() { + echo "$(basename ${0}) -c /path/to/configuration/file" + exit 1 +} + +info() { + out="${@}" + printf "INFO:\t${out}\n" >/dev/stdout + unset out +} + +verbose() { + if [ -z ${debug} ] || [ ${debug} -eq 0 ]; then + return 0 + fi + out="${@}" + printf "DEBUG:\t${out}\n" >/dev/stdout + unset out +} + +runcmd() { + verbose "${rev} ${arch} ${kernel} ${type}" + eval "$@" +} + +loop_revs() { + verbose "loop_revs() start" + for rev in ${revs}; do + verbose "loop_revs() arguments: $@" + eval runcmd "$@" + done + unset rev + verbose "loop_revs() stop" +} + +loop_archs() { + verbose "loop_archs() start" + for arch in ${archs}; do + verbose "loop_archs() arguments: $@" + eval runcmd "$@" + done + unset arch + verbose "loop_archs() stop" +} + +loop_kernels() { + verbose "loop_kernels() start" + for kernel in ${kernels}; do + verbose "loop_kernels() arguments: $@" + eval runcmd "$@" + done + unset kernel + verbose "loop_kernel() stop" +} + +loop_types() { + verbose "loop_types() start" + for type in ${types}; do + verbose "loop_types() arguments: $@" + eval runcmd "$@" + done + unset type + verbose "loop_types() stop" +} + +runall() { + verbose "runall() start" + verbose "runall() arguments: $@" + eval loop_revs loop_archs loop_kernels loop_types "$@" + verbose "runall() stop" +} + +check_use_zfs() { + if [ -z ${use_zfs} ]; then + return 1 + fi + if [ ! -c /dev/zfs ]; then + return 1 + fi + return 0 +} + +source_config() { + local configfile + configfile="${scriptdir}/${rev}-${arch}-${kernel}-${type}.conf" + if [ ! -e "${configfile}" ]; then + return 1 + fi + . "${configfile}" + return 0 +} + +zfs_mount_tree() { + source_config || return 0 + _tree=${1} + [ -z ${_tree} ] && return 0 + seed_src= + case ${_tree} in + src) + seed_src=1 + ;; + doc) + [ ! -z ${NODOC} ] && return 0 + ;; + ports) + [ ! -z ${NOPORTS} ] && return 0 + ;; + *) + info "Unknown source tree type: ${_tree}" + return 0 + ;; + esac + _clone="${zfs_parent}/${rev}-${_tree}-${type}" + _mount="/${zfs_mount}/${rev}-${arch}-${kernel}-${type}" + _target="${zfs_parent}/${rev}-${arch}-${kernel}-${type}-${_tree}" + info "Cloning ${_clone}@clone to ${_target}" + zfs clone -p -o mountpoint=${_mount}/usr/${_tree} \ + ${_clone}@clone ${_target} + if [ ! -z ${seed_src} ]; then + # Only create chroot seeds for x86. + if [ "${arch}" = "amd64" ] || [ "${arch}" = "i386" ]; then + _seedmount=${chroots}/${rev}/${arch}/${type} + _seedtarget="${zfs_parent}/${rev}-${arch}-${type}-chroot" + zfs clone -p -o mountpoint=${_seedmount} \ + ${_clone}@clone ${_seedtarget} + fi + fi + unset _clone _mount _target _tree _seedmount _seedtarget +} + +zfs_create_tree() { + source_config || return 0 + _tree=${1} + [ -z ${_tree} ] && return 0 + [ ! -z $(eval echo \${zfs_${_tree}_seed_${rev}_${type}}) ] && return 0 + case ${_tree} in + src) + _svnsrc="${SVNROOT}/${SRCBRANCH}" + ;; + doc) + [ ! -z ${NODOC} ] && return 0 + _svnsrc="${SVNROOT}/${DOCBRANCH}" + ;; + ports) + [ ! -z ${NOPORTS} ] && return 0 + _svnsrc="${SVNROOT}/${PORTBRANCH}" + ;; + *) + info "Unknown source tree type: ${_tree}" + return 0 + ;; + esac + _clone="${zfs_parent}/${rev}-${_tree}-${type}" + _mount="/${zfs_mount}/${rev}-${_tree}-${type}" + info "Creating ${_clone}" + zfs create -o atime=off -o mountpoint=${_mount} ${_clone} + info "Source checkout ${_svnsrc} to ${_mount}" + svn co -q ${_svnsrc} ${_mount} + info "Creating ZFS snapshot ${_clone}@clone" + zfs snapshot ${_clone}@clone + eval zfs_${_tree}_seed_${rev}_${type}=1 + unset _clone _mount _tree _svnsrc +} + +zfs_bootstrap() { + [ -z ${use_zfs} ] && return 0 + runall zfs_create_tree src + runall zfs_create_tree ports + runall zfs_create_tree doc + runall zfs_mount_tree src + runall zfs_mount_tree ports + runall zfs_mount_tree doc + zfs_bootstrap_done=1 +} + +prebuild_setup() { + info "Creating ${logdir}" + info "Creating ${srcdir}" + mkdir -p "${logdir}" "${srcdir}" + info "Checking out src/release to ${srcdir}" + svn co -q --force svn://svn.freebsd.org/base/head/release ${srcdir} + info "Reverting any changes to ${srcdir}/release.sh" + svn revert ${srcdir}/release.sh +} + +# Email log output when a stage has completed +send_logmail() { + [ -z "${emailgoesto}" ] && return 0 + local _body + local _subject + _body="${1}" + _subject="${2}" + tail -n 10 "${_body}" | \ + mail -s "${_subject} done" ${emailgoesto} + return 0 +} + +# Run the release builds. +build_release() { + _build="${rev}-${arch}-${kernel}-${type}" + _conf="${scriptdir}/${_build}.conf" + [ ! -e ${_conf} ] && return 0 + info "Building release: ${_build}" + printenv >> ${logdir}/${_build}.log + env -i /bin/sh ${srcdir}/release.sh -c ${_conf} \ + >> ${logdir}/${_build}.log 2>&1 + + send_logmail ${logdir}/${_build}.log ${_build} + + # Short circuit to skip vm image creation for non-x86 architectures. + # Also recreate the memstick.img for i386 while here. + case ${arch} in + amd64) + ;; + i386) + /bin/sh ${scriptdir}/remake-memstick.sh \ + -c ${_conf} >> ${logdir}/${_build}.log + ;; + *) + return 0 + ;; + esac + case ${kernel} in + GENERIC) + info "Building vm image: ${_build}" + env -i /bin/sh ${scriptdir}/mk-vmimage.sh -c ${_conf} \ + >> ${logdir}/${_build}.log 2>&1 + ;; + *) + return 0 + ;; + esac + send_logmail ${logdir}/${_build}.log ${_build} + unset _build _conf +} + +check_x86() { + case ${arch} in + amd64|i386) + return 0 + ;; + *) + return 1 + ;; + esac +} + +# Install amd64/i386 "seed" chroots for all branches being built. +install_chroots() { + source_config || return 0 + if [ ${rev} -le 8 ]; then + info "This script does not support rev=${rev}" + return 0 + fi + case ${arch} in + i386) + _chrootarch="i386" + ;; + *) + _chrootarch="amd64" + ;; + esac + _build="${rev}-${arch}-${kernel}-${type}" + _dest="${__WRKDIR_PREFIX}/${_build}" + _srcdir="${chroots}/${rev}/${_chrootarch}/${type}" + _objdir="${chroots}/${rev}-obj/${_chrootarch}/${type}" + info "Creating ${_dest}" + mkdir -p "${_dest}" + info "Installing ${_dest}" + env MAKEOBJDIRPREFIX=${_objdir} \ + make -C ${_srcdir} \ + __MAKE_CONF=/dev/null SRCCONF=/dev/null \ + TARGET=${_chrootarch} TARGET_ARCH=${_chrootarch} \ + DESTDIR=${_dest} \ + installworld distribution 2>&1 >> \ + ${logdir}/${_build}.log + unset _build _dest _objdir _srcdir +} + +# Build amd64/i386 "seed" chroots for all branches being built. +build_chroots() { + source_config || return 0 + if [ ${rev} -le 8 ]; then + info "This script does not support rev ${rev}" + return 0 + fi + # Only build for amd64 and i386. + check_x86 || return 0 + # Building stable/9 on head/ is particularly race-prone when + # building make(1) for the first time. I have no idea why. + # Apply duct tape for now. + if [ ${rev} -lt 10 ]; then + __makecmd="make" + else + __makecmd="bmake" + fi + case ${arch} in + i386) + _chrootarch="i386" + ;; + *) + _chrootarch="amd64" + ;; + esac + [ ! -z $(eval echo \${chroot_${_chrootarch}_build_${rev}_${type}}) ] && return 0 + _build="${rev}-${_chrootarch}-${type}" + _srcdir="${chroots}/${rev}/${_chrootarch}/${type}" + _objdir="${chroots}/${rev}-obj/${_chrootarch}/${type}" + mkdir -p "${_srcdir}" + # Source the build configuration file to get + # the SRCBRANCH to use + if [ -z ${zfs_bootstrap_done} ]; then + # Skip svn checkout, the trees are there. + info "SVN checkout ${SRCBRANCH} for ${_chrootarch} ${type}" + svn co -q ${SVNROOT}/${SRCBRANCH} \ + ${_srcdir} \ + 2>&1 >> ${logdir}/${_build}.log + fi + info "Building ${_srcdir} make(1)" + env MAKEOBJDIRPREFIX=${_objdir} \ + make -C ${_srcdir} ${WORLD_FLAGS} \ + __MAKE_CONF=/dev/null SRCCONF=/dev/null \ + TARGET=${_chrootarch} TARGET_ARCH=${_chrootarch} \ + ${__makecmd} 2>&1 >> \ + ${logdir}/${_build}.log + info "Building ${_srcdir} world" + env MAKEOBJDIRPREFIX=${_objdir} \ + make -C ${_srcdir} ${WORLD_FLAGS} \ + __MAKE_CONF=/dev/null SRCCONF=/dev/null \ + TARGET=${_chrootarch} TARGET_ARCH=${_chrootarch} \ + buildworld 2>&1 >> \ + ${logdir}/${_build}.log + eval chroot_${_chrootarch}_build_${rev}_${type}=1 + unset _build _dest _objdir _srcdir +} + +main() { + while getopts c: opt; do + case ${opt} in + c) + CONF=${OPTARG} + [ -e ${CONF} ] && . $(realpath ${CONF}) + ;; + \?) + usage + ;; + esac + done + shift $(($OPTIND - 1)) + [ -z ${CONF} ] && usage + zfs_bootstrap_done= + prebuild_setup + zfs_bootstrap + runall build_chroots + runall install_chroots + runall build_release +} + +main "$@" +
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201403060248.s262mt1b042116>