From owner-svn-src-user@FreeBSD.ORG Wed Mar 26 10:17:19 2014 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [8.8.178.115]) (using TLSv1 with cipher ADH-AES256-SHA (256/256 bits)) (No client certificate requested) by hub.freebsd.org (Postfix) with ESMTPS id 07A90C47; Wed, 26 Mar 2014 10:17:19 +0000 (UTC) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mx1.freebsd.org (Postfix) with ESMTPS id DB88B26A; Wed, 26 Mar 2014 10:17:18 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.8/8.14.8) with ESMTP id s2QAHIa3020888; Wed, 26 Mar 2014 10:17:18 GMT (envelope-from jmmv@svn.freebsd.org) Received: (from jmmv@localhost) by svn.freebsd.org (8.14.8/8.14.8/Submit) id s2QAHIRS020885; Wed, 26 Mar 2014 10:17:18 GMT (envelope-from jmmv@svn.freebsd.org) Message-Id: <201403261017.s2QAHIRS020885@svn.freebsd.org> From: Julio Merino Date: Wed, 26 Mar 2014 10:17:18 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: svn commit: r263767 - in user/jmmv/autotest/node: . configs/kyua1 configs/kyua2 X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.17 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 26 Mar 2014 10:17:19 -0000 Author: jmmv Date: Wed Mar 26 10:17:17 2014 New Revision: 263767 URL: http://svnweb.freebsd.org/changeset/base/263767 Log: Make setup.sh allow a controlled deployment of specific autotest versions. With this change, setup.sh will fetch a separate copy of autotest (and its dependencies!) based on a revision specification in a per-host configuration file. This separate copy is the one used on the worker node to run the tests. Additionally, setup.sh will register itself as a cron job to maintain the autotest code and all configuration files up-to-date. The goal of this change is to permit having one of the kyua[123] workers as a canary of configuration changes while the others remain stable. This is much necessary as I've been known to routinely break the nodes when I manually push to them untested changes. Added: user/jmmv/autotest/node/configs/kyua1/host.conf (contents, props changed) user/jmmv/autotest/node/configs/kyua2/host.conf (contents, props changed) Modified: user/jmmv/autotest/node/setup.sh Directory Properties: user/jmmv/autotest/node/ (props changed) Added: user/jmmv/autotest/node/configs/kyua1/host.conf ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/jmmv/autotest/node/configs/kyua1/host.conf Wed Mar 26 10:17:17 2014 (r263767) @@ -0,0 +1,12 @@ +# $FreeBSD$ +# +# Configuration to bootstrap autotest on this host and to maintain the host +# up-to-date with changes to the autotest code, its configuration files and +# its dependencies. +# + +AUTOTEST_SVNROOT="svn://svn.freebsd.org/base/user/jmmv" +AUTOTEST_REVISION="r263766" + +SHTK_REMOTE="https://github.com/jmmv/shtk/" +SHTK_REVISION="shtk-1.5" Added: user/jmmv/autotest/node/configs/kyua2/host.conf ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/jmmv/autotest/node/configs/kyua2/host.conf Wed Mar 26 10:17:17 2014 (r263767) @@ -0,0 +1,12 @@ +# $FreeBSD$ +# +# Configuration to bootstrap autotest on this host and to maintain the host +# up-to-date with changes to the autotest code, its configuration files and +# its dependencies. +# + +AUTOTEST_SVNROOT="svn://svn.freebsd.org/base/user/jmmv" +AUTOTEST_REVISION="r263766" + +SHTK_REMOTE="https://github.com/jmmv/shtk/" +SHTK_REVISION="shtk-1.5" Modified: user/jmmv/autotest/node/setup.sh ============================================================================== --- user/jmmv/autotest/node/setup.sh Wed Mar 26 08:43:05 2014 (r263766) +++ user/jmmv/autotest/node/setup.sh Wed Mar 26 10:17:17 2014 (r263767) @@ -32,41 +32,271 @@ # Configures the current machine as an autotest node. shtk_import cli +shtk_import config -# Installs any required packages and ensures they are up-to-date. -install_deps() { - pkg update - pkg install -y qemu-devel kyua - pkg upgrade -y +# List of valid configuration variables. +# +# Please remember to update sysbuild.conf(5) if you change this list. +SETUP_CONFIG_VARS="ATF_REMOTE ATF_REVISION \ + AUTOTEST_SVNROOT AUTOTEST_REVISION \ + ROOT \ + SHTK_REMOTE SHTK_REVISION" + + +# Paths to installed files. +# +# Can be overriden for test purposes only. +: ${SETUP_ETCDIR="__AUTOTEST_ETCDIR__"} +: ${SETUP_ROOT="__AUTOTEST_ROOT__"} + + +# Packages needed to bootstrap autotest. +PACKAGES="automake autoconf kyua libtool qemu-devel" + + +# Sets defaults for configuration variables and hooks that need to exist. +# +# This function should be before the configuration file has been loaded. This +# means that the user can undefine a required configuration variable, but we let +# him shoot himself in the foot if he so desires. +setup_set_defaults() { + shtk_config_set ATF_REMOTE "https://github.com/jmmv/atf/" + shtk_config_set ATF_REVISION "HEAD" + shtk_config_set AUTOTEST_SVNROOT \ + "svn://svn.FreeBSD.org/base/user/jmmv/autotest/" + shtk_config_set AUTOTEST_REVISION "HEAD" + shtk_config_set ROOT "${SETUP_ROOT}" + shtk_config_set SHTK_REMOTE "https://github.com/jmmv/shtk/" + shtk_config_set SHTK_REVISION "HEAD" +} + + +# Prints the source directory of a specific component. +# +# \param component Name of the source component. +srcdir() { + local component="${1}"; shift + + echo "$(shtk_config_get ROOT)/src/${component}" +} + + +# Prints the directory of the installed built programs. +localdir() { + echo "$(shtk_config_get ROOT)/local" +} + + +# Fetches a repository from git and syncs it to a specific revision. +# +# \param remote Address of the repository. +# \param revision Revision to sync to. +# \param target Directory into which to fetch the sources. +fetch_git() { + local remote="${1}"; shift + local revision="${1}"; shift + local target="${1}"; shift + + if [ ! -d "${target}" ]; then + mkdir -p "$(dirname "${dir}")" + git clone "${remote}" "${target}" + fi + + cd "${target}" + git fetch + git checkout "${revision}" + cd - } -# Builds the source code. +# Builds a source package. # -# \param ... Arguments to make. Use to pass variable overrides for the host, -# such as the location of shtk(1). +# \param dir Directory in which the source code lives. build() { - make -C "$(shtk_cli_dirname)" clean - make -C "$(shtk_cli_dirname)" all "${@}" + local dir="${1}"; shift + + ( + cd "${dir}" + PATH="$(localdir)/bin:$(localdir)/sbin:${PATH}" + PKG_CONFIG_PATH="$(localdir)/share/pkgconfig:/usr/libdata/pkgconfig"; \ + export PKG_CONFIG_PATH + autoreconf -is -I "$(srcdir atf)/atf-c" -I "$(srcdir atf)/atf-sh" + ./configure --prefix "$(localdir)" + gmake -j4 + gmake install + ) +} + + +# Dumps the loaded configuration. +# +# \params ... The options and arguments to the command. +setup_config() { + [ ${#} -eq 0 ] || shtk_cli_usage_error "config does not take any arguments" + + for var in ${SETUP_CONFIG_VARS}; do + if shtk_config_has "${var}"; then + echo "${var} = $(shtk_config_get "${var}")" + else + echo "${var} is undefined" + fi + done +} + + +# Installs a cron job to periodically run setup. +setup_enable_cron() { + local dir="$(cd $(shtk_cli_dirname) && pwd)" + + local timespec="30 */1 * * *" + local entry="( cd '${dir}'" + entry="${entry}; svnlite update" + entry="${entry}; make" + entry="${entry}; ./setup all" + entry="${entry} ) >/dev/null 2>/dev/null # AUTOTEST" + + crontab -l | awk " +/# AUTOTEST/ { + next +} + +END { + print \"${timespec} ${entry}\" +} +" | crontab - } # Sets up rc.conf to start autotest_node on boot. -enable_daemon() { - grep "local_startup.*$(shtk_cli_dirname)/rc.d" /etc/rc.conf \ - || echo "local_startup=\"\${local_startup} $(shtk_cli_dirname)/rc.d\"" \ +setup_enable_daemon() { + local dir="$(srcdir autotest)/node" + + grep "local_startup.*${dir}/rc.d" /etc/rc.conf \ + || echo "local_startup=\"\${local_startup} ${dir}/rc.d\"" \ >>/etc/rc.conf grep "autotest_node_enable=yes" /etc/rc.conf \ || echo "autotest_node_enable=yes" >>/etc/rc.conf - "$(shtk_cli_dirname)/rc.d/autotest_node" start + #"${dir}/rc.d/autotest_node" start +} + + +# Fetches the source code of ATF to have access to its autoconf files. +# TODO(jmmv): Remove once we install the atf-*.m4 files in the base system. +setup_sync_atf() { + local dir="$(srcdir atf)" + + fetch_git "$(shtk_config_get ATF_REMOTE)" \ + "$(shtk_config_get ATF_REVISION)" "${dir}" +} + + +# Syncs and builds autotest to a specific revision. +setup_sync_autotest() { + local dir="$(srcdir autotest)" + mkdir -p "$(dirname "${dir}")" + + local svnroot="$(shtk_config_get AUTOTEST_SVNROOT)" + local revision="$(shtk_config_get AUTOTEST_REVISION)" + if [ -d "${dir}" ]; then + ( cd "${dir}" && svnlite update -r "${revision}" ) + else + svnlite co "${svnroot}@${revision}" "$(dirname "${dir}")" + fi + + make -C "${dir}/node" clean + make -C "${dir}/node" SHTK="$(localdir)/bin/shtk" all "${@}" +} + + +# Installs any required packages and ensures they are up-to-date. +setup_sync_packages() { + pkg update + pkg install -y ${PACKAGES} + pkg upgrade -y +} + + +# Syncs and builds shtk to a specific revision. +setup_sync_shtk() { + local dir="$(srcdir shtk)" + + fetch_git "$(shtk_config_get SHTK_REMOTE)" \ + "$(shtk_config_get SHTK_REVISION)" "${dir}" + build "${dir}" +} + + +# Performs the host setup. +# +# This operation can be run both to set up a new host or to update an existing +# host to newer autotest sources or configurations. +setup_all() { + setup_sync_packages + setup_sync_atf + setup_sync_shtk + setup_sync_autotest + setup_enable_daemon + setup_enable_cron } # Program entry. main() { - install_deps - build "${@}" - enable_daemon + local config_file="${SETUP_ETCDIR}/host.conf" + local expert_mode=false + + shtk_config_init ${SETUP_CONFIG_VARS} + + local OPTIND + while getopts ':c:o:X' arg "${@}"; do + case "${arg}" in + c) # Name of the configuration to load. + config_file="${OPTARG}" + ;; + + o) # Override for a particular configuration variable. + shtk_config_override "${OPTARG}" + ;; + + X) # Enable expert-mode. + expert_mode=true + ;; + + \?) + shtk_cli_usage_error "Unknown option -${OPTARG}" + ;; + esac + done + shift $((${OPTIND} - 1)) + + [ ${#} -ge 1 ] || shtk_cli_usage_error "No command specified" + + local exit_code=0 + + local command="${1}"; shift + local function="setup_$(echo ${command} | tr - _)" + case "${command}" in + all|config) + setup_set_defaults + shtk_config_load "${config_file}" + "${function}" "${@}" || exit_code="${?}" + ;; + + enable-cron|enable-daemon|sync-atf|sync-autotest|sync-packages|sync-shtk) + shtk_bool_check "${expert_mode}" \ + || shtk_cli_usage_error "Using ${command} requires expert" \ + "mode; try passing -X" + setup_set_defaults + shtk_config_load "${config_file}" + "${function}" "${@}" || exit_code="${?}" + ;; + + *) + shtk_cli_usage_error "Unknown command ${command}" + ;; + esac + + return "${exit_code}" }