Date: Thu, 2 Jul 2015 16:17:06 +0000 (UTC) From: Glen Barber <gjb@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-10@freebsd.org Subject: svn commit: r285049 - in stable/10/release: . scripts tools Message-ID: <201507021617.t62GH6d3047585@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: gjb Date: Thu Jul 2 16:17:05 2015 New Revision: 285049 URL: https://svnweb.freebsd.org/changeset/base/285049 Log: MFC r284893, r284895-r284897, r284942, r284968, r284996, r285005: r284893 (brd): Add initial support for building Vagrant images for VMWare. Next steps will be adding Virtualbox support and uploading to Hashicorp Atlas for others to consume. r284895: Add default VAGRANT_IMG variable. r284896: Remove _ACCOUNT and add _USERNAME, _NAME, _VERSION for the VAGRANT_${VAR} variables extracted from VAGRANT_UPLOAD_CONF. Set ATLAS_${VAR} to VAGRANT_${VAR} if VAGRANT_UPLOAD_CONF is set. There is intent to intentionally have separate variants of configuration entries, but the defaults do not yet have any reason to be different. r284897: Instead of hard-coding the PROVIDERS for upload, add the VAGRANT_PROVIDERS variable. Right now, it defaults to only vmware_desktop, virtualbox support is to follow at some point. While here, fix the hashicorp URL: s/vagrant/atlas/, which was result of a sed(1) replace (and my fault). r284942 (brd): Add Support for uploading Vagrant images to Hashicorp Atlas. r284968: Default the VAGRANT_VERSION to ${REVISION}-${BRANCH} if not set, which expands to '11.0-CURRENT', for example. If the branch is -CURRENT, -STABLE, or -PRERELEASE, suffix the VAGRANT_VERSION with the snapshot date. r284996: Fix the gcloud port/package name. r285005: Remove the HH-MM suffix from the build date suffix. It was useful when working out several kinks when testing automated image uploading when retrying was necessary, but now it is making things much too messy. Sponsored by: The FreeBSD Foundation Added: stable/10/release/Makefile.vagrant - copied, changed from r284893, head/release/Makefile.vagrant stable/10/release/scripts/atlas-upload.sh - copied unchanged from r284942, head/release/scripts/atlas-upload.sh stable/10/release/tools/vagrant.conf - copied unchanged from r284893, head/release/tools/vagrant.conf Modified: stable/10/release/Makefile.azure stable/10/release/Makefile.ec2 stable/10/release/Makefile.gce stable/10/release/Makefile.vm Directory Properties: stable/10/ (props changed) Modified: stable/10/release/Makefile.azure ============================================================================== --- stable/10/release/Makefile.azure Thu Jul 2 16:13:29 2015 (r285048) +++ stable/10/release/Makefile.azure Thu Jul 2 16:17:05 2015 (r285049) @@ -17,7 +17,7 @@ AZURE${VAR}!= grep -E ^AZURE${VAR} ${AZU .endif .if ${BRANCH} == "STABLE" || ${BRANCH} == "CURRENT" || ${BRANCH} == "PRERELEASE" -SNAPSHOT_DATE!= date +-%Y-%m-%d-%H-%M +SNAPSHOT_DATE!= date +-%Y-%m-%d .endif AZURE_TARGET:= ${OSRELEASE}${SNAPSHOT_DATE}.vhd Modified: stable/10/release/Makefile.ec2 ============================================================================== --- stable/10/release/Makefile.ec2 Thu Jul 2 16:13:29 2015 (r285048) +++ stable/10/release/Makefile.ec2 Thu Jul 2 16:17:05 2015 (r285049) @@ -6,7 +6,7 @@ # .if ${BRANCH} == "CURRENT" || ${BRANCH} == "STABLE" || ${BRANCH} == "PRERELEASE" -AMINAMESUFFIX!= date +-%Y-%m-%d-%H-%M +AMINAMESUFFIX!= date +-%Y-%m-%d .endif .if defined(EC2PUBLIC) PUBLISH= --public Modified: stable/10/release/Makefile.gce ============================================================================== --- stable/10/release/Makefile.gce Thu Jul 2 16:13:29 2015 (r285048) +++ stable/10/release/Makefile.gce Thu Jul 2 16:17:05 2015 (r285049) @@ -19,7 +19,7 @@ CLEANFILES+= ${GCE_UPLOAD_TGTS} GCE_BUCKET?= .if ${BRANCH} == "STABLE" || ${BRANCH} == "CURRENT" || ${BRANCH} == "PRERELEASE" -SNAPSHOT_DATE!= date +-%Y-%m-%d-%H-%M +SNAPSHOT_DATE!= date +-%Y-%m-%d .endif # Really? Uppercase characters are not allowed? Sigh... @@ -36,13 +36,13 @@ gce-check-depends: . endif .endfor .if !exists(/usr/local/bin/gcutil) -. if !exists(${PORTSDIR}/net/google-cloud-api/Makefile) +. if !exists(${PORTSDIR}/net/google-cloud-sdk/Makefile) . if !exists(/usr/local/sbin/pkg-static) env ASSUME_ALWAYS_YES=yes pkg bootstrap -yf . endif - env ASSUME_ALWAYS_YES=yes pkg install -y net/google-cloud-api + env ASSUME_ALWAYS_YES=yes pkg install -y net/google-cloud-sdk . else - make -C ${PORTSDIR}/net/google-cloud-api BATCH=1 all install clean + make -C ${PORTSDIR}/net/google-cloud-sdk BATCH=1 all install clean . endif .endif Copied and modified: stable/10/release/Makefile.vagrant (from r284893, head/release/Makefile.vagrant) ============================================================================== --- head/release/Makefile.vagrant Sat Jun 27 20:39:13 2015 (r284893, copy source) +++ stable/10/release/Makefile.vagrant Thu Jul 2 16:17:05 2015 (r285049) @@ -5,27 +5,32 @@ # Makefile for uploading Vagrant boxes to Hashicorp Atlas # +VAGRANT_IMG?= ${.OBJDIR}/vagrant.vmdk VAGRANT_UPLOAD_TGTS= vagrant-check-depends \ - vagrant-do-package-vmware -# atlas-do-upload + atlas-do-upload CLEANFILES+= ${VAGRANT_UPLOAD_TGTS} .if defined(VAGRANT_UPLOAD_CONF) && !empty(VAGRANT_UPLOAD_CONF) -. for VAR in _ACCOUNT _KEY -VAGRANT${VAR}!= grep -E ^VAGRANT${VAR} ${VAGRANT_UPLOAD_CONF} | awk -F' ' '{print $$2}' +. for VAR in _KEY _USERNAME +VAGRANT${VAR}!= grep -E ^VAGRANT${VAR} ${VAGRANT_UPLOAD_CONF} | awk -F' ' '{print $$2}' +ATLAS${VAR}:= ${VAGRANT${VAR}} . endfor .endif .if ${BRANCH} == "STABLE" || ${BRANCH} == "CURRENT" || ${BRANCH} == "PRERELEASE" -SNAPSHOT_DATE!= date +-%Y-%m-%d-%H-%M +SNAPSHOT_DATE!= date +-%Y-%m-%d .endif +VAGRANT_VERSION?= ${REVISION}-${BRANCH}${SNAPSHOT_DATE} + VAGRANT_TARGET:= ${OSRELEASE}${SNAPSHOT_DATE}.box +VAGRANT_PROVIDERS?= vmware_desktop +#VAGRANT_PROVIDERS+= virtualbox vagrant-upload: ${VAGRANT_UPLOAD_TGTS} vagrant-check-depends: -.for VAR in _ACCOUNT _KEY +.for VAR in _KEY _USERNAME _VERSION . if !defined(VAGRANT${VAR}) || empty(VAGRANT${VAR}) @echo "Variable VAGRANT${VAR} cannot be empty." @false @@ -36,7 +41,7 @@ vagrant-check-depends: . if !exists(/usr/local/sbin/pkg-static) env ASSUME_ALWAYS_YES=yes pkg bootstrap -yf . endif - env ASSUME_ALWAYS_YES=yes pkg install -y ftp/curl + env ASSUME_ALWAYS_YES=yes pkg install -y curl . else make -C ${PORTSDIR}/ftp/curl BATCH=1 all install clean . endif @@ -49,17 +54,12 @@ vagrant-do-package-vmware: vagrant-creat cd ${.OBJDIR} && tar -czf ${VAGRANT_TARGET} metadata.json vagrant.vmx vagrant.vmdk touch ${.OBJDIR}/${.TARGET} -atlas-create-upload: -.for PROVIDER in vmware_desktop virtualbox - /usr/local/bin/curl "https://vagrant.hashicorp.com/api/v1/box/${ATLAS_USERNAME}/${ATLAS_NAME}/version/${ATLAS_VERSION}/provider/${PROVIDER}/upload?access_token=${ATLAS_KEY}" +atlas-do-upload: vagrant-do-package-vmware +.for PROVIDER in ${VAGRANT_PROVIDERS} + ${.CURDIR}/scripts/atlas-upload.sh -b FreeBSD-${REVISION}-${BRANCH} -f ${VAGRANT_TARGET} -p ${PROVIDER} -k ${VAGRANT_KEY} -u ${VAGRANT_USERNAME} -v ${VAGRANT_VERSION} .endfor touch ${.OBJDIR}/${.TARGET} -atlas-do-upload: - /usr/local/bin/curl -X PUT --upload-file \ - ${VAGRANT_IMG} ${ATLAS_UPLOAD_PATH} - touch ${.OBJDIR}/${.TARGET} - vagrant-create-vmware-vmx: @cd ${.OBJDIR} && echo '.encoding = "UTF-8"' > vagrant.vmx @cd ${.OBJDIR} && echo 'bios.bootorder = "hdd,CDROM"' >> vagrant.vmx Modified: stable/10/release/Makefile.vm ============================================================================== --- stable/10/release/Makefile.vm Thu Jul 2 16:13:29 2015 (r285048) +++ stable/10/release/Makefile.vm Thu Jul 2 16:17:05 2015 (r285049) @@ -18,7 +18,8 @@ RAW_DESC= Unformatted raw disk image CLOUDWARE?= AZURE \ EC2 \ GCE \ - OPENSTACK + OPENSTACK \ + VAGRANT AZURE_FORMAT= vhdf AZURE_DESC= Microsoft Azure platform image AZURE_DISK= ${OSRELEASE}.${AZURE_FORMAT} @@ -31,6 +32,9 @@ GCE_DISK= disk.${GCE_FORMAT} OPENSTACK_FORMAT=qcow2 OPENSTACK_DESC= OpenStack platform image OPENSTACK_DISK= ${OSRELEASE}.${OPENSTACK_FORMAT} +VAGRANT_FORMAT= vmdk +VAGRANT_DESC= Vagrant Image +VAGRANT_DISK= ${OSRELEASE}.${VAGRANT_FORMAT} .if defined(WITH_CLOUDWARE) && !empty(WITH_CLOUDWARE) && !empty(CLOUDWARE) . for _CW in ${CLOUDWARE} @@ -156,3 +160,4 @@ cloudware-install: .include "${.CURDIR}/Makefile.ec2" .include "${.CURDIR}/Makefile.azure" .include "${.CURDIR}/Makefile.gce" +.include "${.CURDIR}/Makefile.vagrant" Copied: stable/10/release/scripts/atlas-upload.sh (from r284942, head/release/scripts/atlas-upload.sh) ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ stable/10/release/scripts/atlas-upload.sh Thu Jul 2 16:17:05 2015 (r285049, copy of r284942, head/release/scripts/atlas-upload.sh) @@ -0,0 +1,155 @@ +#!/bin/sh +#- +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# Upload a Vagrant image to Hashicorp's Atlas service +# +# $FreeBSD$ +# + +ATLAS_API_URL='' +ATLAS_UPLOAD_URL='https://binstore.hashicorp.com' +VERSION_DESCRIPTION="FreeBSD Snapshot Build" + +usage() { + echo "${0} usage:" + echo "-b box-name -f box-to-upload -k api-key -p provider -u user -v version" + return 1 +} + +main () { + while getopts "b:f:k:p:u:v:" arg; do + case "${arg}" in + b) + BOX="${OPTARG}" + ;; + f) + FILE="${OPTARG}" + ;; + k) + KEY="${OPTARG}" + ;; + p) + PROVIDER="${OPTARG}" + ;; + u) + USERNAME="${OPTARG}" + ;; + v) + VERSION="${OPTARG}" + ;; + *) + ;; + esac + done + + if [ -z "${BOX}" -o \ + -z "${FILE}" -o \ + -z "${KEY}" -o \ + -z "${PROVIDER}" -o \ + -z "${USERNAME}" -o \ + -z "${VERSION}" ]; + then + usage || exit 0 + fi + + # Check to see if the box exists or create it + BOXRESULT=$(/usr/local/bin/curl -s "https://atlas.hashicorp.com/api/v1/box/${USERNAME}/${BOX}?access_token=${KEY}") + if [ $? != 0 ]; then + echo "Failed to connect to the API" + exit 2; + fi + echo $BOXRESULT | grep "\"name\":\"${BOX}\"" > /dev/null + if [ $? != 0 ]; then + echo "Creating box: ${BOX}" + /usr/local/bin/curl -s https://atlas.hashicorp.com/api/v1/boxes -X POST -d "box[name]=${BOX}" -d "access_token=${KEY}" > /dev/null + /usr/local/bin/curl -s https://atlas.hashicorp.com/api/v1/box/${USERNAME}/${BOX} -X PUT -d "box[is_private]=false" -d "access_token=${KEY}" > /dev/null + else + echo "Box already exists" + fi + + # Check to see if the version exists or create it + VERSIONRESULT=$(/usr/local/bin/curl -s "https://atlas.hashicorp.com/api/v1/box/${USERNAME}/${BOX}/version/${VERSION}?access_token=${KEY}") + if [ $? != 0 ]; then + echo "Failed to connect to the API" + exit 2; + fi + echo $VERSIONRESULT | grep "\"version\":\"${VERSION}\"" > /dev/null + if [ $? != 0 ]; then + echo "Creating version: ${VERSION}" + /usr/local/bin/curl -s https://atlas.hashicorp.com/api/v1/box/${USERNAME}/${BOX}/versions -X POST -d "version[version]=${VERSION}" -d "access_token=${KEY}" > /dev/null + /usr/local/bin/curl -s https://atlas.hashicorp.com/api/v1/box/${USERNAME}/${BOX}/version/${VERSION} -X PUT -d "version[description]=${VERSION_DESCRIPTION}" -d "access_token=${KEY}" > /dev/null + VERSIONRESULT=$(/usr/local/bin/curl -s "https://atlas.hashicorp.com/api/v1/box/${USERNAME}/${BOX}/version/${VERSION}?access_token=${KEY}") + echo $VERSIONRESULT | grep "\"version\":\"${VERSION}\"" > /dev/null + if [ $? != 0 ]; then + echo "Failed to create version" + exit 2 + fi + else + echo "Version already exists" + fi + + # Check to see if the provider exists or create it + PROVIDERRESULT=$(/usr/local/bin/curl -s "https://atlas.hashicorp.com/api/v1/box/${USERNAME}/${BOX}/version/${VERSION}/provider/${PROVIDER}?access_token=${KEY}") + if [ $? != 0 ]; then + echo "Failed to connect to the API" + exit 2; + fi + echo $PROVIDERRESULT | grep "\"name\":\"${PROVIDER}\"" > /dev/null + if [ $? != 0 ]; then + echo "Creating provider: ${PROVIDER}" + /usr/local/bin/curl -s https://atlas.hashicorp.com/api/v1/box/${USERNAME}/${BOX}/version/${VERSION}/providers -X POST -d "provider[name]=${PROVIDER}" -d "access_token=${KEY}" > /dev/null + else + echo "Provider already exists" + fi + + # Request an upload token + TOKENRESULT=$(/usr/local/bin/curl -s "https://atlas.hashicorp.com/api/v1/box/${USERNAME}/${BOX}/version/${VERSION}/provider/${PROVIDER}/upload?access_token=${KEY}") + if [ $? != 0 ]; then + echo "Failed to get the token from the API" + exit 2; + fi + echo ${TOKENRESULT} | grep "\"token\":" > /dev/null + if [ $? != 0 ]; then + echo "No token found from the API" + exit 2 + else + TOKEN=$(echo $TOKENRESULT | sed -e 's/.*token":"//' -e 's/".*//') + echo "Uploading to Atlas" + UPLOADRESULT=$(/usr/local/bin/curl -s -X PUT --upload-file ${FILE} ${ATLAS_UPLOAD_URL}/${TOKEN}) + + # Validate the Upload + echo "Validating" + VALIDRESULT=$(/usr/local/bin/curl -s "https://atlas.hashicorp.com/api/v1/box/${USERNAME}/${BOX}/version/${VERSION}/provider/${PROVIDER}?access_token=${KEY}") + HOSTED_TOKEN=$(echo $VALIDRESULT | sed -e 's/.*hosted_token":"//' -e 's/".*//') + if [ ! -z ${HOSTED_TOKEN} -a ! -z ${TOKEN} -a ${HOSTED_TOKEN} != ${TOKEN} ]; then + echo "Upload failed, try again." + exit 2 + fi + + # Release the version + echo "Releasing ${VERSION} of ${BOX} in Atlas" + /usr/local/bin/curl -s https://atlas.hashicorp.com/api/v1/box/${USERNAME}/${BOX}/version/${VERSION}/release -X PUT -d "access_token=${KEY}" > /dev/null + fi +} + +main "$@" Copied: stable/10/release/tools/vagrant.conf (from r284893, head/release/tools/vagrant.conf) ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ stable/10/release/tools/vagrant.conf Thu Jul 2 16:17:05 2015 (r285049, copy of r284893, head/release/tools/vagrant.conf) @@ -0,0 +1,81 @@ +#!/bin/sh +# +# $FreeBSD$ +# + +# Packages to install into the image we're creating. This is a deliberately +# minimalist set, providing only the packages necessary to bootstrap. +export VM_EXTRA_PACKAGES="firstboot-freebsd-update firstboot-pkgs" + +# Set to a list of third-party software to enable in rc.conf(5). +export VM_RC_LIST="firstboot_freebsd_update firstboot_pkgs" + +vm_extra_pre_umount() { + # The firstboot_pkgs rc.d script will download the repository + # catalogue and install or update pkg when the instance first + # launches, so these files would just be replaced anyway; removing + # them from the image allows it to boot faster. + env ASSUME_ALWAYS_YES=yes pkg -c ${DESTDIR} delete -f -y pkg + rm ${DESTDIR}/var/db/pkg/repo-*.sqlite + + # The size of the EC2 root disk can be configured at instance launch + # time; expand our filesystem to fill the disk. + echo 'growfs_enable="YES"' >> ${DESTDIR}/etc/rc.conf + + # Vagrant instances use DHCP to get their network configuration. + echo 'ifconfig_DEFAULT="SYNCDHCP"' >> ${DESTDIR}/etc/rc.conf + + # Enable sshd by default + echo 'sshd_enable="YES"' >> ${DESTDIR}/etc/rc.conf + # Disable DNS lookups by default to make SSH connect quickly + echo 'UseDNS no' >> ${DESTDIR}/etc/ssh/sshd_config + + # Disable sendmail + echo 'sendmail_enable="NO"' >> ${DESTDIR}/etc/rc.conf + echo 'sendmail_submit_enable="NO"' >> ${DESTDIR}/etc/rc.conf + echo 'sendmail_outbound_enable="NO"' >> ${DESTDIR}/etc/rc.conf + echo 'sendmail_msp_queue_enable="NO"' >> ${DESTDIR}/etc/rc.conf + + # sudo is required + echo 'firstboot_pkgs_list="sudo rsync"' >> ${DESTDIR}/etc/rc.conf + + # Create the vagrant user with a password of vagrant + /usr/sbin/pw -R ${DESTDIR} \ + groupadd vagrant -g 1001 + chroot ${DESTDIR} mkdir -p /home/vagrant + /usr/sbin/pw -R ${DESTDIR} \ + useradd vagrant \ + -m -M 0755 -w yes -n vagrant -u 1001 -g 1001 -G 0 \ + -c 'Vagrant User' -d '/home/vagrant' -s '/bin/csh' + + # Change root's password to vagrant + echo 'vagrant' | /usr/sbin/pw -R ${DESTDIR} \ + usermod root -h 0 + + # Configure sudo to allow the vagrant user + echo 'vagrant ALL=(ALL) NOPASSWD: ALL' >> ${DESTDIR}/usr/local/etc/sudoers + + # Configure the vagrant ssh key + mkdir ${DESTDIR}/home/vagrant/.ssh + chmod 700 ${DESTDIR}/home/vagrant/.ssh + echo "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key" > ${DESTDIR}/home/vagrant/.ssh/authorized_keys + chown -R 1001 ${DESTDIR}/home/vagrant/.ssh + chmod 600 ${DESTDIR}/home/vagrant/.ssh/authorized_keys + + # Reboot quickly, Don't wait at the panic screen + echo 'debug.trace_on_panic=1' >> ${DESTDIR}/etc/sysctl.conf + echo 'debug.debugger_on_panic=0' >> ${DESTDIR}/etc/sysctl.conf + echo 'kern.panic_reboot_wait_time=0' >> ${DESTDIR}/etc/sysctl.conf + + # The console is not interactive, so we might as well boot quickly. + echo 'autoboot_delay="-1"' >> ${DESTDIR}/boot/loader.conf + + # The first time the VM boots, the installed "first boot" scripts + # should be allowed to run: + # * growfs (expand the filesystem to fill the provided disk) + # * firstboot_freebsd_update (install critical updates) + # * firstboot_pkgs (install packages) + touch ${DESTDIR}/firstboot + + return 0 +}
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201507021617.t62GH6d3047585>