Date: Wed, 22 Mar 2017 22:48:16 +0000 (UTC) From: Glen Barber <gjb@FreeBSD.org> To: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: svn commit: r315757 - user/gjb/thermite Message-ID: <201703222248.v2MMmGCS057036@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: gjb Date: Wed Mar 22 22:48:16 2017 New Revision: 315757 URL: https://svnweb.freebsd.org/changeset/base/315757 Log: Add extra helper scripts for thermite.sh. Sponsored by: The FreeBSD Foundation Added: user/gjb/thermite/generate-email.pl (contents, props changed) user/gjb/thermite/get-checksums.sh (contents, props changed) user/gjb/thermite/setrev.sh (contents, props changed) user/gjb/thermite/zfs-setup.sh (contents, props changed) Added: user/gjb/thermite/generate-email.pl ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/gjb/thermite/generate-email.pl Wed Mar 22 22:48:16 2017 (r315757) @@ -0,0 +1,266 @@ +#!/usr/bin/env perl +# +# $FreeBSD$ +# + +use strict; +use warnings; +use locale; + +use Getopt::Std; +use File::Basename; + +my $prog = basename($0); + +our $opt_h; +our $builddate; +our $svnrev; +our $junk = ""; +our $arch = ""; +our $kernel = ""; +our $branch = ""; +our $revision = 0; +our $branchname = ""; +our $version = 0; +our $hasarmv6 = 0; +our $hasarm64 = 0; +our $hasbranch = 0; + +sub usage() { + print("Usage: ./get-checksums.sh -c ./builds-NN.conf | $prog > outfile\n"); + exit(0); +} + +sub main() { + getopts('h'); + my @lines = (); + my @builds = (); + my @amis = (); + my @vmimages = (); + my $endisos = 0; + $builddate = 0; + $svnrev = 0; + $junk = ""; + $arch = ""; + $kernel = ""; + $branch = ""; + $branchname = ""; + $revision = 0; + $version = 0; + $hasbranch = 0; + + if ($opt_h) { + &usage(); + } + + while(<STDIN>) { + chomp($_); + push(@lines, $_); + if ($_ =~ m/^== VM IMAGE CHECKSUMS ==/) { + $endisos = 1; + } + if ($_ =~ m/^BUILDDATE/) { + $builddate = $_; + $builddate =~ s/^BUILDDATE=//; + pop(@lines); + next; + } + if ($_ =~ m/^SVNREV/) { + $svnrev = $_; + $svnrev =~ s/^SVNREV=//; + pop(@lines); + next; + } + if ($_ =~ m/^o /) { + $_ =~ s/:$//; + if ($_ =~ m/^o .* armv6 .*/) { + $hasarmv6 = 1; + } + if ($_ =~ m/^o .* aarch64 .*/) { + $hasarm64 = 1; + } + if ($hasbranch eq 0) { + ($junk, $branch, $arch, $kernel) = split(" ", $_); + $revision = $branch; + $revision =~ s/-\w+.*//; + $version = $revision; + $version =~ s/\.\d.*//; + $branchname = $branch; + $branchname =~ s/\d+\.\d-//; + $branchname =~ s/ .*$//; + if ($branchname =~ m/(ALPHA|CURRENT)/) { + $branch = "head"; + } elsif ($branchname =~ m/(BETA|PRERELEASE|RC|STABLE)/) { + $branch = "stable/$version"; + } else { + $branch = "unknown"; + } + $hasbranch = 1; + } + if ($endisos == 0) { + push(@builds, $_); + } else { + push(@vmimages, $_); + } + } + if ($_ =~ m/^Created AMI in /) { + $_ =~ s/^Created AMI in //; + # Exclude ca-central-1 eu-west-2 for now + #if ($_ !~ m/(ca-central-1|eu-west-2)/) { + push(@amis, $_); + pop(@lines); + #} + } + } + + print <<HEADER; +To: freebsd-snapshots\@FreeBSD.org +Subject: New FreeBSD snapshots available: $branch ($builddate r$svnrev) + +HEADER + print <<OPENING; +New FreeBSD development branch installation ISOs and virtual machine +disk images have been uploaded to the FTP mirrors. + +As with any development branch, the installation snapshots are not +intended for use on production systems. We do, however, encourage +testing on non-production systems as much as possible. + +Please also consider installing the sysutils/panicmail port, which can +help in providing FreeBSD developers the necessary information regarding +system crashes. + +Checksums for the installation ISOs and the VM disk images follow at +the end of this email. + +=== Installation ISOs === + +Installation images are available for: + +OPENING + + foreach my $build (@builds) { + print("$build\n"); + } + + if ($hasarmv6 ne 0) { + print <<ARMINFO; + +Note regarding arm/armv6 images: For convenience for those without +console access to the system, a freebsd user with a password of +freebsd is available by default for ssh(1) access. Additionally, +the root user password is set to root, which it is strongly +recommended to change the password for both users after gaining +access to the system. +ARMINFO + } + + print <<OPENING; + +Snapshots may be downloaded from the corresponding architecture +directory from: + + ftp://ftp.freebsd.org/pub/FreeBSD/snapshots/ISO-IMAGES/ + +Please be patient if your local FTP mirror has not yet caught +up with the changes. + +Problems, bug reports, or regression reports should be reported through +the Bugzilla PR system or the appropriate mailing list such as -current\@ +or -stable\@ . + +=== Virtual Machine Disk Images === + +VM disk images are available for the following architectures: + +OPENING + + foreach my $vmimage (@vmimages) { + print("$vmimage\n"); + } + + print <<OPENING; + +Disk images may be downloaded from the following URL (or any of the +FreeBSD FTP mirrors): + + ftp://ftp.freebsd.org/pub/FreeBSD/snapshots/VM-IMAGES/ + +Images are available in the following disk image formats: + + ~ RAW + ~ QCOW2 (qemu) + ~ VMDK (qemu, VirtualBox, VMWare) + ~ VHD (qemu, xen) + +The partition layout is: + + ~ 512k - freebsd-boot GPT partition type (bootfs GPT label) + ~ 1GB - freebsd-swap GPT partition type (swapfs GPT label) + ~ ~17GB - freebsd-ufs GPT partition type (rootfs GPT label) +OPENING + + if ($hasarm64 ne 0) { + print <<AARCH64; + +Note regarding arm64/aarch64 virtual machine images: a modified QEMU EFI +loader file is needed for qemu-system-aarch64 to be able to boot the +virtual machine images. See this page for more information: + + https://wiki.freebsd.org/arm64/QEMU + +To boot the VM image, run: + + % qemu-system-aarch64 -m 4096M -cpu cortex-a57 -M virt \\ + -bios QEMU_EFI.fd -serial telnet::4444,server -nographic \\ + -drive if=none,file=VMDISK,id=hd0 \\ + -device virtio-blk-device,drive=hd0 \\ + -device virtio-net-device,netdev=net0 \\ + -netdev user,id=net0 + +Be sure to replace "VMDISK" with the path to the virtual machine image. +AARCH64 + } + + if ($#amis ne 0) { + print <<AMIS; + +=== Amazon EC2 AMI Images === + +FreeBSD/amd64 EC2 AMIs are available in the following regions: + +AMIS + } + foreach my $ami (@amis) { + print(" $ami\n"); + } + + print <<VAGRANT; + +=== Vagrant Images === + +FreeBSD/amd64 images are available on the Hashicorp Atlas site for the +VMWare Desktop and VirtualBox providers, and can be installed by +running: + + % vagrant init freebsd/FreeBSD-$revision-$branchname + % vagrant up + +VAGRANT + foreach my $line (@lines) { + if ($line !~ m/^Created AMI in /) { + print("$line\n"); + } + } + + print <<FOOTER; + +Love FreeBSD? Support this and future releases with a donation to +the FreeBSD Foundation! https://www.freebsdfoundation.org/donate/ + +FOOTER + + return(0); +} + +&main(); Added: user/gjb/thermite/get-checksums.sh ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/gjb/thermite/get-checksums.sh Wed Mar 22 22:48:16 2017 (r315757) @@ -0,0 +1,164 @@ +#!/bin/sh +# +# $FreeBSD$ +# + +usage() { + echo "$(basename ${0}) -c /path/to/configuration/file" + exit 1 +} + +get_vm_checksum() { + local _s="${r}-${a}-${k}-${t}" + sumfiles="SHA512 SHA256" + if [ -e ${scriptdir}/${_s}.conf ]; then + . ${scriptdir}/${_s}.conf + else + return 0 + fi + case ${t} in + release) + type="releases" + ;; + *) + type="snapshots" + ;; + esac + if [ ! -e "${CHROOTDIR}/R/ftp-stage/${type}/VM-IMAGES" ]; then + return 0 + fi + __REVISION=$(make -C ${CHROOTDIR}/usr/src/release -V REVISION) + __BRANCH=$(make -C ${CHROOTDIR}/usr/src/release -V BRANCH) + for _f in ${sumfiles}; do + case ${_f} in + SHA512) + echo "o ${__REVISION}-${__BRANCH} ${a}:" + ;; + *) + ;; + esac + cat ${CHROOTDIR}/R/ftp-stage/${type}/VM-IMAGES/${__REVISION}-${__BRANCH}/${TARGET_ARCH}/Latest/CHECKSUM.${_f}* | \ + sed -e 's/^/ /' + echo + done + echo + return 0 +} + + +get_iso_checksum() { + local _s="${r}-${a}-${k}-${t}" + sumfiles="SHA512 SHA256" + if [ -e ${scriptdir}/${_s}.conf ]; then + . ${scriptdir}/${_s}.conf + else + return 0 + fi + if [ ! -e ${CHROOTDIR}/R/ ]; then + return 0 + fi + case ${t} in + release) + type="releases" + ;; + *) + type="snapshots" + ;; + esac + __REVISION=$(make -C ${CHROOTDIR}/usr/src/release -V REVISION) + __BRANCH=$(make -C ${CHROOTDIR}/usr/src/release -V BRANCH) + if [ ! -z "${EMBEDDEDBUILD}" ]; then + TARGET="${EMBEDDED_TARGET}" + TARGET_ARCH="${EMBEDDED_TARGET_ARCH}" + fi + for _f in ${sumfiles}; do + case ${_f} in + SHA512) + echo "o ${__REVISION}-${__BRANCH} ${a} ${k}:" + ;; + *) + ;; + esac + cat ${CHROOTDIR}/R/ftp-stage/${type}/${TARGET}/${TARGET_ARCH}/ISO-IMAGES/${__REVISION}/CHECKSUM.${_f}* | \ + sed -e 's/^/ /' + echo + done + unset EMBEDDEDBUILD + echo + return 0 +} + +main() { + export __BUILDCONFDIR="$(dirname $(realpath ${0}))" + CSCONF= + + while getopts "c:" opt; do + case ${opt} in + c) + CSCONF="${OPTARG}" + ;; + *) + ;; + esac + done + + if [ -z "${CSCONF}" ]; then + echo "Build configuration file is required." + usage + fi + + CSCONF="$(realpath ${CSCONF})" + + if [ ! -f "${CSCONF}" ]; then + echo "Build configuration is not a regular file." + exit 1 + fi + + . "${CSCONF}" + + case ${t} in + release) + ;; + *) + if [ -e "builddate" ]; then + echo "BUILDDATE=$(cat builddate)" + fi + if [ -e "svnrev_src" ]; then + echo "SVNREV=$(cat svnrev_src)" + fi + tail -n50 ../logs/*.ec2* | grep -E '^Created AMI in' \ + || true + ;; + esac + + echo "== ISO CHECKSUMS ==" + echo + for r in ${revs}; do + for a in ${archs}; do + for k in ${kernels}; do + for t in ${types}; do + get_iso_checksum + done + done + done + done + echo "== VM IMAGE CHECKSUMS ==" + echo + for r in ${revs}; do + for a in ${archs}; do + for k in ${kernels}; do + for t in ${types}; do + case ${a} in + amd64|i386|aarch64) + get_vm_checksum + ;; + *) + ;; + esac + done + done + done + done +} + +main "$@" Added: user/gjb/thermite/setrev.sh ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/gjb/thermite/setrev.sh Wed Mar 22 22:48:16 2017 (r315757) @@ -0,0 +1,63 @@ +#!/bin/sh +# +# $FreeBSD$ +# + +# TODO: add support for ports/ and doc/ tree + +usage() { + echo "Usage:" + echo "${0} -b <branch>" + exit 1 +} + +get_build_rev() { + if [ -e "${svnfile}" -a ! -f "${svnfile}" ]; then + echo "SVN file is not a regular file." + echo "Renaming file." + mv ${svnfile} ${svnfile}.bak.${today} + fi + svnrev=$(svn info ${svnhost}/${branch} | \ + awk -F ': ' '/^Last Changed Rev/ {print $2}') + svnrev=$(echo ${svnrev} | tr -d '[a-z]') + echo ${svnrev} > ${svnfile} +} + +get_build_date() { + if [ -e "${datefile}" -a ! -f "${datefile}" ]; then + echo "SVN file is not a regular file." + echo "Renaming file." + mv ${svnfile} ${svnfile}.bak.${today} + fi + echo ${today} > ${datefile} +} + +main() { + export PATH="/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin" + export TZ='UTC' + export __BUILDCONFDIR="$(dirname $(realpath ${0}))" + svnhost="svn://svn.FreeBSD.org/base" + svnfile="${__BUILDCONFDIR}/svnrev_src" + datefile="${__BUILDCONFDIR}/builddate" + today="$(date +%Y%m%d)" + + while getopts "b:" opt; do + case ${opt} in + b) + branch="${OPTARG}" + ;; + *) + ;; + esac + done + + if [ -z "${branch}" ]; then + echo "Branch not specified." + usage + fi + + get_build_rev + get_build_date +} + +main "$@" Added: user/gjb/thermite/zfs-setup.sh ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/gjb/thermite/zfs-setup.sh Wed Mar 22 22:48:16 2017 (r315757) @@ -0,0 +1,161 @@ +#!/bin/sh +# +# $FreeBSD$ +# + +usage() { + echo "$(basename ${0}) [-d] -c /path/to/configuration/file" + exit 1 +} + +zfs_teardown() { + for r in ${revs}; do + for a in ${archs}; do + for k in ${kernels}; do + for t in ${types}; do + s="${r}-${a}-${k}-${t}" + c="${r}-${a}-${t}" + if [ -e ${scriptdir}/${s}.conf ]; + then + zfs list ${zfs_parent}/${s}-src >/dev/null 2>&1 + rc=$? + if [ ${rc} -eq 0 ]; then + echo -n "${pfx} Destroying " \ + >/dev/stdout + echo " ${zfs_parent}/${s}-src" \ + >/dev/stdout + zfs destroy -f ${zfs_parent}/${s}-src + fi + zfs list ${zfs_parent}/${s}-ports >/dev/null 2>&1 + rc=$? + if [ ${rc} -eq 0 ]; then + echo -n "${pfx} Destroying " \ + >/dev/stdout + echo " ${zfs_parent}/${s}-ports" \ + >/dev/stdout + zfs destroy -f ${zfs_parent}/${s}-ports + fi + zfs list ${zfs_parent}/${s}-doc >/dev/null 2>&1 + rc=$? + if [ ${rc} -eq 0 ]; then + echo -n "${pfx} Destroying " \ + >/dev/stdout + echo " ${zfs_parent}/${s}-doc" \ + >/dev/stdout + zfs destroy -f ${zfs_parent}/${s}-doc + fi + zfs list ${zfs_parent}/${c}-chroot >/dev/null 2>&1 + rc=$? + if [ ${rc} -eq 0 ]; then + echo -n "${pfx} Destroying " \ + >/dev/stdout + echo " ${zfs_parent}/${c}-chroot" \ + >/dev/stdout + zfs destroy -f ${zfs_parent}/${c}-chroot + fi + zfs list ${zfs_parent}/${s} >/dev/null 2>&1 + rc=$? + if [ ${rc} -eq 0 ]; then + echo -n "${pfx} Destroying " \ + >/dev/stdout + echo " ${zfs_parent}/${s}" \ + >/dev/stdout + zfs destroy -f ${zfs_parent}/${s} + fi + fi + done + done + done + done + + for r in ${revs}; do + for t in ${types}; do + for i in src doc ports; do + zfs list ${zfs_parent}/${r}-${i}-${t}@clone >/dev/null 2>&1 + rc=$? + if [ ${rc} -eq 0 ]; then + echo -n "${pfx} Destroying " \ + >/dev/stdout + echo " ${zfs_parent}/${r}-${i}-${t}@clone" \ + >/dev/stdout + zfs destroy -f ${zfs_parent}/${r}-${i}-${t}@clone + fi + zfs list ${zfs_parent}/${r}-${i}-${t} >/dev/null 2>&1 + rc=$? + if [ ${rc} -eq 0 ]; then + echo -n "${pfx} Destroying " \ + >/dev/stdout + echo " ${zfs_parent}/${r}-${i}-${t}" \ + >/dev/stdout + zfs destroy -f ${zfs_parent}/${r}-${i}-${t} + fi + done + done + done + return 0 +} + +zfs_setup() { + [ ! -z ${delete_only} ] && return 0 + for r in ${revs}; do + for a in ${archs}; do + for k in ${kernels}; do + for t in ${types}; do + s="${r}-${a}-${k}-${t}" + if [ -e ${scriptdir}/${s}.conf ]; + then + echo "${pfx} Creating ${zfs_parent}/${s}" \ + >/dev/stdout + zfs create -o atime=off ${zfs_parent}/${s} + fi + done + done + done + done + return 0 +} + +main() { + export __BUILDCONFDIR="$(dirname $(realpath ${0}))" + CSCONF= + + while getopts "c:d" opt; do + case ${opt} in + c) + CSCONF="${OPTARG}" + ;; + d) + delete_only=1 + ;; + *) + ;; + esac + done + shift $(( ${OPTIND} - 1 )) + + if [ -z "${CSCONF}" ]; then + echo "Build configuration file is required." + usage + fi + + CSCONF="$(realpath ${CSCONF})" + + if [ ! -f "${CSCONF}" ]; then + echo "Build configuration is not a regular file." + exit 1 + fi + + . "${CSCONF}" + + if [ ${use_zfs} -eq 0 ]; then + echo "== use_zfs is set to '0'; skipping." >/dev/stdout + exit 0 + fi + + pfx="===" + + zfs_teardown + zfs_setup +} + +main "$@"
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201703222248.v2MMmGCS057036>