From owner-svn-src-user@freebsd.org  Wed Mar 22 22:48:17 2017
Return-Path: <owner-svn-src-user@freebsd.org>
Delivered-To: svn-src-user@mailman.ysv.freebsd.org
Received: from mx1.freebsd.org (mx1.freebsd.org
 [IPv6:2001:1900:2254:206a::19:1])
 by mailman.ysv.freebsd.org (Postfix) with ESMTP id A865BD18498
 for <svn-src-user@mailman.ysv.freebsd.org>;
 Wed, 22 Mar 2017 22:48:17 +0000 (UTC) (envelope-from gjb@FreeBSD.org)
Received: from repo.freebsd.org (repo.freebsd.org
 [IPv6:2610:1c1:1:6068::e6a:0])
 (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits))
 (Client did not present a certificate)
 by mx1.freebsd.org (Postfix) with ESMTPS id 759868FF;
 Wed, 22 Mar 2017 22:48:17 +0000 (UTC) (envelope-from gjb@FreeBSD.org)
Received: from repo.freebsd.org ([127.0.1.37])
 by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v2MMmGEH057040;
 Wed, 22 Mar 2017 22:48:16 GMT (envelope-from gjb@FreeBSD.org)
Received: (from gjb@localhost)
 by repo.freebsd.org (8.15.2/8.15.2/Submit) id v2MMmGCS057036;
 Wed, 22 Mar 2017 22:48:16 GMT (envelope-from gjb@FreeBSD.org)
Message-Id: <201703222248.v2MMmGCS057036@repo.freebsd.org>
X-Authentication-Warning: repo.freebsd.org: gjb set sender to gjb@FreeBSD.org
 using -f
From: Glen Barber <gjb@FreeBSD.org>
Date: Wed, 22 Mar 2017 22:48:16 +0000 (UTC)
To: src-committers@freebsd.org, svn-src-user@freebsd.org
Subject: svn commit: r315757 - user/gjb/thermite
X-SVN-Group: user
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
X-BeenThere: svn-src-user@freebsd.org
X-Mailman-Version: 2.1.23
Precedence: list
List-Id: "SVN commit messages for the experimental &quot; user&quot;
 src tree" <svn-src-user.freebsd.org>
List-Unsubscribe: <https://lists.freebsd.org/mailman/options/svn-src-user>,
 <mailto:svn-src-user-request@freebsd.org?subject=unsubscribe>
List-Archive: <http://lists.freebsd.org/pipermail/svn-src-user/>
List-Post: <mailto:svn-src-user@freebsd.org>
List-Help: <mailto:svn-src-user-request@freebsd.org?subject=help>
List-Subscribe: <https://lists.freebsd.org/mailman/listinfo/svn-src-user>,
 <mailto:svn-src-user-request@freebsd.org?subject=subscribe>
X-List-Received-Date: Wed, 22 Mar 2017 22:48:17 -0000

Author: gjb
Date: Wed Mar 22 22:48:16 2017
New Revision: 315757
URL: https://svnweb.freebsd.org/changeset/base/315757

Log:
  Add extra helper scripts for thermite.sh.
  
  Sponsored by:	The FreeBSD Foundation

Added:
  user/gjb/thermite/generate-email.pl   (contents, props changed)
  user/gjb/thermite/get-checksums.sh   (contents, props changed)
  user/gjb/thermite/setrev.sh   (contents, props changed)
  user/gjb/thermite/zfs-setup.sh   (contents, props changed)

Added: user/gjb/thermite/generate-email.pl
==============================================================================
--- /dev/null	00:00:00 1970	(empty, because file is newly added)
+++ user/gjb/thermite/generate-email.pl	Wed Mar 22 22:48:16 2017	(r315757)
@@ -0,0 +1,266 @@
+#!/usr/bin/env perl
+#
+# $FreeBSD$
+#
+
+use strict;
+use warnings;
+use locale;
+
+use Getopt::Std;
+use File::Basename;
+
+my $prog = basename($0);
+
+our $opt_h;
+our $builddate;
+our $svnrev;
+our $junk = "";
+our $arch = "";
+our $kernel = "";
+our $branch = "";
+our $revision = 0;
+our $branchname = "";
+our $version = 0;
+our $hasarmv6 = 0;
+our $hasarm64 = 0;
+our $hasbranch = 0;
+
+sub usage() {
+	print("Usage: ./get-checksums.sh -c ./builds-NN.conf | $prog > outfile\n");
+	exit(0);
+}
+
+sub main() {
+	getopts('h');
+	my @lines = ();
+	my @builds = ();
+	my @amis = ();
+	my @vmimages = ();
+	my $endisos = 0;
+	$builddate = 0;
+	$svnrev = 0;
+	$junk = "";
+	$arch = "";
+	$kernel = "";
+	$branch = "";
+	$branchname = "";
+	$revision = 0;
+	$version = 0;
+	$hasbranch = 0;
+
+	if ($opt_h) {
+		&usage();
+	}
+
+	while(<STDIN>) {
+		chomp($_);
+		push(@lines, $_);
+		if ($_ =~ m/^== VM IMAGE CHECKSUMS ==/) {
+			$endisos = 1;
+		}
+		if ($_ =~ m/^BUILDDATE/) {
+			$builddate = $_;
+			$builddate =~ s/^BUILDDATE=//;
+			pop(@lines);
+			next;
+		}
+		if ($_ =~ m/^SVNREV/) {
+			$svnrev = $_;
+			$svnrev =~ s/^SVNREV=//;
+			pop(@lines);
+			next;
+		}
+		if ($_ =~ m/^o /) {
+			$_ =~ s/:$//;
+			if ($_ =~ m/^o .* armv6 .*/) {
+				$hasarmv6 = 1;
+			}
+			if ($_ =~ m/^o .* aarch64 .*/) {
+				$hasarm64 = 1;
+			}
+			if ($hasbranch eq 0) {
+				($junk, $branch, $arch, $kernel) = split(" ", $_);
+				$revision = $branch;
+				$revision =~ s/-\w+.*//;
+				$version = $revision;
+				$version =~ s/\.\d.*//;
+				$branchname = $branch;
+				$branchname =~ s/\d+\.\d-//;
+				$branchname =~ s/ .*$//;
+				if ($branchname =~ m/(ALPHA|CURRENT)/) {
+					$branch = "head";
+				} elsif ($branchname =~ m/(BETA|PRERELEASE|RC|STABLE)/) {
+					$branch = "stable/$version";
+				} else {
+					$branch = "unknown";
+				}
+				$hasbranch = 1;
+			}
+			if ($endisos == 0) {
+				push(@builds, $_);
+			} else {
+				push(@vmimages, $_);
+			}
+		}
+		if ($_ =~ m/^Created AMI in /) {
+			$_ =~ s/^Created AMI in //;
+			# Exclude ca-central-1 eu-west-2 for now
+			#if ($_ !~ m/(ca-central-1|eu-west-2)/) {
+				push(@amis, $_);
+				pop(@lines);
+			#}
+		}
+	}
+
+	print <<HEADER;
+To: freebsd-snapshots\@FreeBSD.org
+Subject: New FreeBSD snapshots available: $branch ($builddate r$svnrev)
+
+HEADER
+	print <<OPENING;
+New FreeBSD development branch installation ISOs and virtual machine
+disk images have been uploaded to the FTP mirrors.
+
+As with any development branch, the installation snapshots are not
+intended for use on production systems.  We do, however, encourage
+testing on non-production systems as much as possible.
+
+Please also consider installing the sysutils/panicmail port, which can
+help in providing FreeBSD developers the necessary information regarding
+system crashes.
+
+Checksums for the installation ISOs and the VM disk images follow at
+the end of this email.
+
+=== Installation ISOs ===
+
+Installation images are available for:
+
+OPENING
+
+	foreach my $build (@builds) {
+		print("$build\n");
+	}
+
+	if ($hasarmv6 ne 0) {
+		print <<ARMINFO;
+
+Note regarding arm/armv6 images: For convenience for those without
+console access to the system, a freebsd user with a password of
+freebsd is available by default for ssh(1) access.  Additionally,
+the root user password is set to root, which it is strongly
+recommended to change the password for both users after gaining
+access to the system.
+ARMINFO
+	}
+
+	print <<OPENING;
+
+Snapshots may be downloaded from the corresponding architecture
+directory from:
+
+    ftp://ftp.freebsd.org/pub/FreeBSD/snapshots/ISO-IMAGES/
+
+Please be patient if your local FTP mirror has not yet caught
+up with the changes.
+
+Problems, bug reports, or regression reports should be reported through
+the Bugzilla PR system or the appropriate mailing list such as -current\@
+or -stable\@ .
+
+=== Virtual Machine Disk Images ===
+ 
+VM disk images are available for the following architectures:
+
+OPENING
+
+	foreach my $vmimage (@vmimages) {
+		print("$vmimage\n");
+	}
+
+	print <<OPENING;
+
+Disk images may be downloaded from the following URL (or any of the
+FreeBSD FTP mirrors):
+
+    ftp://ftp.freebsd.org/pub/FreeBSD/snapshots/VM-IMAGES/
+
+Images are available in the following disk image formats:
+
+    ~ RAW
+    ~ QCOW2 (qemu)
+    ~ VMDK (qemu, VirtualBox, VMWare)
+    ~ VHD (qemu, xen)
+
+The partition layout is:
+
+    ~ 512k - freebsd-boot GPT partition type (bootfs GPT label)
+    ~ 1GB  - freebsd-swap GPT partition type (swapfs GPT label)
+    ~ ~17GB - freebsd-ufs GPT partition type (rootfs GPT label)
+OPENING
+
+	if ($hasarm64 ne 0) {
+		print <<AARCH64;
+
+Note regarding arm64/aarch64 virtual machine images: a modified QEMU EFI
+loader file is needed for qemu-system-aarch64 to be able to boot the
+virtual machine images.  See this page for more information:
+
+    https://wiki.freebsd.org/arm64/QEMU
+
+To boot the VM image, run:
+
+    % qemu-system-aarch64 -m 4096M -cpu cortex-a57 -M virt  \\
+	-bios QEMU_EFI.fd -serial telnet::4444,server -nographic \\
+	-drive if=none,file=VMDISK,id=hd0 \\
+	-device virtio-blk-device,drive=hd0 \\
+	-device virtio-net-device,netdev=net0 \\
+	-netdev user,id=net0
+
+Be sure to replace "VMDISK" with the path to the virtual machine image.
+AARCH64
+	}
+
+	if ($#amis ne 0) {
+	print <<AMIS;
+
+=== Amazon EC2 AMI Images ===
+
+FreeBSD/amd64 EC2 AMIs are available in the following regions:
+
+AMIS
+	}
+	foreach my $ami (@amis) {
+		print(" $ami\n");
+	}
+
+	print <<VAGRANT;
+
+=== Vagrant Images ===
+
+FreeBSD/amd64 images are available on the Hashicorp Atlas site for the
+VMWare Desktop and VirtualBox providers, and can be installed by
+running:
+
+    % vagrant init freebsd/FreeBSD-$revision-$branchname
+    % vagrant up
+
+VAGRANT
+	foreach my $line (@lines) {
+		if ($line !~ m/^Created AMI in /) {
+			print("$line\n");
+		}
+	}
+
+	print <<FOOTER;
+
+Love FreeBSD?  Support this and future releases with a donation to
+the FreeBSD Foundation!  https://www.freebsdfoundation.org/donate/
+
+FOOTER
+
+	return(0);
+}
+
+&main();

Added: user/gjb/thermite/get-checksums.sh
==============================================================================
--- /dev/null	00:00:00 1970	(empty, because file is newly added)
+++ user/gjb/thermite/get-checksums.sh	Wed Mar 22 22:48:16 2017	(r315757)
@@ -0,0 +1,164 @@
+#!/bin/sh
+#
+# $FreeBSD$
+#
+
+usage() {
+	echo "$(basename ${0}) -c /path/to/configuration/file"
+	exit 1
+}
+
+get_vm_checksum() {
+	local _s="${r}-${a}-${k}-${t}"
+	sumfiles="SHA512 SHA256"
+	if [ -e ${scriptdir}/${_s}.conf ]; then
+		. ${scriptdir}/${_s}.conf
+	else
+		return 0
+	fi
+	case ${t} in
+		release)
+			type="releases"
+			;;
+		*)
+			type="snapshots"
+			;;
+	esac
+	if [ ! -e "${CHROOTDIR}/R/ftp-stage/${type}/VM-IMAGES" ]; then
+		return 0
+	fi
+	__REVISION=$(make -C ${CHROOTDIR}/usr/src/release -V REVISION)
+	__BRANCH=$(make -C ${CHROOTDIR}/usr/src/release -V BRANCH)
+	for _f in ${sumfiles}; do
+		case ${_f} in
+			SHA512)
+				echo "o ${__REVISION}-${__BRANCH} ${a}:"
+				;;
+			*)
+				;;
+		esac
+		cat ${CHROOTDIR}/R/ftp-stage/${type}/VM-IMAGES/${__REVISION}-${__BRANCH}/${TARGET_ARCH}/Latest/CHECKSUM.${_f}* | \
+			sed -e 's/^/  /'
+		echo
+	done
+	echo
+	return 0
+}
+
+
+get_iso_checksum() {
+	local _s="${r}-${a}-${k}-${t}"
+	sumfiles="SHA512 SHA256"
+	if [ -e ${scriptdir}/${_s}.conf ]; then
+		. ${scriptdir}/${_s}.conf
+	else
+		return 0
+	fi
+	if [ ! -e ${CHROOTDIR}/R/ ]; then
+		return 0
+	fi
+	case ${t} in
+		release)
+			type="releases"
+			;;
+		*)
+			type="snapshots"
+			;;
+	esac
+	__REVISION=$(make -C ${CHROOTDIR}/usr/src/release -V REVISION)
+	__BRANCH=$(make -C ${CHROOTDIR}/usr/src/release -V BRANCH)
+	if [ ! -z "${EMBEDDEDBUILD}" ]; then
+		TARGET="${EMBEDDED_TARGET}"
+		TARGET_ARCH="${EMBEDDED_TARGET_ARCH}"
+	fi
+	for _f in ${sumfiles}; do
+		case ${_f} in
+			SHA512)
+				echo "o ${__REVISION}-${__BRANCH} ${a} ${k}:"
+				;;
+			*)
+				;;
+		esac
+		cat ${CHROOTDIR}/R/ftp-stage/${type}/${TARGET}/${TARGET_ARCH}/ISO-IMAGES/${__REVISION}/CHECKSUM.${_f}* | \
+			sed -e 's/^/  /'
+		echo
+	done
+	unset EMBEDDEDBUILD
+	echo
+	return 0
+}
+
+main() {
+	export __BUILDCONFDIR="$(dirname $(realpath ${0}))"
+	CSCONF=
+
+	while getopts "c:" opt; do
+		case ${opt} in
+			c)
+				CSCONF="${OPTARG}"
+				;;
+			*)
+				;;
+		esac
+	done
+
+	if [ -z "${CSCONF}" ]; then
+		echo "Build configuration file is required."
+		usage
+	fi
+
+	CSCONF="$(realpath ${CSCONF})"
+
+	if [ ! -f "${CSCONF}" ]; then
+		echo "Build configuration is not a regular file."
+		exit 1
+	fi
+
+	. "${CSCONF}"
+
+	case ${t} in
+		release)
+			;;
+		*)
+			if [ -e "builddate" ]; then
+				echo "BUILDDATE=$(cat builddate)"
+			fi
+			if [ -e "svnrev_src" ]; then
+				echo "SVNREV=$(cat svnrev_src)"
+			fi
+			tail -n50 ../logs/*.ec2* | grep -E '^Created AMI in' \
+				|| true
+			;;
+	esac
+
+	echo "== ISO CHECKSUMS =="
+	echo
+	for r in ${revs}; do
+		for a in ${archs}; do
+			for k in ${kernels}; do
+			for t in ${types}; do
+				get_iso_checksum
+			done
+			done
+		done
+	done
+	echo "== VM IMAGE CHECKSUMS =="
+	echo
+	for r in ${revs}; do
+		for a in ${archs}; do
+			for k in ${kernels}; do
+			for t in ${types}; do
+				case ${a} in
+					amd64|i386|aarch64)
+						get_vm_checksum
+						;;
+					*)
+					;;
+				esac
+			done
+			done
+		done
+	done
+}
+
+main "$@"

Added: user/gjb/thermite/setrev.sh
==============================================================================
--- /dev/null	00:00:00 1970	(empty, because file is newly added)
+++ user/gjb/thermite/setrev.sh	Wed Mar 22 22:48:16 2017	(r315757)
@@ -0,0 +1,63 @@
+#!/bin/sh
+#
+# $FreeBSD$
+#
+
+# TODO: add support for ports/ and doc/ tree
+
+usage() {
+	echo "Usage:"
+	echo "${0} -b <branch>"
+	exit 1
+}
+
+get_build_rev() {
+	if [ -e "${svnfile}" -a ! -f "${svnfile}" ]; then
+		echo "SVN file is not a regular file."
+		echo "Renaming file."
+		mv ${svnfile} ${svnfile}.bak.${today}
+	fi
+	svnrev=$(svn info ${svnhost}/${branch} | \
+		awk -F ': ' '/^Last Changed Rev/ {print $2}')
+	svnrev=$(echo ${svnrev} | tr -d '[a-z]')
+	echo ${svnrev} > ${svnfile}
+}
+
+get_build_date() {
+	if [ -e "${datefile}" -a ! -f "${datefile}" ]; then
+		echo "SVN file is not a regular file."
+		echo "Renaming file."
+		mv ${svnfile} ${svnfile}.bak.${today}
+	fi
+	echo ${today} > ${datefile}
+}
+
+main() {
+	export PATH="/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin"
+	export TZ='UTC'
+	export __BUILDCONFDIR="$(dirname $(realpath ${0}))"
+	svnhost="svn://svn.FreeBSD.org/base"
+	svnfile="${__BUILDCONFDIR}/svnrev_src"
+	datefile="${__BUILDCONFDIR}/builddate"
+	today="$(date +%Y%m%d)"
+
+	while getopts "b:" opt; do
+		case ${opt} in
+			b)
+				branch="${OPTARG}"
+				;;
+			*)
+				;;
+		esac
+	done
+
+	if [ -z "${branch}" ]; then
+		echo "Branch not specified."
+		usage
+	fi
+
+	get_build_rev
+	get_build_date
+}
+
+main "$@"

Added: user/gjb/thermite/zfs-setup.sh
==============================================================================
--- /dev/null	00:00:00 1970	(empty, because file is newly added)
+++ user/gjb/thermite/zfs-setup.sh	Wed Mar 22 22:48:16 2017	(r315757)
@@ -0,0 +1,161 @@
+#!/bin/sh
+#
+# $FreeBSD$
+#
+
+usage() {
+	echo "$(basename ${0}) [-d] -c /path/to/configuration/file"
+	exit 1
+}
+
+zfs_teardown() {
+	for r in ${revs}; do
+		for a in ${archs}; do
+			for k in ${kernels}; do
+			for t in ${types}; do
+				s="${r}-${a}-${k}-${t}"
+				c="${r}-${a}-${t}"
+				if [ -e ${scriptdir}/${s}.conf ];
+				then
+					zfs list ${zfs_parent}/${s}-src >/dev/null 2>&1
+					rc=$?
+					if [ ${rc} -eq 0 ]; then
+						echo -n "${pfx} Destroying " \
+							>/dev/stdout
+						echo " ${zfs_parent}/${s}-src" \
+							>/dev/stdout
+						zfs destroy -f ${zfs_parent}/${s}-src
+					fi
+					zfs list ${zfs_parent}/${s}-ports >/dev/null 2>&1
+					rc=$?
+					if [ ${rc} -eq 0 ]; then
+						echo -n "${pfx} Destroying " \
+							>/dev/stdout
+						echo " ${zfs_parent}/${s}-ports" \
+							>/dev/stdout
+						zfs destroy -f ${zfs_parent}/${s}-ports
+					fi
+					zfs list ${zfs_parent}/${s}-doc >/dev/null 2>&1
+					rc=$?
+					if [ ${rc} -eq 0 ]; then
+						echo -n "${pfx} Destroying " \
+							>/dev/stdout
+						echo " ${zfs_parent}/${s}-doc" \
+							>/dev/stdout
+						zfs destroy -f ${zfs_parent}/${s}-doc
+					fi
+					zfs list ${zfs_parent}/${c}-chroot >/dev/null 2>&1
+					rc=$?
+					if [ ${rc} -eq 0 ]; then
+						echo -n "${pfx} Destroying " \
+							>/dev/stdout
+						echo " ${zfs_parent}/${c}-chroot" \
+							>/dev/stdout
+						zfs destroy -f ${zfs_parent}/${c}-chroot
+					fi
+					zfs list ${zfs_parent}/${s} >/dev/null 2>&1
+					rc=$?
+					if [ ${rc} -eq 0 ]; then
+						echo -n "${pfx} Destroying " \
+							>/dev/stdout
+						echo " ${zfs_parent}/${s}" \
+							>/dev/stdout
+						zfs destroy -f ${zfs_parent}/${s}
+					fi
+				fi
+			done
+			done
+		done
+	done
+
+	for r in ${revs}; do
+		for t in ${types}; do
+			for i in src doc ports; do
+				zfs list ${zfs_parent}/${r}-${i}-${t}@clone >/dev/null 2>&1
+				rc=$?
+				if [ ${rc} -eq 0 ]; then
+					echo -n "${pfx} Destroying " \
+						>/dev/stdout
+					echo " ${zfs_parent}/${r}-${i}-${t}@clone" \
+						>/dev/stdout
+					zfs destroy -f ${zfs_parent}/${r}-${i}-${t}@clone
+				fi
+				zfs list ${zfs_parent}/${r}-${i}-${t} >/dev/null 2>&1
+				rc=$?
+				if [ ${rc} -eq 0 ]; then
+					echo -n "${pfx} Destroying " \
+						>/dev/stdout
+					echo " ${zfs_parent}/${r}-${i}-${t}" \
+						>/dev/stdout
+					zfs destroy -f ${zfs_parent}/${r}-${i}-${t}
+				fi
+			done
+		done
+	done
+	return 0
+}
+
+zfs_setup() {
+	[ ! -z ${delete_only} ] && return 0
+	for r in ${revs}; do
+		for a in ${archs}; do
+			for k in ${kernels}; do
+			for t in ${types}; do
+				s="${r}-${a}-${k}-${t}"
+				if [ -e ${scriptdir}/${s}.conf ];
+				then
+					echo "${pfx} Creating ${zfs_parent}/${s}" \
+						>/dev/stdout
+					zfs create -o atime=off ${zfs_parent}/${s}
+				fi
+			done
+			done
+		done
+	done
+	return 0
+}
+
+main() {
+	export __BUILDCONFDIR="$(dirname $(realpath ${0}))"
+	CSCONF=
+
+	while getopts "c:d" opt; do
+		case ${opt} in
+			c)
+				CSCONF="${OPTARG}"
+				;;
+			d)
+				delete_only=1
+				;;
+			*)
+				;;
+		esac
+	done
+	shift $(( ${OPTIND} - 1 ))
+
+	if [ -z "${CSCONF}" ]; then
+		echo "Build configuration file is required."
+		usage
+	fi
+
+	CSCONF="$(realpath ${CSCONF})"
+
+	if [ ! -f "${CSCONF}" ]; then
+		echo "Build configuration is not a regular file."
+		exit 1
+	fi
+
+	. "${CSCONF}"
+
+	if [ ${use_zfs} -eq 0 ]; then
+		echo "== use_zfs is set to '0'; skipping." >/dev/stdout
+		exit 0
+	fi
+
+	pfx="==="
+
+	zfs_teardown
+	zfs_setup
+}
+
+main "$@"