Date: Tue, 29 Jan 2013 03:27:24 +0000 (UTC) From: Mark Linimon <linimon@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r246051 - in projects/portbuild: admin/scripts scripts Message-ID: <201301290327.r0T3ROB1033678@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: linimon (doc,ports committer) Date: Tue Jan 29 03:27:24 2013 New Revision: 246051 URL: http://svnweb.freebsd.org/changeset/base/246051 Log: Move scripts that need to be only owned by administrator to admin/scripts. These must not be writeable by portbuild due to security concerns. Added: projects/portbuild/admin/scripts/updatesnap - copied unchanged from r245975, projects/portbuild/scripts/updatesnap projects/portbuild/admin/scripts/updatesnap.ports - copied unchanged from r245975, projects/portbuild/scripts/updatesnap.ports projects/portbuild/admin/scripts/zbackup - copied unchanged from r245975, projects/portbuild/scripts/zbackup projects/portbuild/admin/scripts/zexpire - copied unchanged from r245975, projects/portbuild/scripts/zexpire Deleted: projects/portbuild/scripts/updatesnap projects/portbuild/scripts/updatesnap.ports projects/portbuild/scripts/zbackup projects/portbuild/scripts/zexpire Copied: projects/portbuild/admin/scripts/updatesnap (from r245975, projects/portbuild/scripts/updatesnap) ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ projects/portbuild/admin/scripts/updatesnap Tue Jan 29 03:27:24 2013 (r246051, copy of r245975, projects/portbuild/scripts/updatesnap) @@ -0,0 +1,97 @@ +#!/bin/sh +# +# Update the master source trees that are used by package builds +# and other consumers + +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} + +. ${pbc}/conf/server.conf + +base=${ZFS_MOUNTPOINT}/${SNAP_DIRECTORY} +zbase=${ZFS_VOLUME}/${SNAP_DIRECTORY} +base_prefix=${ZFS_MOUNTPOINT}/${SNAP_DIRECTORY}/${SRC_DIRECTORY_PREFIX} +zbase_prefix=${ZFS_VOLUME}/${SNAP_DIRECTORY}/${SRC_DIRECTORY_PREFIX} + +VERBOSE=1 + +stamp() { + fulldate=$1 + date -j -f %+ "${fulldate}" +%Y%m%d%H%M%S +} + +finish() { + err=$1 + + end=$(date +%s) + echo "Finished at $(date)" + len=$((end-begin)) + echo "Duration = $(date -j -f %s +%H:%M:%S ${len})" + exit 1 +} + +begin=$(date +%s) +echo "Started at $(date)" + +uid=${PORTBUILD_USER} +if [ ! -z "${PORTBUILD_GROUP}" ]; then + gid=${PORTBUILD_GROUP} +else + gid=${uid} +fi + +# create /a/snap if not already there +if [ ! -d ${base} ]; then + echo "creating new base directory using ${base}" + zfs create ${zbase} || finish 1 + chown -R ${uid}:${gid} ${base} + chmod -R g+w ${base} +fi + +for branch in $SRC_BRANCHES; do + # create /a/snap/src-<branch> if not already there + mountpoint=${base_prefix}${branch} + if [ ! -d ${mountpoint} ]; then + echo "creating new source branch using ${mountpoint}" + zfs create ${zbase_prefix}${branch} || finish 1 + chown -R ${uid}:${gid} ${mountpoint} + chmod -R g+w ${mountpoint} + fi + # create /a/snap/src-<branch>/src if not already there + if [ ! -d ${mountpoint}/src ]; then + echo "creating new source branch subdirectory using ${mountpoint}/src" + zfs create ${zbase_prefix}${branch}/src || finish 1 + chown -R ${uid}:${gid} ${mountpoint}/src + chmod -R g+w ${mountpoint}/src + fi + cd ${mountpoint} + fulldate=$(date) + + # TODO: fix identation post-rework + + # example destination directory: /a/snap/src-7/src/ (tricky!) + if [ ! -d ${mountpoint}/src ]; then + mkdir ${mountpoint}/src || finish 1 + fi + if [ ! -d ${mountpoint}/src/${VCS_SUBDIR} ]; then + eval subdir=\$SRC_BRANCH_${branch}_SUBDIR + if [ $VERBOSE ]; then + echo "${VCS_CHECKOUT_COMMAND} ${VCS_SRC_REPOSITORY}/${subdir} ${mountpoint}/src" + fi + ${VCS_CHECKOUT_COMMAND} ${VCS_SRC_REPOSITORY}/${subdir} ${mountpoint}/src || finish 1 + else + if [ $VERBOSE ]; then + echo "${VCS_OVERWRITE_COMMAND} ${mountpoint}/src" + fi + ${VCS_OVERWRITE_COMMAND} ${mountpoint}/src || finish 1 + fi + # hack for zfs breakiness + find . -group wheel | xargs chgrp ${gid} + snapdate=$(stamp ${fulldate}) + if [ $VERBOSE ]; then + echo "zfs snapshot ${zbase_prefix}${branch}/src@${snapdate}" + fi + zfs snapshot ${zbase_prefix}${branch}/src@${snapdate} || finish 1 + echo ${fulldate} > src/.updated +done + +finish 0 Copied: projects/portbuild/admin/scripts/updatesnap.ports (from r245975, projects/portbuild/scripts/updatesnap.ports) ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ projects/portbuild/admin/scripts/updatesnap.ports Tue Jan 29 03:27:24 2013 (r246051, copy of r245975, projects/portbuild/scripts/updatesnap.ports) @@ -0,0 +1,88 @@ +#!/bin/sh +# +# Update the master ports tree that is used by package builds +# and other consumers + +pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} + +. ${pbc}/conf/server.conf + +base=${ZFS_MOUNTPOINT}/${SNAP_DIRECTORY} +zbase=${ZFS_VOLUME}/${SNAP_DIRECTORY} +head=${ZFS_MOUNTPOINT}/${SNAP_PORTS_DIRECTORY} +zhead=${ZFS_VOLUME}/${SNAP_PORTS_DIRECTORY} + +VERBOSE=1 + +stamp() { + fulldate=$1 + date -j -f %+ "${fulldate}" +%Y%m%d%H%M%S +} + +finish() { + err=$1 + + end=$(date +%s) + echo "Finished at $(date)" + len=$((end-begin)) + echo "Duration = $(date -j -f %s +%H:%M:%S ${len})" + exit 1 +} + +begin=$(date +%s) +echo "Started at $(date)" + +uid=${PORTBUILD_USER} +if [ ! -z "${PORTBUILD_GROUP}" ]; then + gid=${PORTBUILD_GROUP} +else + gid=${uid} +fi + +# create /a/snap/ if not already there +if [ ! -d ${base} ]; then + echo "creating new base directory using ${base}" + zfs create ${zbase} || finish 1 + chown -R ${uid}:${gid} ${base} + chmod -R g+w ${base} +fi + +# create /a/snap/ports-head/ if not already there +if [ ! -d ${head} ]; then + echo "creating new ports-head directory using ${head}" + zfs create ${zhead} || finish 1 + chown -R ${uid}:${gid} ${head} + chmod -R g+w ${head} +fi + +# create /a/snap/ports-head/ports/ if not already there +if [ ! -d ${head}/ports ]; then + echo "creating new ports-head/ports directory using ${head}/ports" + zfs create ${zhead}/ports || finish 1 + chown -R ${uid}:${gid} ${head}/ports + chmod -R g+w ${head}/ports +fi + +cd ${head} +fulldate=$(date) +if [ ! -d ${head}/ports/${VCS_SUBDIR} ]; then + if [ $VERBOSE ]; then + echo "${VCS_CHECKOUT_COMMAND} ${VCS_PORTS_REPOSITORY}/head ${head}/ports" + fi + ${VCS_CHECKOUT_COMMAND} ${VCS_PORTS_REPOSITORY}/head ${head}/ports || finish 1 +else + if [ $VERBOSE ]; then + echo "${VCS_OVERWRITE_COMMAND} ${head}/ports" + fi + ${VCS_OVERWRITE_COMMAND} ${head}/ports || finish 1 +fi +# hack for zfs breakiness +find . -group wheel | xargs chgrp ${gid} +snapdate=$(stamp ${fulldate}) +if [ $VERBOSE ]; then + echo "zfs snapshot ${zhead}/ports@${snapdate}" +fi +zfs snapshot ${zhead}/ports@${snapdate} || finish 1 +echo ${fulldate} > ports/.updated + +finish 0 Copied: projects/portbuild/admin/scripts/zbackup (from r245975, projects/portbuild/scripts/zbackup) ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ projects/portbuild/admin/scripts/zbackup Tue Jan 29 03:27:24 2013 (r246051, copy of r245975, projects/portbuild/scripts/zbackup) @@ -0,0 +1,236 @@ +#!/usr/bin/env python + +# Back up a list of ZFS filesystems, doing a full backup periodically +# and using incremental diffs in between + +import commands, datetime, sys, os, bz2 +from signal import * + +sys.path.insert(0, '/var/portbuild/lib/python') + +import zfs + +# List of filesystems to backup +# XXX MCL +backuplist=["a", "a/nfs", "a/local", "a/portbuild", + "a/portbuild/amd64", "a/portbuild/arm", "a/portbuild/i386", + "a/portbuild/ia64", "a/portbuild/mips", "a/portbuild/powerpc", + "a/portbuild/sparc64"] + +# Directory to store backups +# XXX MCL +backupdir="/dumpster/pointyhat/backup" + +# How many days between full backups +# XXX MCL +fullinterval=9 + +def validate(): + fslist = zfs.getallfs() + + missing = set(backuplist).difference(set(fslist)) + if len(missing) > 0: + print "Backup list refers to filesystems that do not exist: %s" % missing + sys.exit(1) + +def mkdirp(path): + + plist = path.split("/") + + for i in xrange(2,len(plist)+1): + sofar = "/".join(plist[0:i]) + if not os.path.isdir(sofar): + os.mkdir(sofar) + +class node(object): + child=None + parent=None + name=None + visited=0 + + def __init__(self, name): + self.name = name + self.child = [] + self.parent = None + self.visited = 0 + +print "zbackup: starting at " + datetime.datetime.now().ctime() + +for fs in backuplist: + + print + + dir = backupdir + "/" + fs + mkdirp(dir) + + snaplist = None + try: + snaplist = [snap[0] for snap in zfs.getallsnaps(fs) if snap[0].isdigit()] + except zfs.NoSuchFS: + print "no such fs %s, skipping" % fs + continue + + dofull = 0 + + # Mapping from backup date tag to node + backups={} + + # list of old-new pairs seen + seen=[] + + # Most recent snapshot date + latest = "0" + for j in os.listdir(dir): + (old, sep, new) = j.partition('-') + if not old.isdigit() or not new.isdigit(): + continue + + seen.append("%s-%s" % (old, new)) + + if int(old) >= int(new): + print "Warning: backup sequence not monotonic: %s >= %s" % (old, new) + continue + + try: + oldnode = backups[old] + except KeyError: + oldnode = node(old) + backups[old] = oldnode + + try: + newnode = backups[new] + except KeyError: + newnode = node(new) + backups[new] = newnode + + if int(new) > int(latest): + latest = new + + oldnode.child.append(newnode) + if newnode.parent: + # We are not a tree! + if not dofull: + print "Multiple backup sequences found, forcing full dump!" + dofull = 1 + continue + + newnode.parent = oldnode + + if not "0" in backups and not dofull: + # No root! + print "No full backup found!" + dofull = 1 + + if not latest in snaplist and not dofull: + print "Latest dumped snapshot no longer exists: forcing full dump" + dofull = 1 + + now = datetime.datetime.now() + nowdate = now.strftime("%Y%m%d%H%M") + + try: + prev = datetime.datetime.strptime(latest, "%Y%m%d%H%M") + except ValueError: + if not dofull: + print "Unable to parse latest snapshot as a date, forcing full dump!" + dofull = 1 + + print "Creating zfs snapshot %s@%s" % (fs, nowdate) + zfs.createsnap(fs, nowdate) + + # Find path from latest back to root + try: + cur = backups[latest] + except KeyError: + cur = None + + chain = [] + firstname = "0" + # Skip if latest doesn't exist or chain is corrupt + while cur: + chain.append("%s-%s" % (cur.parent.name, cur.name)) + par = cur.parent + + # Remove from the backup tree so we can delete the leftovers + # below + par.child.remove(cur) + cur.parent=None + + if par.name == "0": + firstname = cur.name + break + cur = par + + chain.reverse() + + print "chain is " + str( chain ) + + # Prune stale links not in the backup chain + for j in backups.iterkeys(): + cur = backups[j] + for k in cur.child: + stale="%s-%s" % (cur.name, k.name) + print "Deleting stale backup %s" % stale + os.remove("%s/%s/%s" % (backupdir, fs, stale)) + + # Lookup date of full dump + try: + first = datetime.datetime.strptime(firstname, "%Y%m%d%H%M") + except ValueError: + if not dofull: + print "Unable to parse first snapshot as a date, forcing full dump!" + dofull = 1 + + if not dofull and (now - first) > datetime.timedelta(days=fullinterval): + print "Previous full backup too old, forcing full dump!" + dofull = 1 + + # In case we are interrupted don't leave behind a truncated file + # that will corrupt the backup chain + + if dofull: + latest = "0" + + outfile="%s/%s/.%s-%s" % (backupdir, fs, latest, nowdate) + + # zfs send aborts on receiving a signal + signal(SIGTSTP, SIG_IGN) + if not dofull: + print "Doing incremental backup of %s: %s-%s" % (fs, latest, nowdate) + (err, out) = \ + commands.getstatusoutput("zfs send -i %s %s@%s | bzip2 > %s" % + (latest, fs, nowdate, outfile)) + else: + print "Doing full backup of %s" % fs + latest = "0" + (err, out) = \ + commands.getstatusoutput("zfs send %s@%s | bzip2 > %s" % + (fs, nowdate, outfile)) + signal(SIGTSTP, SIG_DFL) + + if err: + print "Error from snapshot: (%s, %s)" % (err, out) + try: + os.remove(outfile) + print "Deleted file %s" % outfile + except OSError, err: + print "OSError: " + repr(err) + if err.errno != 2: + raise + finally: + sys.exit(1) + + # We seem to be finished + try: + os.rename(outfile, "%s/%s/%s-%s" % (backupdir, fs, latest, nowdate)) + except: + print "Error renaming dump file" + outfile + "!" + raise + + if dofull: + for i in seen: + print "Removing stale snapshot %s/%s" % (dir, i) + os.remove("%s/%s" % (dir, i)) + +print +print "zbackup: ending at " + datetime.datetime.now().ctime() Copied: projects/portbuild/admin/scripts/zexpire (from r245975, projects/portbuild/scripts/zexpire) ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ projects/portbuild/admin/scripts/zexpire Tue Jan 29 03:27:24 2013 (r246051, copy of r245975, projects/portbuild/scripts/zexpire) @@ -0,0 +1,129 @@ +#!/usr/bin/env python +# +# Expire old snapshots + +import sys, commands, datetime, os, string + +pbc = os.getenv('PORTBUILD_CHECKOUT') \ + if os.getenv('PORTBUILD_CHECKOUT') else "/a/portbuild" +pbd = os.getenv('PORTBUILD_DATA') \ + if os.getenv('PORTBUILD_DATA') else "/a/portbuild" + +sys.path.insert(0, '%s/lib/python' % pbc) + +from freebsd import * +from freebsd_config import * +import zfs + +CONFIG_SUBDIR="conf" +CONFIG_FILENAME="server.conf" + +ENABLED = True +VERBOSE= True + +expirelist={} + +now = datetime.datetime.now() +print "zexpire: starting at " + now.ctime() + +config = getConfig( pbc, CONFIG_SUBDIR, CONFIG_FILENAME ) +ZFS_VOLUME = config.get( 'ZFS_VOLUME' ) +if not ZFS_VOLUME: + print "you must define ZFS_VOLUME" + sys.exit( 1 ) +ZFS_MOUNTPOINT = config.get( 'ZFS_MOUNTPOINT' ) +if not ZFS_MOUNTPOINT: + print "you must define ZFS_MOUNTPOINT" + sys.exit( 1 ) +PORTBUILD_DIRECTORY = config.get( 'PORTBUILD_DIRECTORY' ) +if not PORTBUILD_DIRECTORY: + print "you must define PORTBUILD_DIRECTORY" + sys.exit( 1 ) +portbuild_directory = os.path.join( ZFS_VOLUME, PORTBUILD_DIRECTORY ) +SNAP_DIRECTORY = config.get( 'SNAP_DIRECTORY' ) +if not SNAP_DIRECTORY: + print "you must define SNAP_DIRECTORY" + sys.exit( 1 ) +snap_directory = os.path.join( ZFS_VOLUME, SNAP_DIRECTORY ) +snap_mountpoint = os.path.join( ZFS_MOUNTPOINT, SNAP_DIRECTORY ) +SUPPORTED_ARCHS = config.get( 'SUPPORTED_ARCHS' ) +if not SUPPORTED_ARCHS: + print "you must define SUPPORTED_ARCHS" + sys.exit( 1 ) +supported_archs = string.split( SUPPORTED_ARCHS ) +ZFS_DEFAULT_EXPIRATION = config.get( 'ZFS_DEFAULT_EXPIRATION' ) +if not ZFS_DEFAULT_EXPIRATION: + print "you must define ZFS_DEFAULT_EXPIRATION" + sys.exit( 1 ) +ZFS_SNAPSHOT_EXPIRATION = config.get( 'ZFS_SNAPSHOT_EXPIRATION' ) +if not ZFS_SNAPSHOT_EXPIRATION: + print "you must define ZFS_SNAPSHOT_EXPIRATION" + sys.exit( 1 ) + +expirelist[ ZFS_VOLUME ] = ZFS_DEFAULT_EXPIRATION +expirelist[ portbuild_directory ] = ZFS_DEFAULT_EXPIRATION +for arch in supported_archs: + expirelist[ os.path.join( portbuild_directory, arch ) ] = ZFS_DEFAULT_EXPIRATION +expirelist[ snap_directory ] = ZFS_SNAPSHOT_EXPIRATION +try: + snapdirs = os.listdir( snap_mountpoint ) + for snapdir in snapdirs: + subdir = os.path.join( snap_mountpoint, snapdir ) + tmp = os.path.join( snap_directory, snapdir ) + expirelist[ tmp ] = ZFS_SNAPSHOT_EXPIRATION + try: + snapsubdirs = os.listdir( subdir ) + for snapsubdir in snapsubdirs: + expirelist[ os.path.join( tmp, snapsubdir ) ] = ZFS_SNAPSHOT_EXPIRATION + except: + pass +except: + pass + +keys = expirelist.keys() +keys.sort() +for key in keys: + fs = key + maxage = 2 + try: + maxage = int( expirelist[ key ] ) + except: + pass + + print + + if VERBOSE: + print "fs: " + str(fs) + try: + snapdata = zfs.getallsnaps(fs) + if VERBOSE: + print "snapdata: " + str(snapdata) + except zfs.NoSuchFS: + print "zexpire: no such fs %s, skipping" % fs + continue + + snaps = (i[0] for i in snapdata) + + for snap in snaps: + try: + snapdate = datetime.datetime.strptime(snap, "%Y%m%d%H%M") + except ValueError: + try: + snapdate = datetime.datetime.strptime(snap, "%Y%m%d%H%M%S") + except ValueError: + print "zexpire: don't know what to do with snap `" + snap + "'" + continue + + if VERBOSE: + print "zexpire: examining snapshot %s@%s" % (fs, snap) + if (now - snapdate) > datetime.timedelta(days=maxage): + print "zexpire: snapshot %s@%s too old, attempting zfs destroy" % (fs, snap) + if ENABLED: + (err, out) = commands.getstatusoutput("zfs destroy %s@%s" % (fs,snap)) + + if err: + print "zexpire: error deleting snapshot", out + +then = datetime.datetime.now() +print +print "zexpire: ending at " + then.ctime()
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201301290327.r0T3ROB1033678>