From owner-svn-src-projects@FreeBSD.ORG Tue Apr 26 20:25:17 2011 Return-Path: Delivered-To: svn-src-projects@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 513341065676; Tue, 26 Apr 2011 20:25:17 +0000 (UTC) (envelope-from flz@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 413B78FC1A; Tue, 26 Apr 2011 20:25:17 +0000 (UTC) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.4/8.14.4) with ESMTP id p3QKPHlT037295; Tue, 26 Apr 2011 20:25:17 GMT (envelope-from flz@svn.freebsd.org) Received: (from flz@localhost) by svn.freebsd.org (8.14.4/8.14.4/Submit) id p3QKPHnn037284; Tue, 26 Apr 2011 20:25:17 GMT (envelope-from flz@svn.freebsd.org) Message-Id: <201104262025.p3QKPHnn037284@svn.freebsd.org> From: Florent Thoumie Date: Tue, 26 Apr 2011 20:25:17 +0000 (UTC) To: src-committers@freebsd.org, svn-src-projects@freebsd.org X-SVN-Group: projects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r221080 - in projects/portbuild: lib lib/python qmanager scripts X-BeenThere: svn-src-projects@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the src " projects" tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 26 Apr 2011 20:25:17 -0000 Author: flz Date: Tue Apr 26 20:25:16 2011 New Revision: 221080 URL: http://svn.freebsd.org/changeset/base/221080 Log: portbuild: add homemade python modules. And adjust sys.path so we look into /var/portbuild before site-packages. Added: projects/portbuild/lib/ projects/portbuild/lib/python/ projects/portbuild/lib/python/freebsd.py projects/portbuild/lib/python/freebsd_config.py projects/portbuild/lib/python/zfs.py Modified: projects/portbuild/qmanager/dumpdb.py projects/portbuild/qmanager/packagebuild projects/portbuild/qmanager/qmanager projects/portbuild/qmanager/qmanager.py projects/portbuild/scripts/buildproxy projects/portbuild/scripts/buildproxy-client projects/portbuild/scripts/zbackup projects/portbuild/scripts/zexpire Added: projects/portbuild/lib/python/freebsd.py ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ projects/portbuild/lib/python/freebsd.py Tue Apr 26 20:25:16 2011 (r221080) @@ -0,0 +1,60 @@ +#!/usr/bin/env python + +# FreeBSD support modules + +import struct, pwd, grp + +def getpeerid(sock): + """ Get peer credentials on a UNIX domain socket. + + Returns a nested tuple: (uid, (gids)) """ + + LOCAL_PEERCRED = 0x001 + NGROUPS = 16 + +#struct xucred { +# u_int cr_version; /* structure layout version */ +# uid_t cr_uid; /* effective user id */ +# short cr_ngroups; /* number of groups */ +# gid_t cr_groups[NGROUPS]; /* groups */ +# void *_cr_unused1; /* compatibility with old ucred */ +#}; + + xucred_fmt = '2ih16iP' + res = tuple(struct.unpack(xucred_fmt, sock.getsockopt(0, LOCAL_PEERCRED, struct.calcsize(xucred_fmt)))) + + # Check this is the above version of the structure + if res[0] != 0: + raise OSError + + return (res[1], res[3:3+res[2]]) + +def getuidbyname(username): + if str(username).isdigit(): + return int(username) + return pwd.getpwnam(username)[2] + +def getgidbyname(grname): + if str(grname).isdigit(): + return int(grname) + return grp.getgrnam(grname)[2] + +if __name__ == "__main__": + + MYSOCK='/tmp/mysock' + import os, socket + + if os.path.exists(MYSOCK): + os.unlink(MYSOCK) + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + s.bind(MYSOCK) + os.chmod(MYSOCK, 0666) + s.listen(10) + (conn, addr) = s.accept() + + print getpeerid(conn) + + for i in conn.makefile(): + print i.rstrip() + conn.close() + Added: projects/portbuild/lib/python/freebsd_config.py ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ projects/portbuild/lib/python/freebsd_config.py Tue Apr 26 20:25:16 2011 (r221080) @@ -0,0 +1,29 @@ +#!/usr/bin/env python + +# utility to read a config file and return a dict. Author: linimon. + +# note that the config file is shared between /bin/sh scripts and +# Python programs. If we ever wanted to deal with the {} expansions, +# we would have to include that here. Note that this is NOT currently +# implemented. + +from configobj import ConfigObj +import os + +def getConfig( configDir, configSubdir, configFilename ): + + return getConfigFromFile( \ + os.path.join( \ + os.path.join( configDir, configSubdir ), \ + configFilename ) ) + + +def getConfigFromFile( configFile ): + + config = None + try: + config = ConfigObj( configFile ) + return config + except Exception, e: + printf( "getConfig: could not read config file %s", configFile ) + return None Added: projects/portbuild/lib/python/zfs.py ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ projects/portbuild/lib/python/zfs.py Tue Apr 26 20:25:16 2011 (r221080) @@ -0,0 +1,120 @@ +import commands, os + +class NoSuchFS(Exception): + pass + +class NoSuchSnap(Exception): + pass + +def getallfs(): + """ Get list of all filesystems """ + + (err, out) = commands.getstatusoutput("zfs list -Ht filesystem") + if err: + raise (OSError, err) + + res=[] + for i in out.split('\n'): + (fs, used, avail, refer, mountpt) = i.split() + res.append((fs, used, avail, refer, mountpt)) + + return tuple(res) + +def getfs(fs): + """ Return status of a specific filesystem """ + + (err, out) = commands.getstatusoutput("zfs list -Ht filesystem '%s'" % fs) + if err: + if "dataset does not exist" in out: + raise NoSuchFS + print "err = %s, out = %s" % (err, out) + raise (OSError, err) + + (fs, used, avail, refer, mountpt) = out.split() + return (fs, used, avail, refer, mountpt) + +def getallsnaps(fs): + + """ + Return list of all snapshots for a specific filesystem + + Entries are: (snap, used, avail, refer, mountpoint) + where snap is the snapshot name (i.e. after "@") + """ + + if len(getfs(fs)) == 0: + raise NoSuchFS + + (err, out) = commands.getstatusoutput("zfs list -Ht snapshot") + if err: + print "err = %s, out = %s" % (err, out) + raise (OSError, err) + + res = [] + for i in out.split('\n'): + (name, used, avail, refer, mountpoint) = i.split() + if name.startswith("%s@" % fs): + snap=name.partition("@")[2] + res.append((snap, used, avail, refer, mountpoint)) + + return tuple(res) + +def getsnap(fs, snap): + """ Return a specific snapshot on a specific filesystem """ + + snaps = getallsnaps(fs) + + for i in snaps: + if i[0] == snap: + return i + + raise NoSuchSnap + +def createsnap(fs, name): + (err, out) = commands.getstatusoutput("zfs snapshot %s@%s" % (fs, name)) + if err: + print "err = %s, out = %s" % (err, out) + raise (OSError, err) + +def send(fs, old, new = None): + +# if new: +# return os.popen("echo %s@%s %s@%s" % (fs, old, fs, new)) +# else: +# return os.popen("echo %s@%s" % (fs, old)) + + if new: + return os.popen("zfs send -i %s %s@%s" % (old, fs, new)) + else: + return os.popen("zfs send %s@%s" % (fs, old)) + +if __name__ == "__main__": + print getallfs() + + print getfs("a") + + + print getallsnaps("a") + + print getsnap("a", "20080531") + + try: + getsnap("b", "20080531") + except NoSuchFS: + pass + + try: + getfs("b") + except NoSuchFS: + pass + + try: + getallsnaps("b") + except NoSuchFS: + pass + + try: + getsnap("a", "foo") + except NoSuchSnap: + pass + Modified: projects/portbuild/qmanager/dumpdb.py ============================================================================== --- projects/portbuild/qmanager/dumpdb.py Tue Apr 26 20:14:29 2011 (r221079) +++ projects/portbuild/qmanager/dumpdb.py Tue Apr 26 20:25:16 2011 (r221080) @@ -1,14 +1,17 @@ # # try doing some SQL reads as a test # -from freebsd_config import * -import os, threading, socket, Queue +import sys, os, threading, socket, Queue from signal import * from sys import exc_info from itertools import chain +sys.path.insert(0, '/var/portbuild/lib/python') + +from freebsd_config import * + from qmanagerobj import * CONFIG_DIR="/var/portbuild" Modified: projects/portbuild/qmanager/packagebuild ============================================================================== --- projects/portbuild/qmanager/packagebuild Tue Apr 26 20:14:29 2011 (r221079) +++ projects/portbuild/qmanager/packagebuild Tue Apr 26 20:25:16 2011 (r221080) @@ -32,11 +32,15 @@ # * check mtime for package staleness (cf make) # * option to skip phase 2 +import sys + +sys.path.insert(0, '/var/portbuild/lib/python') + from qmanagerclient import * from freebsd_config import * -import os, string, sys, threading, time, subprocess +import os, string, threading, time, subprocess #import random from itertools import chain #import gc Modified: projects/portbuild/qmanager/qmanager ============================================================================== --- projects/portbuild/qmanager/qmanager Tue Apr 26 20:14:29 2011 (r221079) +++ projects/portbuild/qmanager/qmanager Tue Apr 26 20:25:16 2011 (r221080) @@ -49,12 +49,15 @@ # - OR, NOT job description entries # - query jobs.machine properties +import sys + +sys.path.insert(0, '/var/portbuild/lib/python') + from freebsd_config import * import os, threading, socket, Queue from signal import * -from sys import exc_info from itertools import chain from qmanagerobj import * @@ -99,7 +102,7 @@ class Worker(object): cmddict[conn.cmd](conn) except OSError, error: if error.errno != 32: - print exc_info() + print sys.exc_info() self.workq.task_done() Modified: projects/portbuild/qmanager/qmanager.py ============================================================================== --- projects/portbuild/qmanager/qmanager.py Tue Apr 26 20:14:29 2011 (r221079) +++ projects/portbuild/qmanager/qmanager.py Tue Apr 26 20:25:16 2011 (r221080) @@ -49,9 +49,13 @@ # - OR, NOT job description entries # - query jobs.machine properties +import sys + +sys.path.insert(0, '/var/portbuild/lib/python') + from freebsd_config import * -import os, socket, sys, threading, time, Queue +import os, socket, threading, time, Queue from signal import * from itertools import chain @@ -109,7 +113,7 @@ class Worker(object): # ignore EPIPE # XXX MCL why? if error.errno != 32: - print exc_info() + print sys.exc_info() else: print "EPIPE" Modified: projects/portbuild/scripts/buildproxy ============================================================================== --- projects/portbuild/scripts/buildproxy Tue Apr 26 20:14:29 2011 (r221079) +++ projects/portbuild/scripts/buildproxy Tue Apr 26 20:25:16 2011 (r221080) @@ -6,6 +6,8 @@ import sys, socket, os, commands +sys.path.insert(0, '/var/portbuild/lib/python') + from freebsd import * from freebsd_config import * Modified: projects/portbuild/scripts/buildproxy-client ============================================================================== --- projects/portbuild/scripts/buildproxy-client Tue Apr 26 20:14:29 2011 (r221079) +++ projects/portbuild/scripts/buildproxy-client Tue Apr 26 20:25:16 2011 (r221080) @@ -5,6 +5,8 @@ import sys, socket, os, commands +sys.path.insert(0, '/var/portbuild/lib/python') + from freebsd import * from freebsd_config import * Modified: projects/portbuild/scripts/zbackup ============================================================================== --- projects/portbuild/scripts/zbackup Tue Apr 26 20:14:29 2011 (r221079) +++ projects/portbuild/scripts/zbackup Tue Apr 26 20:25:16 2011 (r221080) @@ -3,10 +3,13 @@ # Back up a list of ZFS filesystems, doing a full backup periodically # and using incremental diffs in between -import zfs, commands, datetime, sys, os, bz2 - +import commands, datetime, sys, os, bz2 from signal import * +sys.path.insert(0, '/var/portbuild/lib/python') + +import zfs + # List of filesystems to backup # XXX MCL backuplist=["a", "a/nfs", "a/local", "a/portbuild", Modified: projects/portbuild/scripts/zexpire ============================================================================== --- projects/portbuild/scripts/zexpire Tue Apr 26 20:14:29 2011 (r221079) +++ projects/portbuild/scripts/zexpire Tue Apr 26 20:25:16 2011 (r221080) @@ -2,7 +2,11 @@ # # Expire old snapshots -import zfs, commands, datetime, os +import sys, commands, datetime, os + +sys.path.insert(0, '/var/portbuild/lib/python') + +import zfs # List of filesystems to expire expirelist=(("a", 14),