Date: Tue, 11 Aug 2009 21:23:31 +0000 (UTC) From: Peter Wemm <peter@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-svnadmin@freebsd.org Subject: svn commit: r196101 - svnadmin/tools Message-ID: <200908112123.n7BLNVhj042230@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: peter Date: Tue Aug 11 21:23:31 2009 New Revision: 196101 URL: http://svn.freebsd.org/changeset/base/196101 Log: WIP: split exporter into two halves, persistent driver that tracks the repository, and a backend that exports a single rev. Modified: svnadmin/tools/export1.py svnadmin/tools/export2.py Modified: svnadmin/tools/export1.py ============================================================================== --- svnadmin/tools/export1.py Tue Aug 11 21:21:20 2009 (r196100) +++ svnadmin/tools/export1.py Tue Aug 11 21:23:31 2009 (r196101) @@ -10,103 +10,10 @@ import popen2 import tempfile from svn import core, fs, delta, repos -do_keywords = False - -# SVN's API structure is to do callbacks to a class to get notifications -class ChangeReceiver(delta.Editor): - def __init__(self, fs_root, base_root, rev, fs_ptr, pool, do_kw): - self.fs_root = fs_root - self.base_root = base_root - self.fs_ptr = fs_ptr - self.rev = int(rev) - self.pool = pool - self.do_kw = do_kw - self.changes = [] - - def delete_entry(self, path, revision, parent_baton, pool): - self.changes.append(['D', path]) - - def add_file(self, path, parent_baton, - copyfrom_path, copyfrom_revision, file_pool): - self.changes.append(['A', path]) - return [ '_', ' ', None ] - - def open_file(self, path, parent_baton, base_revision, file_pool): - return [ '_', ' ', path ] - - def apply_textdelta(self, file_baton, base_checksum): - text_mod, prop_mod, path = file_baton - file_baton[0] = 'U' - # no handler - return None - - def change_file_prop(self, file_baton, name, value, pool): - text_mod, prop_mod, path = file_baton - if self.do_kw and name == core.SVN_PROP_KEYWORDS: - file_baton[1] = 'U' - - def close_file(self, file_baton, text_checksum): - text_mod, prop_mod, path = file_baton - # test the path. it will be None if we added this file. - if path: - status = text_mod + prop_mod - # was there some kind of change? - if status != '_ ': - self.changes.append(['U', path]) - -# Last path component -def _basename(path): - idx = path.rfind('/') - if idx == -1: - return path - return path[idx+1:] - -# Directory component -def _dirname(path): - idx = path.rfind('/') - if idx == -1: - return '' - return path[:idx] - -# Keep track of best common prefix per directory group to keep commit time down -class pathcollector(): - def __init__(self): - self.paths = {} - - # Not really a set.. calculates common prefix instead. - def __setitem__(self, prefix, dir): - # If we haven't seen a dir, start here - if not self.paths.has_key(prefix): - self.paths[prefix] = dir - return - # If it is the same dir, we're finished - if self.paths[prefix] == dir: - return - # See if we've found a common parent - parent = dir - while _dirname(parent) != parent: - parent = _dirname(parent) - # See if common prefix works - if self.paths[prefix] == parent: - return - # Raise common prefix - if self.paths[prefix].startswith(parent): - self.paths[prefix] = parent - return - print "WTF?" - assert false - - def __iter__(self): - return self.paths.iteritems() - -# Issue a cvs command, aborting if a problem happens -def do_cvs(cvspath, dir, cmd): +# Issue an export command, aborting if a problem happens +def do_export(cmd): ioerror = False - print("cvs path %s, dir %s, cmd %s" % (cvspath, dir, cmd)) - cwd = os.getcwd() - os.chdir(os.path.join(cvspath, dir)) pipe = popen2.Popen3(cmd) - os.chdir(cwd) output = '' try: output = pipe.fromchild.readlines() @@ -116,256 +23,24 @@ def do_cvs(cvspath, dir, cmd): ioerror = True rv = pipe.wait() failed = (rv != 0) or ioerror - print 'Cvs output: ', output + print 'export2 output: ', output return failed # Dump a file from svn into cvs. This has to apply the delta to the previous rev. -def dump_file(fs_ptr, fs_root, rev, svnpath, cvspath, author, date, pool, workpath): - if do_keywords: - kw = fs.node_prop(fs_root, svnpath, core.SVN_PROP_KEYWORDS) - if not kw: - kw = '' - str = '$' + 'FreeBSD: %s %s %s %s $' % (cvspath, rev, date, author) - #str = '$' + 'FreeBSDId: %s %s %s %s $' % (cvspath, rev, date, author) - - subpool = core.svn_pool_create(pool) - stream = core.Stream(fs.file_contents(fs_root, svnpath, subpool)) - str_list = [] - while 1: - data = stream.read(core.SVN_STREAM_CHUNK_SIZE) - str_list.append(data) - if len(data) < core.SVN_STREAM_CHUNK_SIZE: - break - string = ''.join(str_list) - # Expand keywords - if do_keywords: - if kw == r'FreeBSD=%H': - old = '$' + 'FreeBSD$' - string = string.replace(old, str) - cvsfile = os.path.join(workpath, cvspath) -# sys.stdout.write('File contents:\n=========\n') -# sys.stdout.write(string) -# sys.stdout.write('=========\n') - executable = fs.node_prop(fs_root, svnpath, core.SVN_PROP_EXECUTABLE) - if executable: - mode = 0777 - else: - mode = 0666 - outfile = os.open(cvsfile, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, mode) - if not outfile: - sys.exit('cannot open %s for write' % cvsfile) - n = os.write(outfile, string) - if n != len(string): - sys.exit('short write. %d instead of %d' % (n, len(string))) - os.close(outfile) - core.svn_pool_destroy(subpool) - -# List of paths we export to cvs, and what branch tag (or None for HEAD) -# Hard coded for now. This will be automatically implied based on path transforms. -maptable = [ - # -current head - ( 'head/', None ), - # -stable branches - ( 'stable/2.0.5/', 'RELENG_2_0_5' ), - ( 'stable/2.1/', 'RELENG_2_1_0' ), - ( 'stable/2.2/', 'RELENG_2_2' ), - ( 'stable/3/', 'RELENG_3' ), - ( 'stable/4/', 'RELENG_4' ), - ( 'stable/5/', 'RELENG_5' ), - ( 'stable/6/', 'RELENG_6' ), - ( 'stable/7/', 'RELENG_7' ), - # errata / security / releng branches - ( 'releng/ALPHA_2_0/','ALPHA_2_0' ), - ( 'releng/BETA_2_0/', 'BETA_2_0' ), - ( 'releng/4.3/', 'RELENG_4_3' ), - ( 'releng/4.4/', 'RELENG_4_4' ), - ( 'releng/4.5/', 'RELENG_4_5' ), - ( 'releng/4.6/', 'RELENG_4_6' ), - ( 'releng/4.7/', 'RELENG_4_7' ), - ( 'releng/4.8/', 'RELENG_4_8' ), - ( 'releng/4.9/', 'RELENG_4_9' ), - ( 'releng/4.10/', 'RELENG_4_10' ), - ( 'releng/4.11/', 'RELENG_4_11' ), - ( 'releng/4.12/', 'RELENG_4_12' ), - ( 'releng/5.0/', 'RELENG_5_0' ), - ( 'releng/5.1/', 'RELENG_5_1' ), - ( 'releng/5.2/', 'RELENG_5_2' ), - ( 'releng/5.3/', 'RELENG_5_3' ), - ( 'releng/5.4/', 'RELENG_5_4' ), - ( 'releng/5.5/', 'RELENG_5_5' ), - ( 'releng/5.6/', 'RELENG_5_6' ), - ( 'releng/6.0/', 'RELENG_6_0' ), - ( 'releng/6.1/', 'RELENG_6_1' ), - ( 'releng/6.2/', 'RELENG_6_2' ), - ( 'releng/6.3/', 'RELENG_6_3' ), - ( 'releng/6.4/', 'RELENG_6_4' ), - ( 'releng/7.0/', 'RELENG_7_0' ), - ( 'releng/7.1/', 'RELENG_7_1' ), - ( 'releng/7.2/', 'RELENG_7_2' ), -] - -def map2cvs(svnpath): - for prefix, branch in maptable: - plen = len(prefix) - if svnpath.startswith(prefix): - return 'src/' + svnpath[plen:], branch - return None, None - -# List of special remap cases. -roottable = [ - ( 'svnadmin/conf/access', 'CVSROOT/access', 'CVSROOT' ), - ( 'svnadmin/conf/mentors', 'CVSROOT/mentors', 'CVSROOT' ), -] - -def maproot(p): - for svnpath, path, dir in roottable: - if p == svnpath: - return path, dir - return None, None - -# Add intermediate directories to the cvs checkout area as needed. -# XXX should use 'cvs update -d -l' if the dir exists in cvsroot -def makedirs(cvspath, path, base): - #print 'Makedirs:', cvspath, path - if not path.startswith(base): - sys.exit('Illegal path %s' % path) - if path == base: - return - makedirs(cvspath, _dirname(path), base) - fullpath = os.path.join(cvspath, path) - if os.path.isfile(fullpath): - sys.exit('Dest dir is a file' % path) - if not os.path.isdir(fullpath): - try: - #print "Making directory " + fullpath - os.makedirs(fullpath) - failed = do_cvs(cvspath, _dirname(path), "cvs -q add %s" % _basename(path)) - assert not failed - except OSError: - sys.exit('Cannot mkdir %s' % path) - #print 'Dirpath complete: ' + path - -# Export a single change to cvs. -def exportrev(pool, fs_ptr, rev, cvspath): - def authz_cb(root, path, pool): - return True - - subpool = core.svn_pool_create(pool) - # Connect up to the revision - fs_root = fs.revision_root(fs_ptr, rev, subpool) - base_root = fs.revision_root(fs_ptr, rev - 1, subpool) - editor = ChangeReceiver(fs_root, base_root, rev, fs_ptr, subpool, do_keywords) - e_ptr, e_baton = delta.make_editor(editor, subpool) - repos.dir_delta(base_root, '', '', fs_root, '', e_ptr, e_baton, authz_cb, 0, 1, 0, 0, subpool) - - # Author - author = fs.revision_prop(fs_ptr, rev, core.SVN_PROP_REVISION_AUTHOR) - if not author: - author = 'NoAuthor' - if author == 'davidg': - author = 'dg' - os.environ['CVS_AUTHOR'] = author - print 'Author: ' + author - - # Date - date = fs.revision_prop(fs_ptr, rev, core.SVN_PROP_REVISION_DATE) - if date: - aprtime = core.svn_time_from_cstring(date) - secs = aprtime / 1000000 # aprtime is microseconds; make seconds - tm = time.gmtime(secs) - date = time.strftime('%Y-%m-%d %H:%M:%SZ', tm) - os.environ['CVS_TIMESTAMP'] = "%d" % secs - else: - date = 'NoDate' - if os.environ.has_key('CVS_TIMESTAMP'): - del os.environ['CVS_TIMESTAMP'] - - # Build log message to export - cvslog = 'SVN rev %d on %s by %s\n' % (rev, date, author) - svnlog = fs.revision_prop(fs_ptr, rev, core.SVN_PROP_REVISION_LOG) - if svnlog: - cvslog += '\n' + svnlog - - pc = pathcollector() - - for k, p in editor.changes: - #print 'Path ', p - # Hack, hack - (path, dir) = maproot(p) - if path: - workpath = cvspath - dump_file(fs_ptr, fs_root, rev, p, path, author, date, subpool, workpath) - pc[workpath] = dir - continue - (path, tag) = map2cvs(p) - if not path: - continue - if tag: - workpath = os.path.join(cvspath, tag) - uptag = '-r ' + tag - else: - workpath = cvspath - uptag = '-A' - #print workpath - if not os.path.isdir(workpath): - os.makedirs(workpath) - if not os.path.isdir(os.path.join(workpath, 'src')): - failed = do_cvs(workpath, '', "cvs -Rq co %s src" % uptag) - assert not failed - # at this point, the top directory and /src should exist - print p, path, k - # hacks - #if p == 'head/contrib/file/FREEBSD-upgrade' and k == 'A': - # continue - #if p == 'head/contrib/file/magic2mime' and k == 'A': - # continue - #if p == 'head/lib/libc/stdio/asprintf.c' and k == 'D': - # continue - if p == 'head/tools/build/options/WITH_BIND_LIBS' and k == 'A' and rev == 193280: - continue - makedirs(workpath, _dirname(path), 'src') - # Now the directory for the files must exist, and branch tag will be sticky - assert os.path.isdir(os.path.join(workpath, _dirname(path))) - assert k == 'A' or k == 'U' or k == 'D' - - if k == 'A' or k == 'U': - print 'add/update file ' + path + '.' - destpath = os.path.join(workpath, path) - existed = os.path.isfile(destpath) - dump_file(fs_ptr, fs_root, rev, p, path, author, date, subpool, workpath) - if not existed: - print 'cvs add file ' + path + '.' - failed = do_cvs(workpath, _dirname(path), "cvs -q add %s" % _basename(path)) - assert not failed - elif k == 'D': - print 'cvs rm -f file ' + path + '.' - failed = do_cvs(workpath, _dirname(path), "cvs -q rm -f %s" % _basename(path)) - assert not failed - pc[workpath] = _dirname(path) - - # aggregate the commit - for root, dir in pc: - fd, logfile = tempfile.mkstemp() - os.write(fd, cvslog) - os.close(fd) - failed = do_cvs(root, dir, "cvs -q commit -F %s" % logfile) - assert not failed - os.remove(logfile) - core.svn_pool_destroy(subpool) - # Loop for the export range -def export(pool, repos_path, cvspath): +def export(pool, repos_path, cvspath, cvsroot): repos_path = core.svn_path_canonicalize(repos_path) fs_ptr = repos.fs(repos.open(repos_path, pool)) while True: curr_rev = fs.youngest_rev(fs_ptr) last_rev = int(fs.revision_prop(fs_ptr, 0, 'fbsd:lastexp')) if last_rev < curr_rev: - time.sleep(5) print '%d %s' % (last_rev, curr_rev) rev = '%d' % (last_rev + 1) print '==========> export rev ' + rev - exportrev(pool, fs_ptr, last_rev + 1, cvspath) + cmd = './export2.py %s %s %d %s' % (repos_path, cvspath, last_rev + 1, cvsroot) + failed = do_export(cmd) + assert not failed fs.change_rev_prop(fs_ptr, 0, 'fbsd:lastexp', rev) continue print "." @@ -374,8 +49,4 @@ def export(pool, repos_path, cvspath): if __name__ == '__main__': print "Version: $FreeBSD$" - os.environ['CVSROOT'] = '/r/ncvs' - core.run_app(export, '/r/svnmirror/base', '/r/svn2cvs/cvs') - # test rig - #os.environ['CVSROOT'] = '/home/peter/exp/cvs' - #core.run_app(export, '/home/peter/exp/svn', '/home/peter/exp/co') + core.run_app(export, '/r/svnmirror/base', '/r/svn2cvs/cvs', '/r/ncvs') Modified: svnadmin/tools/export2.py ============================================================================== --- svnadmin/tools/export2.py Tue Aug 11 21:21:20 2009 (r196100) +++ svnadmin/tools/export2.py Tue Aug 11 21:23:31 2009 (r196101) @@ -354,28 +354,14 @@ def exportrev(pool, fs_ptr, rev, cvspath core.svn_pool_destroy(subpool) # Loop for the export range -def export(pool, repos_path, cvspath): +def export(pool, repos_path, cvspath, rev): repos_path = core.svn_path_canonicalize(repos_path) fs_ptr = repos.fs(repos.open(repos_path, pool)) - while True: - curr_rev = fs.youngest_rev(fs_ptr) - last_rev = int(fs.revision_prop(fs_ptr, 0, 'fbsd:lastexp')) - if last_rev < curr_rev: - time.sleep(5) - print '%d %s' % (last_rev, curr_rev) - rev = '%d' % (last_rev + 1) - print '==========> export rev ' + rev - exportrev(pool, fs_ptr, last_rev + 1, cvspath) - fs.change_rev_prop(fs_ptr, 0, 'fbsd:lastexp', rev) - continue - print "." - time.sleep(15) - + print '==========> export rev ' + rev + exportrev(pool, fs_ptr, rev, cvspath) if __name__ == '__main__': print "Version: $FreeBSD$" - os.environ['CVSROOT'] = '/r/ncvs' - core.run_app(export, '/r/svnmirror/base', '/r/svn2cvs/cvs') - # test rig - #os.environ['CVSROOT'] = '/home/peter/exp/cvs' - #core.run_app(export, '/home/peter/exp/svn', '/home/peter/exp/co') + assert len(sys.argv) == 5 + os.environ['CVSROOT'] = sys.argv[4] + core.run_app(export, sys.argv[1], sys.argv[2], int(sys.argv[3]))
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200908112123.n7BLNVhj042230>