Date: Thu, 12 Jan 2017 11:53:33 +0000 (UTC) From: Ulrich Spoerlein <uqs@FreeBSD.org> To: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: svn commit: r311976 - in user/uqs/git_conv/svn2git: . samples src Message-ID: <201701121153.v0CBrXgs037865@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: uqs Date: Thu Jan 12 11:53:33 2017 New Revision: 311976 URL: https://svnweb.freebsd.org/changeset/base/311976 Log: Update our copy of svn2git with the latest upstream plus our local hack Added: user/uqs/git_conv/svn2git/.gitignore user/uqs/git_conv/svn2git/Makefile (contents, props changed) user/uqs/git_conv/svn2git/README.md user/uqs/git_conv/svn2git/samples/ user/uqs/git_conv/svn2git/samples/ignore-branch.rules user/uqs/git_conv/svn2git/samples/merged-branches-tags.rules user/uqs/git_conv/svn2git/samples/min-max-revision.rules user/uqs/git_conv/svn2git/samples/recurse.rules user/uqs/git_conv/svn2git/samples/standardlayout.rules user/uqs/git_conv/svn2git/samples/two-projects.rules Modified: user/uqs/git_conv/svn2git/src/Makefile user/uqs/git_conv/svn2git/src/main.cpp user/uqs/git_conv/svn2git/src/repository.cpp user/uqs/git_conv/svn2git/src/repository.h user/uqs/git_conv/svn2git/src/ruleparser.cpp user/uqs/git_conv/svn2git/src/ruleparser.h user/uqs/git_conv/svn2git/src/svn.cpp Added: user/uqs/git_conv/svn2git/.gitignore ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/uqs/git_conv/svn2git/.gitignore Thu Jan 12 11:53:33 2017 (r311976) @@ -0,0 +1,5 @@ +Makefile +*~ +*.o +svn-all-fast-export +src/local-config.pri Added: user/uqs/git_conv/svn2git/Makefile ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/uqs/git_conv/svn2git/Makefile Thu Jan 12 11:53:33 2017 (r311976) @@ -0,0 +1,128 @@ +############################################################################# +# Makefile for building: fast-export2 +# Generated by qmake (2.01a) (Qt 4.8.7) on: Thu Jan 12 12:51:50 2017 +# Project: fast-export2.pro +# Template: subdirs +# Command: /usr/local/bin/qmake-qt4 -o Makefile fast-export2.pro +############################################################################# + +first: make_default +MAKEFILE = Makefile +QMAKE = /usr/local/bin/qmake-qt4 +DEL_FILE = rm -f +CHK_DIR_EXISTS= test -d +MKDIR = mkdir -p +COPY = cp -f +COPY_FILE = cp -f +COPY_DIR = cp -f -R +INSTALL_FILE = install -m 644 -p +INSTALL_PROGRAM = install -m 755 -p +INSTALL_DIR = $(COPY_DIR) +DEL_FILE = rm -f +SYMLINK = ln -f -s +DEL_DIR = rmdir +MOVE = mv -f +CHK_DIR_EXISTS= test -d +MKDIR = mkdir -p +SUBTARGETS = \ + sub-src + +src/$(MAKEFILE): + @$(CHK_DIR_EXISTS) src/ || $(MKDIR) src/ + cd src/ && $(QMAKE) /data/src/git_conv/svn2git/src/src.pro -o $(MAKEFILE) +sub-src-qmake_all: FORCE + @$(CHK_DIR_EXISTS) src/ || $(MKDIR) src/ + cd src/ && $(QMAKE) /data/src/git_conv/svn2git/src/src.pro -o $(MAKEFILE) +sub-src: src/$(MAKEFILE) FORCE + cd src/ && $(MAKE) -f $(MAKEFILE) +sub-src-make_default: src/$(MAKEFILE) FORCE + cd src/ && $(MAKE) -f $(MAKEFILE) +sub-src-make_first: src/$(MAKEFILE) FORCE + cd src/ && $(MAKE) -f $(MAKEFILE) first +sub-src-all: src/$(MAKEFILE) FORCE + cd src/ && $(MAKE) -f $(MAKEFILE) all +sub-src-clean: src/$(MAKEFILE) FORCE + cd src/ && $(MAKE) -f $(MAKEFILE) clean +sub-src-distclean: src/$(MAKEFILE) FORCE + cd src/ && $(MAKE) -f $(MAKEFILE) distclean +sub-src-install_subtargets: src/$(MAKEFILE) FORCE + cd src/ && $(MAKE) -f $(MAKEFILE) install +sub-src-uninstall_subtargets: src/$(MAKEFILE) FORCE + cd src/ && $(MAKE) -f $(MAKEFILE) uninstall + +Makefile: fast-export2.pro /usr/local/share/qt4/mkspecs/freebsd-clang/qmake.conf /usr/local/share/qt4/mkspecs/common/unix.conf \ + /usr/local/share/qt4/mkspecs/common/freebsd.conf \ + /usr/local/share/qt4/mkspecs/common/gcc-base.conf \ + /usr/local/share/qt4/mkspecs/common/gcc-base-unix.conf \ + /usr/local/share/qt4/mkspecs/common/clang.conf \ + /usr/local/share/qt4/mkspecs/common/clang-unix.conf \ + /usr/local/share/qt4/mkspecs/qconfig.pri \ + /usr/local/share/qt4/mkspecs/features/qt_functions.prf \ + /usr/local/share/qt4/mkspecs/features/qt_config.prf \ + /usr/local/share/qt4/mkspecs/features/exclusive_builds.prf \ + /usr/local/share/qt4/mkspecs/features/default_pre.prf \ + /usr/local/share/qt4/mkspecs/features/release.prf \ + /usr/local/share/qt4/mkspecs/features/default_post.prf \ + /usr/local/share/qt4/mkspecs/features/shared.prf \ + /usr/local/share/qt4/mkspecs/features/unix/gdb_dwarf_index.prf \ + /usr/local/share/qt4/mkspecs/features/warn_on.prf \ + /usr/local/share/qt4/mkspecs/features/qt.prf \ + /usr/local/share/qt4/mkspecs/features/unix/thread.prf \ + /usr/local/share/qt4/mkspecs/features/moc.prf \ + /usr/local/share/qt4/mkspecs/features/resources.prf \ + /usr/local/share/qt4/mkspecs/features/uic.prf \ + /usr/local/share/qt4/mkspecs/features/yacc.prf \ + /usr/local/share/qt4/mkspecs/features/lex.prf \ + /usr/local/share/qt4/mkspecs/features/include_source_dir.prf + $(QMAKE) -o Makefile fast-export2.pro +/usr/local/share/qt4/mkspecs/common/unix.conf: +/usr/local/share/qt4/mkspecs/common/freebsd.conf: +/usr/local/share/qt4/mkspecs/common/gcc-base.conf: +/usr/local/share/qt4/mkspecs/common/gcc-base-unix.conf: +/usr/local/share/qt4/mkspecs/common/clang.conf: +/usr/local/share/qt4/mkspecs/common/clang-unix.conf: +/usr/local/share/qt4/mkspecs/qconfig.pri: +/usr/local/share/qt4/mkspecs/features/qt_functions.prf: +/usr/local/share/qt4/mkspecs/features/qt_config.prf: +/usr/local/share/qt4/mkspecs/features/exclusive_builds.prf: +/usr/local/share/qt4/mkspecs/features/default_pre.prf: +/usr/local/share/qt4/mkspecs/features/release.prf: +/usr/local/share/qt4/mkspecs/features/default_post.prf: +/usr/local/share/qt4/mkspecs/features/shared.prf: +/usr/local/share/qt4/mkspecs/features/unix/gdb_dwarf_index.prf: +/usr/local/share/qt4/mkspecs/features/warn_on.prf: +/usr/local/share/qt4/mkspecs/features/qt.prf: +/usr/local/share/qt4/mkspecs/features/unix/thread.prf: +/usr/local/share/qt4/mkspecs/features/moc.prf: +/usr/local/share/qt4/mkspecs/features/resources.prf: +/usr/local/share/qt4/mkspecs/features/uic.prf: +/usr/local/share/qt4/mkspecs/features/yacc.prf: +/usr/local/share/qt4/mkspecs/features/lex.prf: +/usr/local/share/qt4/mkspecs/features/include_source_dir.prf: +qmake: qmake_all FORCE + @$(QMAKE) -o Makefile fast-export2.pro + +qmake_all: sub-src-qmake_all FORCE + +make_default: sub-src-make_default FORCE +make_first: sub-src-make_first FORCE +all: sub-src-all FORCE +clean: sub-src-clean FORCE +distclean: sub-src-distclean FORCE + -$(DEL_FILE) Makefile +install_subtargets: sub-src-install_subtargets FORCE +uninstall_subtargets: sub-src-uninstall_subtargets FORCE + +sub-src-check: src/$(MAKEFILE) + cd src/ && $(MAKE) -f $(MAKEFILE) check +check: sub-src-check + +mocclean: compiler_moc_header_clean compiler_moc_source_clean + +mocables: compiler_moc_header_make_all compiler_moc_source_make_all +install: install_subtargets FORCE + +uninstall: uninstall_subtargets FORCE + +FORCE: + Added: user/uqs/git_conv/svn2git/README.md ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/uqs/git_conv/svn2git/README.md Thu Jan 12 11:53:33 2017 (r311976) @@ -0,0 +1,40 @@ +svn-all-fast-export aka svn2git +=============================== +This project contains all the tools required to do a conversion of an svn repository (server side, not a checkout) to one or more git repositories. + +This is the tool used to convert KDE's Subversion into multiple Git repositories. You can find more description and usage examples at https://techbase.kde.org/Projects/MoveToGit/UsingSvn2Git + + +How does it work +---------------- +The svn2git repository gets you an application that will do the actual conversion. +The conversion exists of looping over each and every commit in the subversion repository and matching the changes to a ruleset after which the changes are applied to a certain path in a git repo. +The ruleset can specify which git repository to use and thus you can have more than one git repository as a result of running the conversion. +Also noteworthy is that you can have a rule that, for example, changes in svnrepo/branches/foo/2.1/ will appear as a git-branch in a repository. + +If you have a proper ruleset the tool will create the git repositories for you and show progress while converting commit by commit. + +After it is done you likely want to run `git repack -a -d -f` to compress the pack file as it can get quite big. + +Building the tool +----------------- +Run `qmake && make`. You get `./svn-all-fast-export`. +(Do a checkout of the repo .git' and run qmake and make. You can only build it after having installed libsvn-dev, and naturally Qt. Running the command will give you all the options you can pass to the tool.) + +KDE +--- +there is a repository kde-ruleset which has several example files and one file that should become the final ruleset for the whole of KDE called 'kde-rules-main'. + +Write the Rules +--------------- +You need to write a rules file that describes how to slice the Subversion history into Git repositories and branches. See https://techbase.kde.org/Projects/MoveToGit/UsingSvn2Git. +The rules are also documented in the 'samples' directory of the svn2git repository. Feel free to add more documentation here as well. + +Work flow +--------- +Please feel free to fill this section in. + +Some SVN tricks +--------------- +You can access your newly rsynced SVN repo with commands like `svn ls file:///path/to/repo/trunk/KDE`. +A common issue is tracking when an item left playground for kdereview and then went from kdereview to its final destination. There is no straightforward way to do this. So the following command comes in handy: `svn log -v file:///path/to/repo/kde-svn/kde/trunk/kdereview | grep /trunk/kdereview/mplayerthumbs -A 5 -B 5` This will print all commits relevant to the package you are trying to track. You can also pipe the above command to head or tail to see the the first and last commit it was in that directory. Added: user/uqs/git_conv/svn2git/samples/ignore-branch.rules ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/uqs/git_conv/svn2git/samples/ignore-branch.rules Thu Jan 12 11:53:33 2017 (r311976) @@ -0,0 +1,31 @@ +# +# Declare the repositories we know about: +# + +create repository myproject +end repository + +# +# Declare the rules +# Note: rules must end in a slash +# + +match /trunk/ + repository myproject + branch master +end match + +# Ignore this branch: +# We ignore a branch by not telling svn-all-fast-export what to do +# with this path +# Note that rules are applied in order of appearance, so this rule +# must appear before the generic branch rule +match /branches/abandoned-work/ +end match + +match /branches/([^/]+)/ + repository myproject + branch \1 +end match + +# No tag processing Added: user/uqs/git_conv/svn2git/samples/merged-branches-tags.rules ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/uqs/git_conv/svn2git/samples/merged-branches-tags.rules Thu Jan 12 11:53:33 2017 (r311976) @@ -0,0 +1,31 @@ +# +# Declare the repositories we know about: +# + +create repository myproject +end repository + +# +# Declare the rules +# Note: rules must end in a slash +# + +match /trunk/ + repository myproject + branch master +end match + +# Subversion doesn't understand the Git concept of tags +# In Subversion, tags are really branches +# +# Only a post-processing (i.e., after converting to Git) of the tag +# branches can we be sure that a tag wasn't moved or changed from the +# branch it was copied from +# +# So we don't pretend that SVN tags are Git tags and then import +# everything as one + +match /(branches|tags)/([^/]+)/ + repository myproject + branch \2 +end match Added: user/uqs/git_conv/svn2git/samples/min-max-revision.rules ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/uqs/git_conv/svn2git/samples/min-max-revision.rules Thu Jan 12 11:53:33 2017 (r311976) @@ -0,0 +1,39 @@ +# +# Declare the repositories we know about: +# + +create repository myproject +end repository + +# +# Declare the rules +# Note: rules must end in a slash +# + +# Ignore this particular revision in trunk +# See ignore-branch.rules first +# Note that rules are applied in order of appearance, so this rule +# must appear before the generic trunk rule +match /trunk/ + min revision 123 + max revision 123 +end match + +# Stop importing trunk +# If we don't specify a max revision, then there is no limit +# The same applies to min revision (i.e., min revision is 0) +match /trunk/ + min revision 1234 +end match + +match /trunk/ + repository myproject + branch master +end match + +match /branches/([^/]+)/ + repository myproject + branch \1 +end match + +# No tag processing Added: user/uqs/git_conv/svn2git/samples/recurse.rules ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/uqs/git_conv/svn2git/samples/recurse.rules Thu Jan 12 11:53:33 2017 (r311976) @@ -0,0 +1,41 @@ +# +# Declare the repositories we know about: +# + +create repository project1 +end repository + +create repository project2 +end repository + +# +# Declare the rules +# Note: rules must end in a slash +# + +match /trunk/([^/]+)/ + repository \1 + branch master +end match + +# +# SVN layout: +# /branches/branchname/project1 +# /branches/branchname/project2 +match /branches/([^/]+)/([^/]+)/ + repository \2 + branch \1 +end match + + +# +# Example of the recurse rule: +# We tell svn-all-fast-export to not import anything +# but to go inside and recurse in the subdirs +# Note how the ending slash is missing in this particular case + +match /branches/[^/]+ + action recurse +end match + +# No tag processing Added: user/uqs/git_conv/svn2git/samples/standardlayout.rules ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/uqs/git_conv/svn2git/samples/standardlayout.rules Thu Jan 12 11:53:33 2017 (r311976) @@ -0,0 +1,37 @@ +# +# Declare the repositories we know about: +# + +create repository myproject +end repository + +# +# Declare the rules +# Note: rules must end in a slash +# + +match /trunk/ + repository myproject + branch master +end match + +match /branches/([^/]+)/ + repository myproject + branch \1 +end match + +# Important: +# Subversion doesn't understand the Git concept of tags +# In Subversion, tags are really branches +# +# Only a post-processing (i.e., after converting to Git) of the tag +# branches can we be sure that a tag wasn't moved or changed from the +# branch it was copied from +# +# This rule will create tags that don't exist in any of the +# branches. It's not what you want. +# See the merged-branches-tags.rules file +match /tags/([^/]+)/ + repository myproject + branch refs/tags/\1 +end match Added: user/uqs/git_conv/svn2git/samples/two-projects.rules ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/uqs/git_conv/svn2git/samples/two-projects.rules Thu Jan 12 11:53:33 2017 (r311976) @@ -0,0 +1,32 @@ +# +# Declare the repositories we know about: +# + +create repository project1 +end repository + +create repository project2 +end repository + +# +# Declare the rules +# Note: rules must end in a slash +# + +match /project1/trunk/ + repository project1 + branch master +end match + +match /project2/trunk/ + repository project2 + branch master +end match + +# Note how we can use regexp to capture the repository name +match /([^/]+)/branches/([^/]+)/ + repository \1 + branch \2 +end match + +# No tag processing Modified: user/uqs/git_conv/svn2git/src/Makefile ============================================================================== --- user/uqs/git_conv/svn2git/src/Makefile Thu Jan 12 11:35:50 2017 (r311975) +++ user/uqs/git_conv/svn2git/src/Makefile Thu Jan 12 11:53:33 2017 (r311976) @@ -1,6 +1,6 @@ ############################################################################# # Makefile for building: ../svn-all-fast-export -# Generated by qmake (2.01a) (Qt 4.8.7) on: Tue Nov 10 14:23:11 2015 +# Generated by qmake (2.01a) (Qt 4.8.7) on: Tue Jan 10 11:40:19 2017 # Project: src.pro # Template: app # Command: /usr/local/bin/qmake-qt4 -o Makefile src.pro @@ -10,7 +10,7 @@ CC = clang CXX = clang++ -DEFINES = -DVER="\"fa63382f9549c6ae1df7b8880d9cced531ab4e98\"" -DQT_NO_DEBUG -DQT_CORE_LIB -DQT_SHARED +DEFINES = -DVER="\"437beb493db3b1cb532ca5807264cd4403a44157\"" -DQT_NO_DEBUG -DQT_CORE_LIB -DQT_SHARED CFLAGS = -pipe -O2 -Wall -W -pthread -D_THREAD_SAFE $(DEFINES) CXXFLAGS = -pipe -O2 -Wall -W -pthread -D_THREAD_SAFE $(DEFINES) INCPATH = -I/usr/local/share/qt4/mkspecs/freebsd-clang -I. -I/usr/local/include/qt4/QtCore -I/usr/local/include/qt4 -I. -I/usr/include/subversion-1 -I/usr/local/include/subversion-1 -I/usr/include/apr-1.0 -I/usr/include/apr-1 -I/usr/local/include/apr-1 -I. -I/usr/local/include/qt4 -I/usr/local/include @@ -165,8 +165,8 @@ qmake: FORCE @$(QMAKE) -o Makefile src.pro dist: - @$(CHK_DIR_EXISTS) .tmp/svn-all-fast-exportfa63382f9549c6ae1df7b8880d9cced531ab4e98 || $(MKDIR) .tmp/svn-all-fast-exportfa63382f9549c6ae1df7b8880d9cced531ab4e98 - $(COPY_FILE) --parents $(SOURCES) $(DIST) .tmp/svn-all-fast-exportfa63382f9549c6ae1df7b8880d9cced531ab4e98/ && $(COPY_FILE) --parents ruleparser.h repository.h svn.h CommandLineParser.h .tmp/svn-all-fast-exportfa63382f9549c6ae1df7b8880d9cced531ab4e98/ && $(COPY_FILE) --parents ruleparser.cpp repository.cpp svn.cpp main.cpp CommandLineParser.cpp .tmp/svn-all-fast-exportfa63382f9549c6ae1df7b8880d9cced531ab4e98/ && (cd `dirname .tmp/svn-all-fast-exportfa63382f9549c6ae1df7b8880d9cced531ab4e98` && $(TAR) svn-all-fast-exportfa63382f9549c6ae1df7b8880d9cced531ab4e98.tar svn-all-fast-exportfa63382f9549c6ae1df7b8880d9cced531ab4e98 && $(COMPRESS) svn-all-fast-exportfa63382f9549c6ae1df7b8880d9cced531ab4e98.tar) && $(MOVE) `dirname .tmp/svn-all-fast-exportfa63382f9549c6ae1df7b8880d9cced531ab4e98`/svn-all-fast-exportfa63382f9549c6ae1df7b8880d9cced531ab4e98.tar.gz . && $(DEL_FILE) -r .tmp/svn-all-fast-exportfa63382f9549c6ae1df7b8880d9cced531ab4e98 + @$(CHK_DIR_EXISTS) .tmp/svn-all-fast-export437beb493db3b1cb532ca5807264cd4403a44157 || $(MKDIR) .tmp/svn-all-fast-export437beb493db3b1cb532ca5807264cd4403a44157 + $(COPY_FILE) --parents $(SOURCES) $(DIST) .tmp/svn-all-fast-export437beb493db3b1cb532ca5807264cd4403a44157/ && $(COPY_FILE) --parents ruleparser.h repository.h svn.h CommandLineParser.h .tmp/svn-all-fast-export437beb493db3b1cb532ca5807264cd4403a44157/ && $(COPY_FILE) --parents ruleparser.cpp repository.cpp svn.cpp main.cpp CommandLineParser.cpp .tmp/svn-all-fast-export437beb493db3b1cb532ca5807264cd4403a44157/ && (cd `dirname .tmp/svn-all-fast-export437beb493db3b1cb532ca5807264cd4403a44157` && $(TAR) svn-all-fast-export437beb493db3b1cb532ca5807264cd4403a44157.tar svn-all-fast-export437beb493db3b1cb532ca5807264cd4403a44157 && $(COMPRESS) svn-all-fast-export437beb493db3b1cb532ca5807264cd4403a44157.tar) && $(MOVE) `dirname .tmp/svn-all-fast-export437beb493db3b1cb532ca5807264cd4403a44157`/svn-all-fast-export437beb493db3b1cb532ca5807264cd4403a44157.tar.gz . && $(DEL_FILE) -r .tmp/svn-all-fast-export437beb493db3b1cb532ca5807264cd4403a44157 clean:compiler_clean Modified: user/uqs/git_conv/svn2git/src/main.cpp ============================================================================== --- user/uqs/git_conv/svn2git/src/main.cpp Thu Jan 12 11:35:50 2017 (r311975) +++ user/uqs/git_conv/svn2git/src/main.cpp Thu Jan 12 11:53:33 2017 (r311976) @@ -130,15 +130,20 @@ static const CommandLineOption options[] {"--identity-domain DOMAIN", "provide user domain if no map was given"}, {"--revisions-file FILENAME", "provide a file with revision number that should be processed"}, {"--rules FILENAME[,FILENAME]", "the rules file(s) that determines what goes where"}, + {"--msg-filter FILENAME", "External program / script to modify svn log message"}, {"--add-metadata", "if passed, each git commit will have svn commit info"}, {"--add-metadata-notes", "if passed, each git commit will have notes with svn commit info"}, {"--resume-from revision", "start importing at svn revision number"}, {"--max-rev revision", "stop importing at svn revision number"}, {"--dry-run", "don't actually write anything"}, + {"--create-dump", "don't create the repository but a dump file suitable for piping into fast-import"}, {"--debug-rules", "print what rule is being used for each file"}, {"--commit-interval NUMBER", "if passed the cache will be flushed to git every NUMBER of commits"}, {"--stats", "after a run print some statistics about the rules"}, {"--svn-branches", "Use the contents of SVN when creating branches, Note: SVN tags are branches as well"}, + {"--empty-dirs", "Add .gitignore-file for empty dirs"}, + {"--svn-ignore", "Import svn-ignore-properties via .gitignore"}, + {"--propcheck", "Check for svn-properties except svn-ignore"}, {"-h, --help", "show help"}, {"-v, --version", "show version"}, CommandLineLastOption @@ -199,7 +204,7 @@ int main(int argc, char **argv) retry: int min_rev = 1; foreach (Rules::Repository rule, rulesList.allRepositories()) { - Repository *repo = new Repository(rule); + Repository *repo = createRepository(rule, repositories); if (!repo) return EXIT_FAILURE; repositories.insert(rule.name, repo); Modified: user/uqs/git_conv/svn2git/src/repository.cpp ============================================================================== --- user/uqs/git_conv/svn2git/src/repository.cpp Thu Jan 12 11:35:50 2017 (r311975) +++ user/uqs/git_conv/svn2git/src/repository.cpp Thu Jan 12 11:53:33 2017 (r311976) @@ -26,12 +26,204 @@ static const int maxSimultaneousProcesses = 100; -static const int maxMark = (1 << 20) - 2; // some versions of git-fast-import are buggy for larger values of maxMark +typedef unsigned long long mark_t; +static const mark_t maxMark = ULONG_MAX; -class ProcessCache: QLinkedList<Repository *> +class FastImportRepository : public Repository { public: - void touch(Repository *repo) + class Transaction : public Repository::Transaction + { + Q_DISABLE_COPY(Transaction) + friend class FastImportRepository; + + FastImportRepository *repository; + QByteArray branch; + QByteArray svnprefix; + QByteArray author; + QByteArray log; + uint datetime; + int revnum; + + QVector<int> merges; + + QStringList deletedFiles; + QByteArray modifiedFiles; + + inline Transaction() {} + public: + ~Transaction(); + void commit(); + + void setAuthor(const QByteArray &author); + void setDateTime(uint dt); + void setLog(const QByteArray &log); + + void noteCopyFromBranch (const QString &prevbranch, int revFrom); + + void deleteFile(const QString &path); + QIODevice *addFile(const QString &path, int mode, qint64 length); + + void commitNote(const QByteArray ¬eText, bool append, + const QByteArray &commit = QByteArray()); + }; + FastImportRepository(const Rules::Repository &rule); + int setupIncremental(int &cutoff); + void restoreLog(); + ~FastImportRepository(); + + void reloadBranches(); + int createBranch(const QString &branch, int revnum, + const QString &branchFrom, int revFrom); + int deleteBranch(const QString &branch, int revnum); + Repository::Transaction *newTransaction(const QString &branch, const QString &svnprefix, int revnum); + + void createAnnotatedTag(const QString &name, const QString &svnprefix, int revnum, + const QByteArray &author, uint dt, + const QByteArray &log); + void finalizeTags(); + void commit(); + + bool branchExists(const QString& branch) const; + const QByteArray branchNote(const QString& branch) const; + void setBranchNote(const QString& branch, const QByteArray& noteText); + + bool hasPrefix() const; + + QString getName() const; + Repository *getEffectiveRepository(); +private: + struct Branch + { + int created; + QVector<int> commits; + QVector<int> marks; + QByteArray note; + }; + struct AnnotatedTag + { + QString supportingRef; + QByteArray svnprefix; + QByteArray author; + QByteArray log; + uint dt; + int revnum; + }; + + QHash<QString, Branch> branches; + QHash<QString, AnnotatedTag> annotatedTags; + QString name; + QString prefix; + LoggingQProcess fastImport; + int commitCount; + int outstandingTransactions; + QByteArray deletedBranches; + QByteArray resetBranches; + + /* Optional filter to fix up log messages */ + QProcess filterMsg; + QByteArray msgFilter(QByteArray); + + /* starts at 0, and counts up. */ + mark_t last_commit_mark; + + /* starts at maxMark and counts down. Reset after each SVN revision */ + mark_t next_file_mark; + + bool processHasStarted; + + void startFastImport(); + void closeFastImport(); + + // called when a transaction is deleted + void forgetTransaction(Transaction *t); + + int resetBranch(const QString &branch, int revnum, mark_t mark, const QByteArray &resetTo, const QByteArray &comment); + long long markFrom(const QString &branchFrom, int branchRevNum, QByteArray &desc); + + friend class ProcessCache; + Q_DISABLE_COPY(FastImportRepository) +}; + +class ForwardingRepository : public Repository +{ + QString name; + Repository *repo; + QString prefix; +public: + class Transaction : public Repository::Transaction + { + Q_DISABLE_COPY(Transaction) + + Repository::Transaction *txn; + QString prefix; + public: + Transaction(Repository::Transaction *t, const QString &p) : txn(t), prefix(p) {} + ~Transaction() { delete txn; } + void commit() { txn->commit(); } + + void setAuthor(const QByteArray &author) { txn->setAuthor(author); } + void setDateTime(uint dt) { txn->setDateTime(dt); } + void setLog(const QByteArray &log) { txn->setLog(log); } + + void noteCopyFromBranch (const QString &prevbranch, int revFrom) + { txn->noteCopyFromBranch(prevbranch, revFrom); } + + void deleteFile(const QString &path) { txn->deleteFile(prefix + path); } + QIODevice *addFile(const QString &path, int mode, qint64 length) + { return txn->addFile(prefix + path, mode, length); } + + void commitNote(const QByteArray ¬eText, bool append, + const QByteArray &commit) + { return txn->commitNote(noteText, append, commit); } + }; + + ForwardingRepository(const QString &n, Repository *r, const QString &p) : name(n), repo(r), prefix(p) {} + + int setupIncremental(int &) { return 1; } + void restoreLog() {} + + void reloadBranches() { return repo->reloadBranches(); } + int createBranch(const QString &branch, int revnum, + const QString &branchFrom, int revFrom) + { return repo->createBranch(branch, revnum, branchFrom, revFrom); } + + int deleteBranch(const QString &branch, int revnum) + { return repo->deleteBranch(branch, revnum); } + + Repository::Transaction *newTransaction(const QString &branch, const QString &svnprefix, int revnum) + { + Repository::Transaction *t = repo->newTransaction(branch, svnprefix, revnum); + return new Transaction(t, prefix); + } + + void createAnnotatedTag(const QString &name, const QString &svnprefix, int revnum, + const QByteArray &author, uint dt, + const QByteArray &log) + { repo->createAnnotatedTag(name, svnprefix, revnum, author, dt, log); } + void finalizeTags() { /* loop that called this will invoke it on 'repo' too */ } + void commit() { repo->commit(); } + + bool branchExists(const QString& branch) const + { return repo->branchExists(branch); } + const QByteArray branchNote(const QString& branch) const + { return repo->branchNote(branch); } + void setBranchNote(const QString& branch, const QByteArray& noteText) + { repo->setBranchNote(branch, noteText); } + + bool hasPrefix() const + { return !prefix.isEmpty() || repo->hasPrefix(); } + + QString getName() const + { return name; } + Repository *getEffectiveRepository() + { return repo->getEffectiveRepository(); } +}; + +class ProcessCache: QLinkedList<FastImportRepository *> +{ +public: + void touch(FastImportRepository *repo) { remove(repo); @@ -43,7 +235,7 @@ public: append(repo); } - inline void remove(Repository *repo) + inline void remove(FastImportRepository *repo) { #if QT_VERSION >= 0x040400 removeOne(repo); @@ -54,6 +246,18 @@ public: }; static ProcessCache processCache; +Repository *createRepository(const Rules::Repository &rule, const QHash<QString, Repository *> &repositories) +{ + if (rule.forwardTo.isEmpty()) + return new FastImportRepository(rule); + Repository *r = repositories[rule.forwardTo]; + if (!r) { + qCritical() << "no repository with name" << rule.forwardTo << "found at" << rule.info(); + return r; + } + return new ForwardingRepository(rule.name, r, rule.prefix); +} + static QString marksFileName(QString name) { name.replace('/', '_'); @@ -61,7 +265,7 @@ static QString marksFileName(QString nam return name; } -Repository::Repository(const Rules::Repository &rule) +FastImportRepository::FastImportRepository(const Rules::Repository &rule) : name(rule.name), prefix(rule.forwardTo), fastImport(name), commitCount(0), outstandingTransactions(0), last_commit_mark(0), next_file_mark(maxMark), processHasStarted(false) { @@ -76,7 +280,7 @@ Repository::Repository(const Rules::Repo branches["master"].created = 1; fastImport.setWorkingDirectory(name); - if (!CommandLineParser::instance()->contains("dry-run")) { + if (!CommandLineParser::instance()->contains("dry-run") && !CommandLineParser::instance()->contains("create-dump")) { if (!QDir(name).exists()) { // repo doesn't exist yet. qDebug() << "Creating new repository" << name; QDir::current().mkpath(name); @@ -105,17 +309,21 @@ Repository::Repository(const Rules::Repo static QString logFileName(QString name) { name.replace('/', '_'); - name.prepend("log-"); + if (CommandLineParser::instance()->contains("create-dump")) + name.append(".fi"); + else + name.prepend("log-"); return name; } -static int lastValidMark(QString name) +static mark_t lastValidMark(QString name) { QFile marksfile(name + "/" + marksFileName(name)); if (!marksfile.open(QIODevice::ReadOnly)) return 0; - int prev_mark = 0; + qDebug() << "marksfile " << marksfile.fileName() ; + mark_t prev_mark = 0; int lineno = 0; while (!marksfile.atEnd()) { @@ -124,17 +332,17 @@ static int lastValidMark(QString name) if (line.isEmpty()) continue; - int mark = 0; + mark_t mark = 0; if (line[0] == ':') { int sp = line.indexOf(' '); if (sp != -1) { QString m = line.mid(1, sp-1); - mark = m.toInt(); + mark = m.toULongLong(); } } if (!mark) { - qCritical() << marksfile.fileName() << "line" << lineno << "marks file corrupt?"; + qCritical() << marksfile.fileName() << "line" << lineno << "marks file corrupt?" << "mark " << mark; return 0; } @@ -157,7 +365,7 @@ static int lastValidMark(QString name) return prev_mark; } -int Repository::setupIncremental(int &cutoff) +int FastImportRepository::setupIncremental(int &cutoff) { QFile logfile(logFileName(name)); if (!logfile.exists()) @@ -167,7 +375,7 @@ int Repository::setupIncremental(int &cu QRegExp progress("progress SVN r(\\d+) branch (.*) = :(\\d+)"); - int last_valid_mark = lastValidMark(name); + mark_t last_valid_mark = lastValidMark(name); int last_revnum = 0; qint64 pos = 0; @@ -188,7 +396,7 @@ int Repository::setupIncremental(int &cu int revnum = progress.cap(1).toInt(); QString branch = progress.cap(2); - int mark = progress.cap(3).toInt(); + mark_t mark = progress.cap(3).toULongLong(); if (revnum >= cutoff) goto beyond_cutoff; @@ -237,7 +445,7 @@ int Repository::setupIncremental(int &cu return cutoff; } -void Repository::restoreLog() +void FastImportRepository::restoreLog() { QString file = logFileName(name); QString bkup = file + ".old"; @@ -247,13 +455,13 @@ void Repository::restoreLog() QFile::rename(bkup, file); } -Repository::~Repository() +FastImportRepository::~FastImportRepository() { Q_ASSERT(outstandingTransactions == 0); closeFastImport(); } -void Repository::closeFastImport() +void FastImportRepository::closeFastImport() { if (fastImport.state() != QProcess::NotRunning) { fastImport.write("checkpoint\n"); @@ -269,7 +477,7 @@ void Repository::closeFastImport() processCache.remove(this); } -void Repository::reloadBranches() +void FastImportRepository::reloadBranches() { bool reset_notes = false; foreach (QString branch, branches.keys()) { @@ -297,7 +505,7 @@ void Repository::reloadBranches() } } -int Repository::markFrom(const QString &branchFrom, int branchRevNum, QByteArray &branchFromDesc) +long long FastImportRepository::markFrom(const QString &branchFrom, int branchRevNum, QByteArray &branchFromDesc) { Branch &brFrom = branches[branchFrom]; if (!brFrom.created) @@ -327,11 +535,11 @@ int Repository::markFrom(const QString & return brFrom.marks[it - brFrom.commits.begin()]; } -int Repository::createBranch(const QString &branch, int revnum, - const QString &branchFrom, int branchRevNum) +int FastImportRepository::createBranch(const QString &branch, int revnum, + const QString &branchFrom, int branchRevNum) { QByteArray branchFromDesc = "from branch " + branchFrom.toUtf8(); - int mark = markFrom(branchFrom, branchRevNum, branchFromDesc); + long long mark = markFrom(branchFrom, branchRevNum, branchFromDesc); if (mark == -1) { qCritical() << branch << "in repository" << name @@ -358,13 +566,13 @@ int Repository::createBranch(const QStri return resetBranch(branch, revnum, mark, branchFromRef, branchFromDesc); } -int Repository::deleteBranch(const QString &branch, int revnum) +int FastImportRepository::deleteBranch(const QString &branch, int revnum) { static QByteArray null_sha(40, '0'); return resetBranch(branch, revnum, 0, null_sha, "delete"); } -int Repository::resetBranch(const QString &branch, int revnum, int mark, const QByteArray &resetTo, const QByteArray &comment) +int FastImportRepository::resetBranch(const QString &branch, int revnum, mark_t mark, const QByteArray &resetTo, const QByteArray &comment) { QByteArray branchRef = branch.toUtf8(); if (!branchRef.startsWith("refs/")) @@ -399,7 +607,7 @@ int Repository::resetBranch(const QStrin return EXIT_SUCCESS; } -void Repository::commit() +void FastImportRepository::commit() { if (deletedBranches.isEmpty() && resetBranches.isEmpty()) { return; @@ -411,8 +619,8 @@ void Repository::commit() resetBranches.clear(); } -Repository::Transaction *Repository::newTransaction(const QString &branch, const QString &svnprefix, - int revnum) +Repository::Transaction *FastImportRepository::newTransaction(const QString &branch, const QString &svnprefix, + int revnum) { if (!branches.contains(branch)) { qWarning() << "WARN: Transaction:" << branch << "is not a known branch in repository" << name << endl @@ -436,16 +644,16 @@ Repository::Transaction *Repository::new return txn; } -void Repository::forgetTransaction(Transaction *) +void FastImportRepository::forgetTransaction(Transaction *) { if (!--outstandingTransactions) next_file_mark = maxMark; } -void Repository::createAnnotatedTag(const QString &ref, const QString &svnprefix, - int revnum, - const QByteArray &author, uint dt, - const QByteArray &log) +void FastImportRepository::createAnnotatedTag(const QString &ref, const QString &svnprefix, + int revnum, + const QByteArray &author, uint dt, + const QByteArray &log) { QString tagName = ref; if (tagName.startsWith("refs/tags/")) @@ -465,7 +673,7 @@ void Repository::createAnnotatedTag(cons tag.dt = dt; } -void Repository::finalizeTags() +void FastImportRepository::finalizeTags() { if (annotatedTags.isEmpty()) return; @@ -525,7 +733,30 @@ void Repository::finalizeTags() printf("\n"); } -void Repository::startFastImport() + +QByteArray +FastImportRepository::msgFilter(QByteArray msg) +{ + QByteArray output = msg; + + if (CommandLineParser::instance()->contains("msg-filter")) { + if (filterMsg.state() == QProcess::Running) + qFatal("filter process already running?"); + + filterMsg.start(CommandLineParser::instance()->optionArgument("msg-filter")); + + if(!(filterMsg.waitForStarted(-1))) + qFatal("Failed to Start Filter %d %s", __LINE__, qPrintable(filterMsg.errorString())); + + filterMsg.write(msg); + filterMsg.closeWriteChannel(); + filterMsg.waitForFinished(); + output = filterMsg.readAllStandardOutput(); + } + return output; +} + +void FastImportRepository::startFastImport() { processCache.touch(this); @@ -544,7 +775,7 @@ void Repository::startFastImport() fastImport.setStandardOutputFile(logFileName(name), QIODevice::Append); fastImport.setProcessChannelMode(QProcess::MergedChannels); - if (!CommandLineParser::instance()->contains("dry-run")) { + if (!CommandLineParser::instance()->contains("dry-run") && !CommandLineParser::instance()->contains("create-dump")) { fastImport.start("git", QStringList() << "fast-import" << marksOptions); } else { fastImport.start("/bin/cat", QStringList()); @@ -564,50 +795,65 @@ QByteArray Repository::formatMetadataMes return msg; } -bool Repository::branchExists(const QString& branch) const +bool FastImportRepository::branchExists(const QString& branch) const { return branches.contains(branch); } -const QByteArray Repository::branchNote(const QString& branch) const +const QByteArray FastImportRepository::branchNote(const QString& branch) const { return branches.value(branch).note; } -void Repository::setBranchNote(const QString& branch, const QByteArray& noteText) +void FastImportRepository::setBranchNote(const QString& branch, const QByteArray& noteText) { if (branches.contains(branch)) branches[branch].note = noteText; } -Repository::Transaction::~Transaction() +bool FastImportRepository::hasPrefix() const +{ + return !prefix.isEmpty(); +} + +QString FastImportRepository::getName() const +{ + return name; *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201701121153.v0CBrXgs037865>