Date: Sun, 15 Apr 2007 17:18:34 +0200 From: Ulrich Spoerlein <uspoerlein@gmail.com> To: ports@freebsd.org Subject: Re: CFT: Parallel package building script Message-ID: <20070415151834.GE1525@roadrunner.q.local> In-Reply-To: <20070415092815.GB1525@roadrunner.q.local> References: <20070415092815.GB1525@roadrunner.q.local>
next in thread | previous in thread | raw e-mail | index | archive | help
--h56sxpGKRmy85csR Content-Type: multipart/mixed; boundary="yudcn1FV7Hsu/q59" Content-Disposition: inline --yudcn1FV7Hsu/q59 Content-Type: text/plain; charset=us-ascii Content-Disposition: inline Content-Transfer-Encoding: quoted-printable Ulrich Spoerlein wrote: > I wrote a script=20 That got stripped by the mailling lists ... second try. You can also fetch it from http://coyote.dnsalias.net/create_pkg_makefile.sh Ulrich Spoerlein --=20 "The trouble with the dictionary is you have to know how the word is spelled before you can look it up to see how it is spelled." -- Will Cuppy --yudcn1FV7Hsu/q59 Content-Type: text/plain; charset=us-ascii Content-Disposition: attachment; filename="create_pkg_makefile.sh" Content-Transfer-Encoding: quoted-printable #!/usr/bin/env zsh # vi:set sw=3D2 tw=3D78: # # "THE BEER-WARE LICENSE" (Revision 42): # <uspoerlein@gmail.com> wrote this file. As long as you retain this notice= you # can do whatever you want with this stuff. If we meet some day, and you th= ink # this stuff is worth it, you can buy me a beer in return. Ulrich Spoerlein #=20 # What this is: #=20 # This script will output a Makefile, that is self-contained and suitable f= or # building individual sets of packages with maximum parallelism. # # Who it is for: # # For people needing to build clean packages, probably with a custom # make.conf, for system deployment. # # How it works: #=20 # It will recursively walk the $PORTS tree for the specified package origins # (or all POs if none are given) and create the correct dependency tree plus # targets to build those packages. # # To guarantee a clean package without "contamination", as can happen by # configure scripts picking up random installed libraries, it uses a clean # "installworld" provided by either an unionfs mount or by copying the tree= to # the destination. (There are a some problems with unionfs mounts, if packa= ges # don't build, try the cpio method first.) # # How to use: #=20 # First, you need to create a clean environment: # cd $SRC && make buildworld installworld distribution DESTDIR=3D$ROOT/live # mkdir -p $ROOT/live/usr/ports $ROOT/distfiles $ROOT/obj $ROOT/packages/All # # Since we can't change make(1)s .OBJDIR to some arbitrary value, create an # obj link to the package directory: # ln -sf $ROOT/packages/All obj # # Create a make.conf with your desired flags, set CONF=3D$PWD/make.conf # (= default) # vi make.conf # # And torture your machine=20 # ./create_pkg_makefiles.sh editors/vim x11/kde3 | make -f- -j4 all #=20 # To watch the build progressing, use something like: # while sleep 3; do screen -r; done # # The Makefile for all ports can be created by: # ./create_pkg_makefiles.sh > Makefile # # The special "check" target will only print what packages would be build; # use it in conjunction with "all" or your desired package origins. # make -j4 check editors/vim x11/kde3 # make check all # # To update all your installed packages, a heavy handed approach would be t= o: # cp /etc/make.conf . # ./create_pkg_makefiles.sh `pkg_info -qao` | make -f- -j2 all # pkg_delete -fa # pkg_add -F $ROOT/packages/All/* #=20 # User configuration ROOT=3D/vol PORTS=3D/usr/ports SRC=3D/usr/src CONF=3D$PWD/make.conf #CONF=3D/etc/make.conf # End of user configuration. # Global hash, saving pkg-origin -> pkg-name. Is also used to look up # which pkg-origins have already been checked. typeset -A pkg print_make_header() { cat <<EOS # Root dir of build space R=3D${ROOT} B=3D\$R/live O=3D\$R/obj PKG=3D\$R/packages DIST=3D\$R/distfiles # Source directories S=3D${SRC} P=3D${PORTS} CONF=3D${CONF} # Choose your poison MK_LIVE_TREE=3D cd \$B && find . | cpio --quiet -dump \$O/\$\$po UNMK_LIVE_TREE=3D rm -rf \$O/\$\$po #MK_LIVE_TREE=3D zfs clone ... #UNMK_LIVE_TREE=3D zfs remove clone ... MK_LIVE_TREE=3D mount -t unionfs -o below,noatime \$B \$O/\$\$po UNMK_LIVE_TREE=3D umount \$O/\$\$po; rm -rf \$O/\$\$po CLEAN_BUILD_SPACE=3Dclean_build_space() { set +e; po=3D\$\$1; shift; \ umount \$O/\$\$po/usr/ports/packages; \ umount \$O/\$\$po/usr/ports/distfiles; \ umount \$O/\$\$po/usr/ports; \ umount \$O/\$\$po/usr/src; \ umount \$O/\$\$po/compat/linux/proc; \ umount \$O/\$\$po/dev; \ \${UNMK_LIVE_TREE}; return 0; }; clean_build_space CREATE_BUILD_SPACE=3Dcreate_build_space() { po=3D\$\$1; shift; \ mkdir -p \$O/\$\$po/usr/ports; \ mkdir -p \$O/\$\$po/compat/linux/proc; \ \${MK_LIVE_TREE}; \ mount -t devfs devfs \$O/\$\$po/dev; \ mount -t linprocfs linprocfs \$O/\$\$po/compat/linux/proc; \ mount -t nullfs -oro \$S \$O/\$\$po/usr/src; \ mount -t nullfs -oro \$P \$O/\$\$po/usr/ports; \ mount -t nullfs \${DIST} \$O/\$\$po/usr/ports/distfiles; \ mount -t nullfs \${PKG} \$O/\$\$po/usr/ports/packages; \ test ! -r "\${CONF}" || cp \${CONF} \$O/\$\$po/etc; }; create_build_space # The su(1) is required to have a fresh environment, otherwise the # make(1) vars of -jX will interfere with the package build. # screen(1) will always exit with code 0, use test(1) outside of screen # to figure out if everything went well. CREATE_PACKAGE=3Dcreate_package() { set -e; po=3D\$\$1; shift; \ cd \$O/\$\$po && screen -L -D -m chroot . time -lp -o time.out sh -ec "\ for dep in \${.ALLSRC}; do \ echo \"Adding package \\\$\${dep}\"; \ PKG_PATH=3D/usr/ports/packages/All pkg_add -F \\\$\${dep%.tbz}; \ done; \ su -l root -c \"make -C /usr/ports/\$\$po WRKDIRPREFIX=3D/usr/obj \ BATCH=3Dyes build package\""; \ test -f \${PKG}/All/\${.TARGET}; }; create_package PARSE_TIME=3Dparse_time() { po=3D\$\$1; shift; \ awk -v po=3D\$\$po '/real/{r=3D\$\$2} /user/{u=3D\$\$2} \ /sys/{s=3D\$\$2} /resident set size/{rss=3D\$\$1} \ END{print po ": real " r " user " u " sys " s " rss " rss}' \ \$O/\$\$po/time.out 2>/dev/null; }; parse_time =2ESILENT: check: EOS } print_pkg() { local po po=3D$1; shift cat <<EOS $po: ${pkg[$po]} ${pkg[$po]}: $* =2Eif make(check) echo "$po: Building \${.TARGET} <=3D \${.ALLSRC}" =2Eelse for dep in $*; do test -f \${PKG}/All/\$\$dep || \\ (echo \"$po: Missing dep \$\$dep, skipped\"; exit 1); done echo '$po: Preparing build space' \${CLEAN_BUILD_SPACE} $po 2>/dev/null \${CREATE_BUILD_SPACE} $po echo '$po: Building package' \${CREATE_PACKAGE} $po \${PARSE_TIME} $po echo '$po: Cleaning build space' \${CLEAN_BUILD_SPACE} $po =2Eendif EOS } get_pkg_recursive() { local po list depport deplist level po=3D$1; shift level=3D"${1}=3D"; shift # Don't revisit already visited nodes if [ -n "$pkg[$po]" ]; then return fi if [ -z "$pkg[$po]" ]; then pkg[$po]=3D$(make -C $po __MAKE_CONF=3D$CONF -V PKGNAME).tbz fi printf "%-45s %s\n" "$level > $po" $pkg[$po] >&2 # Get dependency dirs list=3D$(make -C $po __MAKE_CONF=3D$CONF -V _DEPEND_DIRS | sed "s|${PORTS= }/*||g") for depport in `eval echo $list`; do if [ -z "$pkg[$depport]" ]; then get_pkg_recursive $depport $level fi case "$deplist" in *${pkg[$depport]}*) ;; *) deplist=3D"$deplist ${pkg[$depport]}" ;; esac done echo "$level < $po" >&2 print_pkg $po `eval echo $deplist` } # Main entry point print_make_header cd $PORTS if [ "$#" -gt 0 ]; then for po; do get_pkg_recursive $po "" done else for sub in $(make -C $PORTS -V SUBDIR); do for port in $(make -C $PORTS/$sub -V SUBDIR); do get_pkg_recursive $sub/$port "" done done fi printf "\nall: $pkg\n" --yudcn1FV7Hsu/q59-- --h56sxpGKRmy85csR Content-Type: application/pgp-signature Content-Disposition: inline -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.4.6 (FreeBSD) iD8DBQFGIkJK524iJyD+6d0RAiGAAJ9ZvF+3MJ3aCz6ljRwt5IsCHVs+LQCeO2IF Ho8SrGfTQrVUImxNZyUlgRU= =ZO6M -----END PGP SIGNATURE----- --h56sxpGKRmy85csR--
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?20070415151834.GE1525>