From owner-svn-src-all@FreeBSD.ORG Tue Feb 17 23:13:47 2015 Return-Path: Delivered-To: svn-src-all@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [8.8.178.115]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by hub.freebsd.org (Postfix) with ESMTPS id F0DDA205; Tue, 17 Feb 2015 23:13:46 +0000 (UTC) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id DB0613B2; Tue, 17 Feb 2015 23:13:46 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.9/8.14.9) with ESMTP id t1HNDklr000103; Tue, 17 Feb 2015 23:13:46 GMT (envelope-from rpaulo@FreeBSD.org) Received: (from rpaulo@localhost) by svn.freebsd.org (8.14.9/8.14.9/Submit) id t1HNDjRq099998; Tue, 17 Feb 2015 23:13:45 GMT (envelope-from rpaulo@FreeBSD.org) Message-Id: <201502172313.t1HNDjRq099998@svn.freebsd.org> X-Authentication-Warning: svn.freebsd.org: rpaulo set sender to rpaulo@FreeBSD.org using -f From: Rui Paulo Date: Tue, 17 Feb 2015 23:13:45 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r278926 - in head: . release share/mk X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.18-1 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 17 Feb 2015 23:13:47 -0000 Author: rpaulo Date: Tue Feb 17 23:13:45 2015 New Revision: 278926 URL: https://svnweb.freebsd.org/changeset/base/278926 Log: release: use xz via pipe when compressing the tarballs. libarchive(3) doesn't support the new liblzma API yet, but this change allows us to enable multi-threaded xz compression. ``make release'' should now finish in half the time on a machine with several cores and fast disks (our typical build server). This behaviour only applies when building a release and it doesn't affect buildworld/installworld. To disable threaded xz compression, set XZ_THREADS=1. Reviewed by: gjb Tested by: gjb Modified: head/Makefile.inc1 head/release/Makefile head/release/Makefile.vm head/share/mk/bsd.own.mk Modified: head/Makefile.inc1 ============================================================================== --- head/Makefile.inc1 Tue Feb 17 22:33:22 2015 (r278925) +++ head/Makefile.inc1 Tue Feb 17 23:13:45 2015 (r278926) @@ -912,25 +912,25 @@ packageworld: .for dist in base ${EXTRA_DISTRIBUTIONS} .if defined(NO_ROOT) ${_+_}cd ${DESTDIR}/${DISTDIR}/${dist}; \ - tar cvJf ${DESTDIR}/${DISTDIR}/${dist}.txz \ - --exclude usr/lib/debug \ - @${DESTDIR}/${DISTDIR}/${dist}.meta + tar cvf - --exclude usr/lib/debug \ + @${DESTDIR}/${DISTDIR}/${dist}.meta | \ + ${XZ_CMD} > ${DESTDIR}/${DISTDIR}/${dist}.txz .else ${_+_}cd ${DESTDIR}/${DISTDIR}/${dist}; \ - tar cvJf ${DESTDIR}/${DISTDIR}/${dist}.txz \ - --exclude usr/lib/debug . + tar cvf - --exclude usr/lib/debug . | \ + ${XZ_CMD} > ${DESTDIR}/${DISTDIR}/${dist}.txz .endif .endfor .for dist in ${DEBUG_DISTRIBUTIONS} . if defined(NO_ROOT) ${_+_}cd ${DESTDIR}/${DISTDIR}/${dist}; \ - tar cvJf ${DESTDIR}/${DISTDIR}/${dist}-dbg.txz \ - @${DESTDIR}/${DISTDIR}/${dist}.debug.meta + tar cvf - @${DESTDIR}/${DISTDIR}/${dist}.debug.meta | \ + ${XZ_CMD} > ${DESTDIR}/${DISTDIR}/${dist}-dbg.txz . else ${_+_}cd ${DESTDIR}/${DISTDIR}/${dist}; \ - tar cvJLf ${DESTDIR}/${DISTDIR}/${dist}-dbg.txz \ - usr/lib/debug + tar cvLf - usr/lib/debug | \ + ${XZ_CMD} > ${DESTDIR}/${DISTDIR}/${dist}-dbg.txz . endif .endfor @@ -1138,19 +1138,21 @@ distributekernel distributekernel.debug: packagekernel: .if defined(NO_ROOT) cd ${DESTDIR}/${DISTDIR}/kernel; \ - tar cvJf ${DESTDIR}/${DISTDIR}/kernel.txz \ - @${DESTDIR}/${DISTDIR}/kernel.meta + tar cvf - @${DESTDIR}/${DISTDIR}/kernel.meta | \ + ${XZ_CMD} > ${DESTDIR}/${DISTDIR}/kernel.txz .for _kernel in ${BUILDKERNELS:S/${INSTALLKERNEL}//} cd ${DESTDIR}/${DISTDIR}/kernel.${_kernel}; \ - tar cvJf ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.txz \ - @${DESTDIR}/${DISTDIR}/kernel.${_kernel}.meta + tar cvf - @${DESTDIR}/${DISTDIR}/kernel.${_kernel}.meta | \ + ${XZ_CMD} > ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.txz .endfor .else cd ${DESTDIR}/${DISTDIR}/kernel; \ - tar cvJf ${DESTDIR}/${DISTDIR}/kernel.txz . + tar cvf - . | \ + ${XZ_CMD} > ${DESTDIR}/${DISTDIR}/kernel.txz .for _kernel in ${BUILDKERNELS:S/${INSTALLKERNEL}//} cd ${DESTDIR}/${DISTDIR}/kernel.${_kernel}; \ - tar cvJf ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.txz . + tar cvf - . | \ + ${XZ_CMD} > ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.txz .endfor .endif Modified: head/release/Makefile ============================================================================== --- head/release/Makefile Tue Feb 17 22:33:22 2015 (r278925) +++ head/release/Makefile Tue Feb 17 23:13:45 2015 (r278926) @@ -33,18 +33,12 @@ # with xz(1) (extremely time consuming) # WITH_CLOUDWARE: if set, build cloud hosting disk images with the release # TARGET/TARGET_ARCH: architecture of built release -# XZ_FLAGS: Additional arguments to pass to xz(1) -# XZ_THREADS: Number of xz(1) threads to use -# NO_XZTHREADS: Disable multi-threaded xz(1) compression # WORLDDIR?= ${.CURDIR}/.. PORTSDIR?= /usr/ports DOCDIR?= /usr/doc RELNOTES_LANG?= en_US.ISO8859-1 -XZCMD?= /usr/bin/xz -XZ_FLAGS?= -XZ_THREADS?= .if !defined(TARGET) || empty(TARGET) TARGET= ${MACHINE} @@ -59,15 +53,6 @@ TARGET_ARCH= ${TARGET} IMAKE= ${MAKE} TARGET_ARCH=${TARGET_ARCH} TARGET=${TARGET} DISTDIR= dist -.if !defined(NO_XZTHREADS) && empty(XZ_THREADS) -XZ_THREADS= 0 -.else -XZ_THREADS= ${XZ_THREADS} -.endif -.if !empty(XZ_THREADS) -XZ_FLAGS+= -T ${XZ_THREADS} -.endif - # Define OSRELEASE by using newvars.sh .if !defined(OSRELEASE) || empty(OSRELEASE) .for _V in TYPE BRANCH REVISION @@ -158,16 +143,18 @@ kernel.txz: src.txz: mkdir -p ${DISTDIR}/usr ln -fs ${WORLDDIR} ${DISTDIR}/usr/src - cd ${DISTDIR} && tar cLvJf ${.OBJDIR}/src.txz --exclude .svn --exclude .zfs \ - --exclude .git --exclude @ --exclude usr/src/release/dist usr/src + cd ${DISTDIR} && tar cLvf - --exclude .svn --exclude .zfs \ + --exclude .git --exclude @ --exclude usr/src/release/dist usr/src | \ + ${XZ_CMD} > ${.OBJDIR}/src.txz ports.txz: mkdir -p ${DISTDIR}/usr ln -fs ${PORTSDIR} ${DISTDIR}/usr/ports - cd ${DISTDIR} && tar cLvJf ${.OBJDIR}/ports.txz \ + cd ${DISTDIR} && tar cLvf - \ --exclude .git --exclude .svn \ --exclude usr/ports/distfiles --exclude usr/ports/packages \ - --exclude 'usr/ports/INDEX*' --exclude work usr/ports + --exclude 'usr/ports/INDEX*' --exclude work usr/ports | \ + ${XZ_CMD} > ${.OBJDIR}/ports.txz reldoc: cd ${.CURDIR}/doc && ${MAKE} all install clean 'FORMATS=html txt' \ @@ -309,7 +296,7 @@ release-install: .for I in ${IMAGES} cp -p ${I} ${DESTDIR}/${OSRELEASE}-${I} . if defined(WITH_COMPRESSED_IMAGES) && !empty(WITH_COMPRESSED_IMAGES) - ${XZCMD} ${XZ_FLAGS} -k ${DESTDIR}/${OSRELEASE}-${I} + ${XZ_CMD} -k ${DESTDIR}/${OSRELEASE}-${I} . endif .endfor cd ${DESTDIR} && sha256 ${OSRELEASE}* > ${DESTDIR}/CHECKSUM.SHA256 Modified: head/release/Makefile.vm ============================================================================== --- head/release/Makefile.vm Tue Feb 17 22:33:22 2015 (r278925) +++ head/release/Makefile.vm Tue Feb 17 23:13:45 2015 (r278926) @@ -101,7 +101,7 @@ vm-install: # the DESTDIR. . for FORMAT in ${VMFORMATS} # Don't keep the originals. There is a copy in ${.OBJDIR} if needed. - ${XZCMD} ${XZ_FLAGS} ${DESTDIR}/vmimages/${OSRELEASE}.${FORMAT} + ${XZ_CMD} ${DESTDIR}/vmimages/${OSRELEASE}.${FORMAT} . endfor . endif cd ${DESTDIR}/vmimages && sha256 ${OSRELEASE}* > \ Modified: head/share/mk/bsd.own.mk ============================================================================== --- head/share/mk/bsd.own.mk Tue Feb 17 22:33:22 2015 (r278925) +++ head/share/mk/bsd.own.mk Tue Feb 17 23:13:45 2015 (r278926) @@ -234,6 +234,15 @@ STRIP?= -s COMPRESS_CMD?= gzip -cn COMPRESS_EXT?= .gz +# Set XZ_THREADS to 1 to disable multi-threading. +XZ_THREADS?= 0 + +.if !empty(XZ_THREADS) +XZ_CMD?= xz -T ${XZ_THREADS} +.else +XZ_CMD?= xz +.endif + # Pointer to the top directory into which tests are installed. Should not be # overriden by Makefiles, but the user may choose to set this in src.conf(5). TESTSBASE?= /usr/tests