From owner-svn-doc-head@freebsd.org Fri Oct 6 09:36:07 2017 Return-Path: Delivered-To: svn-doc-head@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id A9A53E31EE8; Fri, 6 Oct 2017 09:36:07 +0000 (UTC) (envelope-from wosch@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 8354A772F0; Fri, 6 Oct 2017 09:36:07 +0000 (UTC) (envelope-from wosch@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v969a6jN076256; Fri, 6 Oct 2017 09:36:06 GMT (envelope-from wosch@FreeBSD.org) Received: (from wosch@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v969a64c076255; Fri, 6 Oct 2017 09:36:06 GMT (envelope-from wosch@FreeBSD.org) Message-Id: <201710060936.v969a64c076255@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: wosch set sender to wosch@FreeBSD.org using -f From: Wolfram Schneider Date: Fri, 6 Oct 2017 09:36:06 +0000 (UTC) To: doc-committers@freebsd.org, svn-doc-all@freebsd.org, svn-doc-head@freebsd.org Subject: svn commit: r51064 - head/share/tools X-SVN-Group: doc-head X-SVN-Commit-Author: wosch X-SVN-Commit-Paths: head/share/tools X-SVN-Commit-Revision: 51064 X-SVN-Commit-Repository: doc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-doc-head@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the doc tree for head List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 06 Oct 2017 09:36:07 -0000 Author: wosch Date: Fri Oct 6 09:36:06 2017 New Revision: 51064 URL: https://svnweb.freebsd.org/changeset/doc/51064 Log: Import a regression test script for the build of www.freebsd.org The main purpose of the script is to validate the output after a refactoring of the Makefile tool chain, and to improve the build speed. Added: head/share/tools/webupdate-regression (contents, props changed) Added: head/share/tools/webupdate-regression ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/share/tools/webupdate-regression Fri Oct 6 09:36:06 2017 (r51064) @@ -0,0 +1,246 @@ +#!/bin/sh +# Copyright (c) 2017 Wolfram Schneider +# +# regression test of the build of www.freebsd.org +# Based on git +# +# examples: +# +# run on 2 CPUs +# make_opt=-j2 /path/to/webupdate-regression +# +# run from a local GIT repo in $HOME/freebsd-doc +# GIT_REPO_FREEBSD_DOC="$HOME/freebsd-doc" /path/to/webupdate-regression +# +# build html-split format only, on 8 CPUs +# WEBUPDATE_FORMATS="html-split" make_opt=-j8 /path/to/webupdate-regression +# +# build html and html-split format, on 16 CPUs max. +# WEBUPDATE_FORMATS="html html-split" make_opt=-j16 /path/to/webupdate-regression +# +# do not run the `make clean`' test after successfully build +# WEBUPDATE_RUN_CLEAN=NO /path/to/webupdate-regression +# +# test only english pages +# ENGLISH_ONLY=YES /path/to/webupdate-regression +# +# clean git checkout after success +# WEBUPDATE_SANDBOX_CLEANUP=YES /path/to/webupdate-regression +# +# +# $FreeBSD$ + +set -e + +: ${NO_OBJ=YES} +: ${make_opt=-j8} +: ${GIT_REPO_FREEBSD_DOC="$HOME/freebsd-doc"} +: ${WEBUPDATE_FORMATS="html-split html"} +: ${WEBUPDATE_RUN_CLEAN=YES} +: ${WEBUPDATE_LEVEL="3"} +: ${WEBUPDATE_SANDBOX_CLEANUP="NO"} + +FORMATS=$WEBUPDATE_FORMATS +export NO_OBJ +export FORMATS + + +log () { + file=$1 + echo " ...done" +} + +checkout () +{ + dir=$(mktemp -d /tmp/doc-build.XXXXXXX) + + cd $dir + echo "Run from $dir" + echo "date: $(date)" + echo "ENGLISH_ONLY=$ENGLISH_ONLY" + echo "WEB_ONLY=$WEB_ONLY" + echo "FORMATS=\"$FORMATS\"" + echo "WEBUPDATE_RUN_CLEAN=$WEBUPDATE_RUN_CLEAN" + echo "WEBUPDATE_SANDBOX_CLEANUP=$WEBUPDATE_SANDBOX_CLEANUP" + echo "WEBUPDATE_LEVEL=$WEBUPDATE_LEVEL" + echo "NO_OBJ=$NO_OBJ" + echo "make_opt=$make_opt" + + echo "git clone from $GIT_REPO_FREEBSD_DOC -> doc" + git clone -q $GIT_REPO_FREEBSD_DOC doc + cd doc + echo "branch: $(git branch)" + echo "cwd: $(pwd)" + + echo "" +} + +# build doc only, from ./doc +# parallel build, 4-40 minutes +build_doc () +{ + logfile=$1 + echo -n "build doc (4-40min) $(pwd)/$logfile" + time make $make_opt p-all >> $logfile 2>&1; log $logfile +} + +# build web only, from english web pages +build_htdocs_web () +{ + logfile=$1 + ( + cd en_US.ISO8859-1/htdocs + echo -n "build htdocs web (1min) $(pwd)/$logfile" + WEB_ONLY=YES time make $make_opt >> $logfile 2>&1; log $logfile + echo -n "build htdocs web/ports (1min)" + WEB_ONLY=YES time make -C./ports $make_opt >> $logfile 2>&1; log $logfile + ) +} + +# build web + doc, from english web pages +# after build_htdocs_web: 1-3 minutes (?) +build_htdocs () +{ + logfile=$1 + ( + cd en_US.ISO8859-1/htdocs + echo -n "build htdocs (1-3min) $(pwd)/$logfile" + time make $make_opt all >> $logfile 2>&1; log $logfile + ) +} + +# install web only, from english web pages +install_htdocs () +{ + logfile=$1 + ( + cd en_US.ISO8859-1/htdocs + echo -n "install htdocs web (<10 sec) $(pwd)/$logfile" + DESTDIR=$dir/www WEB_ONLY=YES time make $make_opt install >> $logfile 2>&1; log $logfile + #egrep -rl wosch $dir/www >/dev/null + ) +} + +# install web + doc, from english web pages +install_doc () +{ + logfile=$1 + ( + echo "no parallel install" + make_opt= + + cd en_US.ISO8859-1/htdocs + echo -n "install htdocs (<30 sec) $(pwd)/$logfile" + DESTDIR=$dir/www time make $make_opt install >> $logfile 2>&1; log $logfile + ) +} + +# clean doc from ./doc +# parallel clean +clean_doc () +{ + logfile=$1 + echo -n "clean doc (<5 sec) $(pwd)/$logfile" + time make $make_opt p-clean >> $logfile 2>&1; log $logfile +} + +# clean doc from english web pages +# is required for docs which don't have a web pages (./htdocs) +clean_htdocs () +{ + logfile=$1 + ( + cd en_US.ISO8859-1/htdocs + echo -n "clean htdocs web + doc (<20 sec) $(pwd)/$logfile" + time make $make_opt clean >> $logfile 2>&1; log $logfile + ) +} + +######################################################################################## +# + +checkout + +# ./doc +if [ "$WEB_ONLY" != "YES" ]; then + build_doc log.build_doc + if [ $WEBUPDATE_LEVEL -ge 2 ]; then + build_doc log.build_doc2 + fi + if [ $WEBUPDATE_LEVEL -ge 3 ]; then + build_doc log.build_doc3 + fi +fi + +# doc/en_US.ISO8859-1/htdocs +build_htdocs_web log.build_htdocs_web +if [ $WEBUPDATE_LEVEL -ge 2 ]; then + build_htdocs_web log.build_htdocs_web2 +fi +if [ $WEBUPDATE_LEVEL -ge 3 ]; then + build_htdocs_web log.build_htdocs_web3 +fi + +# doc/en_US.ISO8859-1/htdocs/doc +build_htdocs log.build_htdocs +if [ $WEBUPDATE_LEVEL -ge 2 ]; then + build_htdocs log.build_htdocs2 +fi +if [ $WEBUPDATE_LEVEL -ge 3 ]; then + build_htdocs log.build_htdocs3 +fi + +# install doc/en_US.ISO8859-1/htdocs +install_htdocs log.install_htdocs +if [ $WEBUPDATE_LEVEL -ge 2 ]; then + install_htdocs log.install_htdocs2 +fi + if [ $WEBUPDATE_LEVEL -ge 3 ]; then +install_htdocs log.install_htdocs3 +fi + +# install doc/en_US.ISO8859-1/htdocs/doc +install_doc log.install_doc +if [ $WEBUPDATE_LEVEL -ge 2 ]; then + install_doc log.install_doc2 +fi +if [ $WEBUPDATE_LEVEL -ge 3 ]; then + install_doc log.install_doc3 +fi + +# make clean +if [ "$WEBUPDATE_RUN_CLEAN" = "YES" ]; then + clean_htdocs log.clean_htdocs + if [ $WEBUPDATE_LEVEL -ge 2 ]; then + clean_htdocs log.clean_htdocs2 + fi + + clean_doc log.clean_doc + if [ $WEBUPDATE_LEVEL -ge 2 ]; then + clean_doc log.clean_doc2 + fi +fi + +echo "" +echo "successfully done" +echo "" + +# display user time for each task +tail -n 1 $(ls -tr $(find . -name 'log.*' -type f)) + +echo "" +echo "installed data disk usage: $(du -hs $dir/www)" +echo "installed file number: $(find $dir/www | wc -l)" +( cd $dir/www && find -s . > ../find.www ) + +if [ "$WEBUPDATE_SANDBOX_CLEANUP" = "YES" ]; then + echo "cleanup sandbox $dir" + rm -rf $dir +else + echo "" + echo "Please cleanup: rm -rf $dir" +fi + +exit 0 + +#EOF