From owner-svn-soc-all@FreeBSD.ORG Mon May 28 10:49:45 2012 Return-Path: Delivered-To: svn-soc-all@FreeBSD.org Received: from socsvn.FreeBSD.org (unknown [IPv6:2001:4f8:fff6::2f]) by hub.freebsd.org (Postfix) with SMTP id 0490D106564A for ; Mon, 28 May 2012 10:49:43 +0000 (UTC) (envelope-from scher@FreeBSD.org) Received: by socsvn.FreeBSD.org (sSMTP sendmail emulation); Mon, 28 May 2012 10:49:43 +0000 Date: Mon, 28 May 2012 10:49:43 +0000 From: scher@FreeBSD.org To: svn-soc-all@FreeBSD.org MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Message-Id: <20120528104943.0490D106564A@hub.freebsd.org> Cc: Subject: socsvn commit: r236583 - soc2012/scher/par_ports/head/Mk X-BeenThere: svn-soc-all@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: SVN commit messages for the entire Summer of Code repository List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 28 May 2012 10:49:45 -0000 Author: scher Date: Mon May 28 10:49:42 2012 New Revision: 236583 URL: http://svnweb.FreeBSD.org/socsvn/?view=rev&rev=236583 Log: [new_feature] _parv_${_lock_dir}_LOCK_LOOP - main scripts to lock a directory _parv_CHECK_LOCK - script to check if $${dir} is locked. [doc] Submitted by: Alexander Pronin M bsd.parallel.mk M bsd.port.mk Modified: soc2012/scher/par_ports/head/Mk/bsd.parallel.mk soc2012/scher/par_ports/head/Mk/bsd.port.mk Modified: soc2012/scher/par_ports/head/Mk/bsd.parallel.mk ============================================================================== --- soc2012/scher/par_ports/head/Mk/bsd.parallel.mk Mon May 28 09:51:10 2012 (r236582) +++ soc2012/scher/par_ports/head/Mk/bsd.parallel.mk Mon May 28 10:49:42 2012 (r236583) @@ -11,11 +11,15 @@ _parv_= ##################################################### # Debugging specific tools and variable declarations -_dparv_START_OUTPUT_MESSAGE= "=================_PAR_PORTS_SPECIFIC_OUTPUT_==============" -_dparv_END_OUTPUT_MESSAGE= "==============_END_OF_PAR_PORTS_SPECIFIC_OUTPUT_==============" +_dparv_START_OUTPUT_MESSAGE= =================_PAR_PORTS_SPECIFIC_OUTPUT_============== +_dparv_END_OUTPUT_MESSAGE= ==============_END_OF_PAR_PORTS_SPECIFIC_OUTPUT_============== _dparv_START_OUTPUT= ${ECHO_CMD} ${_dparv_START_OUTPUT_MESSAGE} _dparv_END_OUTPUT= ${ECHO_CMD} ${_dparv_END_OUTPUT_MESSAGE} + +# Delay for feedback message if the directory is locked. +# Just not to annoy a user with feedback message on each attempt to lock a directory. +_parv_ON_LOCK_FEEDBACK_TIMEOUT?= 2 # End of Debugging specific tools and variable declarations section ##################################################### ##################################################### @@ -28,8 +32,6 @@ .if !${.TARGETS} _parv_IS_DEFAULT_TARGET= 1 .else -_dparv_TARGETS:=${.TARGETS} - .for _called_target in ${.TARGETS} _tmp_called_target= ${_called_target} . for _def_target in ${_parv_DEFAULT_TARGETS} @@ -44,6 +46,8 @@ # Commands _parv_KILL= /bin/kill _parv_KILL_FLAGS= -- +_parv_PKILL= /bin/pkill +_parv_PKILL_FLAGS= -P DO_NADA?= ${TRUE} # End of Commands section @@ -56,6 +60,13 @@ _parv_WAIT_FOR_LOCK_TIME= 5 _parv_WAIT_FOR_UNLOCK_TIME= 15 +# Delay in seconds between attempts to lock a directory in lock loops +_parv_LOCK_ATTEMPT_TIMEOUT=2 + +# exit status of lock script in case the directory is locked +_parv_ON_LOCK_EXIT_STATUS= 2 +_parv_LOCKF_EX_TEMPFAIL= 75 + # Senquence of commands to lock a directory using ${_parv_LOCK_FILE}. # During evaluation of the following commands lockf(1) is holding lock on ${_parv_LOCK_FILE} file. # Hence NO other process is able to evaluate any commands using lockf(1) @@ -63,13 +74,7 @@ # # Stalled locks cheking enabled. # -# If the directory is locked then make process will be terminated with 143 exit code. -# If ${_parv_LOCK_FILE} is locked then make process will be terminated with 143 exit code. -# -# For loop produces two varuables: -# _parv_PKG_DBDIR_LOCK_SEQ -# _parv_.CURDIR_LOCK_SEQ -# as sequencies of commands to provide locking of ${PKG_DBDIR} and ${.CURDIR} accordingly. +# If the directory is locked this script returns ${_parv_ON_LOCK_EXIT_STATUS}. # # ${${_lock_dir}} == ${PKG_DBDIR} OR ${.CURDIR} # @@ -81,60 +86,145 @@ ${CHMOD} 777 ${${_lock_dir}}/${_parv_LOCK_FILE}; \ pid=$$(${CAT} ${${_lock_dir}}/${_parv_LOCK_FILE}); \ if [ $${pid} ]; then \ - ps -p $${pid} > /dev/null; \ - status=$$(${ECHO_CMD} $$?); \ + ps -p $${pid} > /dev/null && status=$$? || status=$$?; \ if [ $${status} -eq 0 ]; then \ - ${_dparv_START_OUTPUT}; \ - ${ECHO_CMD} Unable to lock ${${_lock_dir}}; \ - ${ECHO_CMD} Dir: ${${_lock_dir}} is already locked by another working process PID=$${pid}...; \ - ${_dparv_END_OUTPUT}; \ - ${_parv_KILL} ${_parv_KILL_FLAGS} -${.MAKE.PID}; \ + exit ${_parv_ON_LOCK_EXIT_STATUS}; \ else \ ${_dparv_START_OUTPUT}; \ - ${ECHO_CMD} Dir: ${${_lock_dir}} Stalled lock Detected!; \ - ${ECHO_CMD} Deleting stalled lock. PID=$${pid}; \ - ${ECHO_CMD} Locking: ${${_lock_dir}}; \ + ${ECHO_CMD} "Dir: ${${_lock_dir}} Stalled lock Detected!"; \ + ${ECHO_CMD} "Deleting stalled lock. PID=$${pid}"; \ + ${ECHO_CMD} "Locking: ${${_lock_dir}}"; \ ${_dparv_END_OUTPUT}; \ ${ECHO_CMD} ${.MAKE.PID} > ${${_lock_dir}}/${_parv_LOCK_FILE}; \ fi; \ else \ ${_dparv_START_OUTPUT}; \ - ${ECHO_CMD} Locking: ${${_lock_dir}}; \ + ${ECHO_CMD} "Locking: ${${_lock_dir}}"; \ ${_dparv_END_OUTPUT}; \ ${ECHO_CMD} ${.MAKE.PID} > ${${_lock_dir}}/${_parv_LOCK_FILE}; \ fi +##################################################### + # _parv_PKG_DBDIR_DO_LOCK # _parv_.CURDIR_DO_LOCK +# This scripts handles exit status of lockf(1) call. +# It substitutes exit status 75 of lockf(1) for ${_parv_ON_LOCK_EXIT_STATUS} and pushes it. # _parv_${_lock_dir}_DO_LOCK= \ - lockf -k -t ${_parv_WAIT_FOR_LOCK_TIME} ${${_lock_dir}}/${_parv_LOCK_FILE} ${SH} -c '${_parv_${_lock_dir}_LOCK_SEQ}' || ( \ - ${_dparv_START_OUTPUT}; \ - ${ECHO_CMD} "Unable to lock ${${_lock_dir}}"; \ - ${ECHO_CMD} "Lock file: ${${_lock_dir}}/${_parv_LOCK_FILE} is alredy locked by another working process ..."; \ - ${_dparv_END_OUTPUT}; \ - ${_parv_KILL} ${_parv_KILL_FLAGS} -${.MAKE.PID}; \ - ) + lockf -k -t ${_parv_WAIT_FOR_LOCK_TIME} ${${_lock_dir}}/${_parv_LOCK_FILE} ${SH} -c '${_parv_${_lock_dir}_LOCK_SEQ}' || { \ + status=$$?; \ + if [ $${status} -eq ${_parv_LOCKF_EX_TEMPFAIL} ] || \ + [ $${status} -eq ${_parv_ON_LOCK_EXIT_STATUS} ]; then \ + exit ${_parv_ON_LOCK_EXIT_STATUS}; \ + else \ + ${_dparv_START_OUTPUT}; \ + ${ECHO_CMD} "Unhandled EXIT STATUS = $${status}. Terminating"; \ + ${_dparv_END_OUTPUT}; \ + exit $${status}; \ + fi; \ + } + +##################################################### + +# Loops to lock directory +# _parv_PKG_DBDIR_LOCK_LOOP +# _parv_.CURDIR_LOCK_LOOP +# $${attempts} - Number of attempts to lock a directory. Exetranl variable. +# Default value = 1, if this var is not set. +# Set this variable to -1 for infinity loop. +# e.g. ( attempts=10; ${_parv_.CURDIR_LOCK_LOOP} ) && ..... || .... +# +_parv_${_lock_dir}_LOCK_LOOP= \ + enable_feedback=${_parv_ON_LOCK_FEEDBACK_TIMEOUT}; \ + if [ ! $${attempts} ]; then attempts=1; fi; \ + while [ $${attempts} -ne 0 ]; do \ + attempts=$$(( $${attempts} - 1 )); \ + ( ${_parv_${_lock_dir}_DO_LOCK} ) && status=$$? || status=$$?; \ + if [ $${status} -eq 0 ]; then \ + exit 0; \ + elif [ $${status} -eq ${_parv_ON_LOCK_EXIT_STATUS} ]; then \ + if [ $$(( $${enable_feedback} % ${_parv_ON_LOCK_FEEDBACK_TIMEOUT} )) -eq 0 ]; then \ + ${_dparv_START_OUTPUT}; \ + ${ECHO_CMD} "Unable to lock ${${_lock_dir}}"; \ + ${ECHO_CMD} "Dir: ${${_lock_dir}} is already locked by another working process ..."; \ + ${ECHO_CMD} "Waiting for unlock ........................................................."; \ + ${_dparv_END_OUTPUT}; \ + enable_feedback=0; \ + fi; \ + enable_feedback=$$(( $${enable_feedback} + 1 )); \ + sleep ${_parv_LOCK_ATTEMPT_TIMEOUT}; \ + continue; \ + else \ + exit 1; \ + fi; \ + done; \ + exit ${_parv_ON_LOCK_EXIT_STATUS} + +##################################################### # _parv_PKG_DBDIR_DO_UNLOCK # _parv_.CURDIR_DO_UNLOCK # -_parv_${_lock_dir}_DO_UNLOCK= ( \ - lockf -k -t ${_parv_WAIT_FOR_UNLOCK_TIME} ${${_lock_dir}}/${_parv_LOCK_FILE} ${RM} ${${_lock_dir}}/${_parv_LOCK_FILE} || \ - ( \ - ${_dparv_START_OUTPUT}; \ - ${ECHO_CMD} Unable to unlock ${${_lock_dir}}; \ - ${_dparv_END_OUTPUT}; \ - ${_parv_KILL} ${_parv_KILL_FLAGS} -${.MAKE.PID}; \ - ) \ -) && ( \ - ${_dparv_START_OUTPUT}; \ - ${ECHO_CMD} "Dir: ${${_lock_dir}} is unloked"; \ - ${_dparv_END_OUTPUT}; \ -) +_parv_${_lock_dir}_DO_UNLOCK= \ + lockf -k -t ${_parv_WAIT_FOR_UNLOCK_TIME} ${${_lock_dir}}/${_parv_LOCK_FILE} \ + ${RM} ${${_lock_dir}}/${_parv_LOCK_FILE} && { \ + ${_dparv_START_OUTPUT}; \ + ${ECHO_CMD} "Dir: ${${_lock_dir}} is unlocked"; \ + ${_dparv_END_OUTPUT}; \ + } || { \ + ${_dparv_START_OUTPUT}; \ + ${ECHO_CMD} "Hard unlock!"; \ + ${RM} -rf ${${_lock_dir}}/${_parv_LOCK_FILE}; \ + ${_dparv_END_OUTPUT}; \ + } + .endfor # _lock_dir in PKG_DBDIR .CURDIR +##################################################### + +# _parv_CHECK_SEQ +# _parv_CHECK_LOCK +# The former variables Implement check for lock utility +# $${dir} - dir to check. External variable for script. Assign this variable +# appropriate value before executing this script e.g. ( dir=/some/dir/to/check; ${_parv_CHECK_LOCK} ) || ... +# Script exits with status ${_parv_ON_LOCK_EXIT_STATUS} if $${dir} is locked +# +_parv_CHECK_SEQ= \ + ${CHMOD} 777 $${dir}/${_parv_LOCK_FILE}; \ + pid=\$$(${CAT} $${dir}/${_parv_LOCK_FILE}); \ + if [ \$${pid} ]; then \ + ps -p \$${pid} > /dev/null && status=\$$? || status=\$$?; \ + if [ \$${status} -eq 0 ]; then \ + ${_dparv_START_OUTPUT}; \ + ${ECHO_CMD} Unable to lock $${dir}; \ + ${ECHO_CMD} Dir: $${dir} is already locked by another working process ...; \ + ${_dparv_START_OUTPUT}; \ + exit ${_parv_ON_LOCK_EXIT_STATUS}; \ + else \ + ${_dparv_START_OUTPUT}; \ + ${ECHO_CMD} Dir: $${dir} Stalled lock Detected!; \ + ${ECHO_CMD} Deleting stalled lock. PID=\$${pid}; \ + ${_dparv_END_OUTPUT}; \ + fi; \ + fi; \ + ${RM} -rf $${dir}/${_parv_LOCK_FILE} + +_parv_CHECK_LOCK= \ + lockf -k -t ${_parv_WAIT_FOR_LOCK_TIME} $${dir}/${_parv_LOCK_FILE} ${SH} -c "${_parv_CHECK_SEQ}" || { \ + status=$$?; \ + if [ $${status} -eq ${_parv_LOCKF_EX_TEMPFAIL} ] || \ + [ $${status} -eq ${_parv_ON_LOCK_EXIT_STATUS} ]; then \ + exit ${_parv_ON_LOCK_EXIT_STATUS}; \ + else \ + ${_dparv_START_OUTPUT}; \ + ${ECHO_CMD} "Unhandled EXIT STATUS = $${status}. Terminating"; \ + ${_dparv_END_OUTPUT}; \ + exit $${status}; \ + fi; \ + } + # End of Locking variables and tools section ##################################################### @@ -142,5 +232,4 @@ @${DO_NADA} do-unlock: - @${DO_NADA} - + @${DO_NADA} \ No newline at end of file Modified: soc2012/scher/par_ports/head/Mk/bsd.port.mk ============================================================================== --- soc2012/scher/par_ports/head/Mk/bsd.port.mk Mon May 28 09:51:10 2012 (r236582) +++ soc2012/scher/par_ports/head/Mk/bsd.port.mk Mon May 28 10:49:42 2012 (r236583) @@ -1531,7 +1531,7 @@ .BEGIN: . if defined(WANT_PARALLEL_BUILD) . if ${_parv_IS_DEFAULT_TARGET} - @${_parv_.CURDIR_DO_LOCK} + @attempts=-1; ${_parv_.CURDIR_LOCK_LOOP} . endif . endif # You can force skipping these test by defining IGNORE_PATH_CHECKS