Date: Mon, 17 Apr 2023 01:40:59 GMT From: "Simon J. Gerraty" <sjg@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org Subject: git: bf7aa99a55a7 - main - Update meta mode makefiles Message-ID: <202304170140.33H1exhm078120@gitrepo.freebsd.org>
next in thread | raw e-mail | index | archive | help
The branch main has been updated by sjg: URL: https://cgit.FreeBSD.org/src/commit/?id=bf7aa99a55a7692da6e55864098fd085392542b0 commit bf7aa99a55a7692da6e55864098fd085392542b0 Author: Simon J. Gerraty <sjg@FreeBSD.org> AuthorDate: 2023-04-17 01:40:53 +0000 Commit: Simon J. Gerraty <sjg@FreeBSD.org> CommitDate: 2023-04-17 01:40:53 +0000 Update meta mode makefiles meta2deps - add checks to detect truncated/corrupted filemon data (only known to happen on Linux hosts), to ensure we do not auto update dependencies based on incomplete data. meta.stage.mk adds STAGE_SHLIB_LINKS_FILTER and STAGE_LINK_AS_* We also allow for hosts where egrep is deprecated for grep -E Reviewed by: stevek --- share/mk/dirdeps-options.mk | 9 +++++++- share/mk/gendirdeps.mk | 8 ++++--- share/mk/meta.autodep.mk | 5 ++--- share/mk/meta.stage.mk | 21 ++++++++++-------- share/mk/meta2deps.py | 52 ++++++++++++++++++++++++++++++++++++++----- share/mk/meta2deps.sh | 54 ++++++++++++++++++++++++++++++++++++++------- 6 files changed, 119 insertions(+), 30 deletions(-) diff --git a/share/mk/dirdeps-options.mk b/share/mk/dirdeps-options.mk index b31d2033ae98..9a97615bbeb8 100644 --- a/share/mk/dirdeps-options.mk +++ b/share/mk/dirdeps-options.mk @@ -1,4 +1,4 @@ -# $Id: dirdeps-options.mk,v 1.20 2022/03/17 20:11:36 sjg Exp $ +# $Id: dirdeps-options.mk,v 1.21 2022/09/06 22:18:45 sjg Exp $ # # @(#) Copyright (c) 2018-2022, Simon J. Gerraty # @@ -42,6 +42,13 @@ # so we qualify MK_FOO with .${TARGET_SPEC} and each component # TARGET_SPEC_VAR (in reverse order) before using MK_FOO. # +# Because Makefile.depend.options are processed at both level 0 (when +# computing DIRDEPS to build) and higher (when updating +# Makefile.depend* after successful build), it is important that +# all references to TARGET_SPEC_VARs should use the syntax +# ${DEP_${TARGET_SPEC_VAR}:U${TARGET_SPEC_VAR}} to ensure correct +# behavior. +# # This should have been set by Makefile.depend.options # before including us diff --git a/share/mk/gendirdeps.mk b/share/mk/gendirdeps.mk index 6d26b3d308b1..1ff2036237ed 100644 --- a/share/mk/gendirdeps.mk +++ b/share/mk/gendirdeps.mk @@ -1,5 +1,4 @@ -# $FreeBSD$ -# $Id: gendirdeps.mk,v 1.46 2020/08/19 17:51:53 sjg Exp $ +# $Id: gendirdeps.mk,v 1.48 2022/09/09 17:44:29 sjg Exp $ # Copyright (c) 2011-2020, Simon J. Gerraty # Copyright (c) 2010-2018, Juniper Networks, Inc. @@ -89,7 +88,7 @@ META_FILES := ${META_FILES:T:O:u} # they should all be absolute paths SKIP_GENDIRDEPS ?= .if !empty(SKIP_GENDIRDEPS) -_skip_gendirdeps = egrep -v '^(${SKIP_GENDIRDEPS:O:u:ts|})' | +_skip_gendirdeps = ${EGREP:Uegrep} -v '^(${SKIP_GENDIRDEPS:O:u:ts|})' | .else _skip_gendirdeps = .endif @@ -340,6 +339,8 @@ CAT_DEPEND ?= .depend .PHONY: ${_DEPENDFILE} .endif +# set this to 'no' and we will not capture any +# local depends LOCAL_DEPENDS_GUARD ?= _{.MAKE.LEVEL} > 0 # 'cat .depend' should suffice, but if we are mixing build modes @@ -352,6 +353,7 @@ ${_DEPENDFILE}: .NOMETA ${CAT_DEPEND:M.depend} ${META_FILES:O:u:@m@${exists($m): echo '${DIRDEPS:@d@ $d \\${.newline}@}'; echo; \ ${_include_src_dirdeps} \ echo '.include <dirdeps.mk>'; \ + [ "${LOCAL_DEPENDS_GUARD:[1]:tl}" != no ] || exit 0; \ echo; \ echo '.if ${LOCAL_DEPENDS_GUARD}'; \ echo '# local dependencies - needed for -jN in clean tree'; \ diff --git a/share/mk/meta.autodep.mk b/share/mk/meta.autodep.mk index 842554beea45..cd08ac3b3520 100644 --- a/share/mk/meta.autodep.mk +++ b/share/mk/meta.autodep.mk @@ -1,5 +1,4 @@ -# $FreeBSD$ -# $Id: meta.autodep.mk,v 1.55 2021/12/13 08:12:01 sjg Exp $ +# $Id: meta.autodep.mk,v 1.56 2022/09/09 17:44:29 sjg Exp $ # # @(#) Copyright (c) 2010, Simon J. Gerraty @@ -175,7 +174,7 @@ DEPEND_SUFFIXES += .c .h .cpp .hpp .cxx .hxx .cc .hh .endif .depend: .NOMETA $${.MAKE.META.CREATED} ${_this} @echo "Updating $@: ${.OODATE:T:[1..8]}" - @egrep -i '^R .*\.(${DEPEND_SUFFIXES:tl:O:u:S,^.,,:ts|})$$' /dev/null ${.MAKE.META.FILES:T:O:u:${META_FILE_FILTER:ts:}:M*o.meta} | \ + @${EGREP:Uegrep} -i '^R .*\.(${DEPEND_SUFFIXES:tl:O:u:S,^.,,:ts|})$$' /dev/null ${.MAKE.META.FILES:T:O:u:${META_FILE_FILTER:ts:}:M*o.meta} | \ sed -e 's, \./, ,${OBJDIR_REFS:O:u:@d@;s, $d/, ,@};/\//d' \ -e 's,^\([^/][^/]*\).meta...[0-9]* ,\1: ,' | \ sort -u | \ diff --git a/share/mk/meta.stage.mk b/share/mk/meta.stage.mk index 9f54f3b1f9c5..168e46d22a82 100644 --- a/share/mk/meta.stage.mk +++ b/share/mk/meta.stage.mk @@ -1,5 +1,4 @@ -# $FreeBSD$ -# $Id: meta.stage.mk,v 1.60 2020/08/19 17:51:53 sjg Exp $ +# $Id: meta.stage.mk,v 1.67 2023/04/17 01:22:10 sjg Exp $ # # @(#) Copyright (c) 2011-2017, Simon J. Gerraty # @@ -31,8 +30,11 @@ _dirdep ?= ${RELDIR} CLEANFILES+= .dirdep # this allows us to trace dependencies back to their src dir -.dirdep: .NOPATH +.dirdep: .NOPATH +.if !commands(.dirdep) +.dirdep: @echo '${_dirdep}' > $@ +.endif .if defined(NO_POSIX_SHELL) || ${type printf:L:sh:Mbuiltin} == "" _stage_file_basename = `basename $$f` @@ -64,7 +66,7 @@ GENDIRDEPS_FILTER += Nnot-empty-is-important \ LN_CP_SCRIPT = LnCp() { \ rm -f $$2 2> /dev/null; \ { [ -z "$$mode" ] && ${LN:Uln} $$1 $$2 2> /dev/null; } || \ - cp -p $$1 $$2; } + cp -p $$1 $$2 2> /dev/null || cp $$1 $$2; } # a staging conflict should cause an error # a warning is handy when bootstapping different options. @@ -171,7 +173,7 @@ stage_libs: .dirdep .if !defined(NO_SHLIB_LINKS) .if !empty(SHLIB_LINKS) @${STAGE_LINKS_SCRIPT}; StageLinks -s ${STAGE_LIBDIR:${STAGE_DIR_FILTER}} \ - ${SHLIB_LINKS:@t@${STAGE_LIBS:T:M$t.*} $t@} + ${SHLIB_LINKS:@t@${STAGE_LIBS:T:M$t.*:${STAGE_SHLIB_LINKS_FILTER:U}} $t@} .elif !empty(SHLIB_LINK) && !empty(SHLIB_NAME) @${STAGE_LINKS_SCRIPT}; StageLinks -s ${STAGE_LIBDIR:${STAGE_DIR_FILTER}} ${SHLIB_NAME} ${SHLIB_LINK} .endif @@ -262,7 +264,8 @@ CLEANFILES += ${STAGE_AS_SETS:@s@stage*$s@} # sometimes things need to be renamed as they are staged # each ${file} will be staged as ${STAGE_AS_${file:T}} # one could achieve the same with SYMLINKS -# stage_as_and_symlink makes the original name a symlink to the new name +# stage_as_and_symlink makes the original name (or ${STAGE_LINK_AS_${name}}) +# a symlink to the new name # it is the same as using stage_as and stage_symlinks but ensures # both operations happen together .for s in ${STAGE_AS_SETS:O:u} @@ -292,7 +295,7 @@ STAGE_AS_AND_SYMLINK.$s ?= ${.ALLSRC:N.dirdep:Nstage_*} stage_as_and_symlink: stage_as_and_symlink.$s stage_as_and_symlink.$s: .dirdep @${STAGE_AS_SCRIPT}; StageAs ${FLAGS.$@} ${STAGE_FILES_DIR.$s:U${STAGE_DIR.$s}:${STAGE_DIR_FILTER}} ${STAGE_AS_AND_SYMLINK.$s:O:@f@$f ${STAGE_AS_${f:tA}:U${STAGE_AS_${f:T}:U${f:T}}}@} - @${STAGE_LINKS_SCRIPT}; StageLinks -s ${STAGE_FILES_DIR.$s:U${STAGE_DIR.$s}:${STAGE_DIR_FILTER}} ${STAGE_AS_AND_SYMLINK.$s:O:@f@${STAGE_AS_${f:tA}:U${STAGE_AS_${f:T}:U${f:T}}} $f@} + @${STAGE_LINKS_SCRIPT}; StageLinks -s ${STAGE_FILES_DIR.$s:U${STAGE_DIR.$s}:${STAGE_DIR_FILTER}} ${STAGE_AS_AND_SYMLINK.$s:O:@f@${STAGE_AS_${f:tA}:U${STAGE_AS_${f:T}:U${f:T}}} ${STAGE_LINK_AS_${f}:U$f}@} @touch $@ .endif .endif @@ -304,7 +307,7 @@ CLEANFILES += ${STAGE_TARGETS} stage_incs stage_includes # this lot also only makes sense the first time... .if !target(__${.PARSEFILE}__) -__${.PARSEFILE}__: +__${.PARSEFILE}__: .NOTMAIN # stage_*links usually needs to follow any others. # for non-jobs mode the order here matters @@ -351,7 +354,7 @@ all: stale_staged # get a list of paths that we have previously staged to those same dirs # anything in the 2nd list but not the first is stale - remove it. stale_staged: staging .NOMETA - @egrep '^[WL] .*${STAGE_OBJTOP}' /dev/null ${.MAKE.META.FILES:M*stage_*} | \ + @${EGREP:Uegrep} '^[WL] .*${STAGE_OBJTOP}' /dev/null ${.MAKE.META.FILES:M*stage_*} | \ sed "/\.dirdep/d;s,.* '*\(${STAGE_OBJTOP}/[^ '][^ ']*\).*,\1," | \ sort > ${.TARGET}.staged1 @grep -l '${_dirdep}' /dev/null ${_STAGED_DIRS:M${STAGE_OBJTOP}*:O:u:@d@$d/*.dirdep@} | \ diff --git a/share/mk/meta2deps.py b/share/mk/meta2deps.py index 1bcf1baedaaa..d7820ec71b5b 100755 --- a/share/mk/meta2deps.py +++ b/share/mk/meta2deps.py @@ -37,8 +37,7 @@ We only pay attention to a subset of the information in the """ RCSid: - $FreeBSD$ - $Id: meta2deps.py,v 1.40 2021/12/13 19:32:46 sjg Exp $ + $Id: meta2deps.py,v 1.45 2023/01/18 01:35:24 sjg Exp $ Copyright (c) 2011-2020, Simon J. Gerraty Copyright (c) 2011-2017, Juniper Networks, Inc. @@ -67,7 +66,10 @@ RCSid: """ -import os, re, sys +import os +import re +import sys +import stat def resolve(path, cwd, last_dir=None, debug=0, debug_out=sys.stderr): """ @@ -245,6 +247,7 @@ class MetaFile: self.curdir = conf.get('CURDIR') self.reldir = conf.get('RELDIR') self.dpdeps = conf.get('DPDEPS') + self.pids = {} self.line = 0 if not self.conf: @@ -445,12 +448,13 @@ class MetaFile: pid_cwd = {} pid_last_dir = {} last_pid = 0 + eof_token = False self.line = 0 if self.curdir: self.seenit(self.curdir) # we ignore this - interesting = 'CEFLRV' + interesting = '#CEFLRVX' for line in f: self.line += 1 # ignore anything we don't care about @@ -477,6 +481,12 @@ class MetaFile: print("%s: CWD=%s" % (self.name, cwd), file=self.debug_out) continue + if w[0] == '#': + # check the file has not been truncated + if line.find('Bye') > 0: + eof_token = True + continue + pid = int(w[1]) if pid != last_pid: if last_pid: @@ -506,6 +516,13 @@ class MetaFile: print("cwd=", cwd, file=self.debug_out) continue + if w[0] == 'X': + try: + del self.pids[pid] + except KeyError: + pass + continue + if w[2] in self.seen: if self.debug > 2: print("seen:", w[2], file=self.debug_out) @@ -519,11 +536,34 @@ class MetaFile: continue elif w[0] in 'ERWS': path = w[2] - if path == '.': + if w[0] == 'E': + self.pids[pid] = path + elif path == '.': continue self.parse_path(path, cwd, w[0], w) - assert(version > 0) + if version == 0: + raise AssertionError('missing filemon data') + if not eof_token: + raise AssertionError('truncated filemon data') + + setid_pids = [] + # self.pids should be empty! + for pid,path in self.pids.items(): + try: + # no guarantee that path is still valid + if os.stat(path).st_mode & (stat.S_ISUID|stat.S_ISGID): + # we do not expect anything after Exec + setid_pids.append(pid) + continue + except: + # we do not care why the above fails, + # we do not want to miss the ERROR below. + pass + print("ERROR: missing eXit for {} pid {}".format(path, pid)) + for pid in setid_pids: + del self.pids[pid] + assert(len(self.pids) == 0) if not file: f.close() diff --git a/share/mk/meta2deps.sh b/share/mk/meta2deps.sh index 4767b83f4343..56367e0105f4 100755 --- a/share/mk/meta2deps.sh +++ b/share/mk/meta2deps.sh @@ -49,8 +49,10 @@ # The output, is a set of absolute paths with "SB" like: #.nf # +# $SB/obj-i386/bsd/gnu/lib/csu +# $SB/obj-i386/bsd/gnu/lib/libgcc # $SB/obj-i386/bsd/include -# $SB/obj-i386/bsd/lib/csu/i386 +# $SB/obj-i386/bsd/lib/csu/i386-elf # $SB/obj-i386/bsd/lib/libc # $SB/src/bsd/include # $SB/src/bsd/sys/i386/include @@ -75,8 +77,7 @@ # RCSid: -# $FreeBSD$ -# $Id: meta2deps.sh,v 1.14 2020/10/02 03:11:17 sjg Exp $ +# $Id: meta2deps.sh,v 1.20 2023/01/18 01:35:24 sjg Exp $ # Copyright (c) 2010-2013, Juniper Networks, Inc. # All rights reserved. @@ -138,6 +139,13 @@ add_list() { eval "$name=\"$list\"" } +# some Linux systems have deprecated egrep in favor of grep -E +# but not everyone supports that +case "`echo bmake | egrep 'a|b' 2>&1`" in +bmake) ;; +*) egrep() { grep -E "$@"; } +esac + _excludes_f() { egrep -v "$EXCLUDES" } @@ -240,8 +248,8 @@ meta2deps() { ;; *) cat /dev/null "$@";; esac 2> /dev/null | - sed -e 's,^CWD,C C,;/^[CREFLMV] /!d' -e "s,',,g" | - $_excludes | ( version=no + sed -e 's,^CWD,C C,;/^[#CREFLMVX] /!d' -e "s,',,g" | + $_excludes | ( version=no epids= xpids= eof_token=no while read op pid path junk do : op=$op pid=$pid path=$path @@ -259,10 +267,15 @@ meta2deps() { *) ;; esac version=0 + case "$eof_token" in + no) ;; # ignore + 0) error "truncated filemon data";; + esac + eof_token=0 continue ;; $pid,$pid) ;; - *) + [1-9]*) case "$lpid" in "") ;; *) eval ldir_$lpid=$ldir;; @@ -272,8 +285,9 @@ meta2deps() { ;; esac + : op=$op path=$path case "$op,$path" in - V,*) version=$path; continue;; + V,*) version=$pid; continue;; W,*srcrel|*.dirdep) continue;; C,*) case "$path" in @@ -289,7 +303,18 @@ meta2deps() { eval cwd_$path=$cwd ldir_$path=$ldir continue ;; + \#,bye) eof_token=1; continue;; + \#*) continue;; *) dir=${path%/*} + case "$op" in + E) # setid apps get no tracing so we won't see eXit + case `'ls' -l $path 2> /dev/null | sed 's, .*,,'` in + *s*) ;; + *) epids="$epids $pid";; + esac + ;; + X) xpids="$xpids $pid"; continue;; + esac case "$path" in $src_re|$obj_re) ;; /*/stage/*) ;; @@ -379,9 +404,22 @@ meta2deps() { echo $dir;; esac done > $tf.dirdep + : version=$version case "$version" in 0) error "no filemon data";; - esac ) || exit 1 + esac + : eof_token=$eof_token + case "$eof_token" in + 0) error "truncated filemon data";; + esac + for p in $epids + do + : p=$p + case " $xpids " in + *" $p "*) ;; + *) error "missing eXit for pid $p";; + esac + done ) || exit 1 _nl=echo for f in $tf.dirdep $tf.qual $tf.srcdep do
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202304170140.33H1exhm078120>