Date: Fri, 3 Oct 2025 14:32:24 GMT From: Mark Johnston <markj@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org Subject: git: ea675a43f09b - main - libexec/kgdb: Add new modules and install them together with debug info Message-ID: <202510031432.593EWOIA097989@gitrepo.freebsd.org>
next in thread | raw e-mail | index | archive | help
The branch main has been updated by markj: URL: https://cgit.FreeBSD.org/src/commit/?id=ea675a43f09ba569adf1dd17b4f1ced970e48de4 commit ea675a43f09ba569adf1dd17b4f1ced970e48de4 Author: Mark Johnston <markj@FreeBSD.org> AuthorDate: 2025-10-03 14:25:53 +0000 Commit: Mark Johnston <markj@FreeBSD.org> CommitDate: 2025-10-03 14:32:09 +0000 libexec/kgdb: Add new modules and install them together with debug info This change simplifies integration of gdb python scripts with our kernel debugging infrastructure. Rather than putting debugging scripts in /usr/libexec/kgdb, move them to <path-to-kernel-debug-symbols>/gdb, and add a kernel-gdb.py which automatically loads modules from that directory. kernel-gdb.py will be automatically executed by kgdb when loading kernel debug symbols (assuming a default configuration), so one no longer needs to do anything to use these modules. The change also adds a couple of new modules, vnet.py and pcpu.py, for conveniently accessing VNET symbols and PCPU/DPCPU fields, respectively. Note that these require a change to the kernel linker when accessing symbols from a loadable kernel module. sys/tools/gdb/README.txt describes the scheme in more detail and provides some rudiementary documentation for the commands and functions added by these modules. It should be updated when adding new features. sys/tools/gdb/selftest.py can be used to do some primitive testing of the modules. All it does is execute a number of gdb commands making use of commands and functions added by these modules. The developer is expected to verify that the commands complete without errors and that the output looks sane. Discussed with: kp, avg, jhb, glebius MFC after: 2 weeks Differential Revision: https://reviews.freebsd.org/D50825 --- ObsoleteFiles.inc | 4 ++ etc/mtree/BSD.usr.dist | 2 - libexec/Makefile | 1 - libexec/kgdb/Makefile | 5 -- sys/conf/kern.post.mk | 15 +++++ sys/tools/gdb/README.txt | 21 ++++++ {libexec/kgdb => sys/tools/gdb}/acttrace.py | 41 ++++-------- sys/tools/gdb/freebsd.py | 75 +++++++++++++++++++++ sys/tools/gdb/pcpu.py | 77 +++++++++++++++++++++ sys/tools/gdb/selftest.py | 31 +++++++++ sys/tools/gdb/selftest.sh | 23 +++++++ sys/tools/gdb/vnet.py | 100 ++++++++++++++++++++++++++++ sys/tools/kernel-gdb.py | 15 +++++ usr.sbin/crashinfo/crashinfo.sh | 3 - 14 files changed, 374 insertions(+), 39 deletions(-) diff --git a/ObsoleteFiles.inc b/ObsoleteFiles.inc index 539ab7d54460..36e8c048661e 100644 --- a/ObsoleteFiles.inc +++ b/ObsoleteFiles.inc @@ -51,6 +51,10 @@ # xargs -n1 | sort | uniq -d; # done +# 20251003: kgdb python scripts moved +OLD_FILES+=usr/libexec/kgdb/acttrace.py +OLD_DIRS+=usr/libexec/kgdb + # 20251001: test helper sendto-IP_MULTICAST_IF renamed OLD_FILES+=usr/tests/sys/netinet/sendto-IP_MULTICAST_IF diff --git a/etc/mtree/BSD.usr.dist b/etc/mtree/BSD.usr.dist index 1945c26ebc5f..d7d839b94b96 100644 --- a/etc/mtree/BSD.usr.dist +++ b/etc/mtree/BSD.usr.dist @@ -181,8 +181,6 @@ .. hyperv .. - kgdb - .. lpr ru .. diff --git a/libexec/Makefile b/libexec/Makefile index 7ce78321f08e..e87b48b153a8 100644 --- a/libexec/Makefile +++ b/libexec/Makefile @@ -10,7 +10,6 @@ SUBDIR= ${_atf} \ flua \ getty \ ${_hyperv} \ - kgdb \ ${_mail.local} \ ${_makewhatis.local} \ ${_mknetid} \ diff --git a/libexec/kgdb/Makefile b/libexec/kgdb/Makefile deleted file mode 100644 index f6b255ab4f60..000000000000 --- a/libexec/kgdb/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -FILESDIR?= /usr/libexec/kgdb - -FILES= acttrace.py - -.include <bsd.prog.mk> diff --git a/sys/conf/kern.post.mk b/sys/conf/kern.post.mk index bb3c7af82a4d..7cdfd17778db 100644 --- a/sys/conf/kern.post.mk +++ b/sys/conf/kern.post.mk @@ -398,6 +398,14 @@ CFLAGS+= -fdebug-prefix-map=./${_link}=${PREFIX_SYSDIR}/${_link}/include .endif .endfor +# Install GDB plugins that are useful for kernel debugging. See the +# README in sys/tools/gdb for more information. +GDB_FILES= acttrace.py \ + freebsd.py \ + pcpu.py \ + selftest.py \ + vnet.py + ${_ILINKS}: @case ${.TARGET} in \ machine) \ @@ -447,6 +455,13 @@ kernel-install: .PHONY .if defined(DEBUG) && !defined(INSTALL_NODEBUG) && ${MK_KERNEL_SYMBOLS} != "no" mkdir -p ${DESTDIR}${KERN_DEBUGDIR}${KODIR} ${INSTALL} -p -m ${KMODMODE} -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_KO}.debug ${DESTDIR}${KERN_DEBUGDIR}${KODIR}/ + ${INSTALL} -m ${KMODMODE} -o ${KMODOWN} -g ${KMODGRP} \ + $S/tools/kernel-gdb.py ${DESTDIR}${KERN_DEBUGDIR}${KODIR}/${KERNEL_KO}-gdb.py + mkdir -p ${DESTDIR}${KERN_DEBUGDIR}${KODIR}/gdb +.for file in ${GDB_FILES} + ${INSTALL} -m ${KMODMODE} -o ${KMODOWN} -g ${KMODGRP} \ + $S/tools/gdb/${file} ${DESTDIR}${KERN_DEBUGDIR}${KODIR}/gdb/${file} +.endfor .endif .if defined(KERNEL_EXTRA_INSTALL) ${INSTALL} -p -m ${KMODMODE} -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_EXTRA_INSTALL} ${DESTDIR}${KODIR}/ diff --git a/sys/tools/gdb/README.txt b/sys/tools/gdb/README.txt new file mode 100644 index 000000000000..8c31565ddc42 --- /dev/null +++ b/sys/tools/gdb/README.txt @@ -0,0 +1,21 @@ +This directory contains Python scripts that can be loaded by GDB to help debug +FreeBSD kernel crashes. + +Add new commands and functions in their own files. Functions with general +utility should be added to freebsd.py. sys/tools/kernel-gdb.py is installed +into the kernel debug directory (typically /usr/lib/debug/boot/kernel). It will +be automatically loaded by kgdb when opening a vmcore, so if you add new GDB +commands or functions, that script should be updated to import them, and you +should document them here. + +To provide some rudimentary testing, selftest.py tries to exercise all of the +commands and functions defined here. To use it, run selftest.sh to panic the +system. Then, create a kernel dump or attach to the panicked kernel, and invoke +the script with "python import selftest" in (k)gdb. + +Commands: +acttrace Display a backtrace for all on-CPU threads + +Functions: +$PCPU(<field>[, <cpuid>]) Display the value of a PCPU/DPCPU field +$V(<variable>[, <vnet>]) Display the value of a VNET variable diff --git a/libexec/kgdb/acttrace.py b/sys/tools/gdb/acttrace.py similarity index 62% rename from libexec/kgdb/acttrace.py rename to sys/tools/gdb/acttrace.py index 3229ff708de1..147effbbddf1 100644 --- a/libexec/kgdb/acttrace.py +++ b/sys/tools/gdb/acttrace.py @@ -1,38 +1,23 @@ -#- +# # Copyright (c) 2022 The FreeBSD Foundation # # This software was developed by Mark Johnston under sponsorship from the # FreeBSD Foundation. # +# SPDX-License-Identifier: BSD-2-Clause +# import gdb - - -def symval(name): - return gdb.lookup_global_symbol(name).value() - - -def tid_to_gdb_thread(tid): - for thread in gdb.inferiors()[0].threads(): - if thread.ptid[2] == tid: - return thread - else: - return None - - -def all_pcpus(): - mp_maxid = symval("mp_maxid") - cpuid_to_pcpu = symval("cpuid_to_pcpu") - - cpu = 0 - while cpu <= mp_maxid: - pcpu = cpuid_to_pcpu[cpu] - if pcpu: - yield pcpu - cpu = cpu + 1 - +from freebsd import * +from pcpu import * class acttrace(gdb.Command): + """ + Register an acttrace command with gdb. + + When run, acttrace prints the stack trace of all threads that were on-CPU + at the time of the panic. + """ def __init__(self): super(acttrace, self).__init__("acttrace", gdb.COMMAND_USER) @@ -40,13 +25,13 @@ class acttrace(gdb.Command): # Save the current thread so that we can switch back after. curthread = gdb.selected_thread() - for pcpu in all_pcpus(): + for pcpu in pcpu_foreach(): td = pcpu['pc_curthread'] tid = td['td_tid'] gdb_thread = tid_to_gdb_thread(tid) if gdb_thread is None: - print("failed to find GDB thread with TID {}".format(tid)) + raise gdb.error(f"failed to find GDB thread with TID {tid}") else: gdb_thread.switch() diff --git a/sys/tools/gdb/freebsd.py b/sys/tools/gdb/freebsd.py new file mode 100644 index 000000000000..81ea60373348 --- /dev/null +++ b/sys/tools/gdb/freebsd.py @@ -0,0 +1,75 @@ +# +# Copyright (c) 2025 Mark Johnston <markj@FreeBSD.org> +# +# SPDX-License-Identifier: BSD-2-Clause +# + +import gdb + +def symval(name): + sym = gdb.lookup_global_symbol(name) + if sym is None: + sym = gdb.lookup_static_symbol(name) + if sym is None: + raise gdb.GdbError(f"Symbol '{name}' not found") + return sym.value() + + +def _queue_foreach(head, field, headf, nextf): + elm = head[headf] + while elm != 0: + yield elm + elm = elm[field][nextf] + + +def list_foreach(head, field): + """sys/queue.h-style iterator.""" + return _queue_foreach(head, field, "lh_first", "le_next") + + +def tailq_foreach(head, field): + """sys/queue.h-style iterator.""" + return _queue_foreach(head, field, "tqh_first", "tqe_next") + + +def linker_file_foreach(): + """Iterate over loaded linker files.""" + return tailq_foreach(symval("linker_files"), "link") + + +def pcpu_foreach(): + mp_maxid = symval("mp_maxid") + cpuid_to_pcpu = symval("cpuid_to_pcpu") + + cpu = 0 + while cpu <= mp_maxid: + pcpu = cpuid_to_pcpu[cpu] + if pcpu: + yield pcpu + cpu = cpu + 1 + + +def tid_to_gdb_thread(tid): + """Convert a FreeBSD kernel thread ID to a gdb inferior thread.""" + for thread in gdb.inferiors()[0].threads(): + if thread.ptid[2] == tid: + return thread + else: + return None + + +def tdfind(tid, pid=-1): + """Convert a FreeBSD kernel thread ID to a struct thread pointer.""" + td = tdfind.cached_threads.get(int(tid)) + if td: + return td + + for p in list_foreach(symval("allproc"), "p_list"): + if pid != -1 and pid != p['p_pid']: + continue + for td in tailq_foreach(p['p_threads'], "td_plist"): + ntid = td['td_tid'] + tdfind.cached_threads[int(ntid)] = td + if ntid == tid: + return td +tdfind.cached_threads = dict() diff --git a/sys/tools/gdb/pcpu.py b/sys/tools/gdb/pcpu.py new file mode 100644 index 000000000000..aadc4b2d42df --- /dev/null +++ b/sys/tools/gdb/pcpu.py @@ -0,0 +1,77 @@ +# +# Copyright (c) 2025 Mark Johnston <markj@FreeBSD.org> +# +# SPDX-License-Identifier: BSD-2-Clause +# + +import gdb +from freebsd import * + +class pcpu(gdb.Function): + """ + Register a function to lookup PCPU and DPCPU variables by name. + + To look up the value of the PCPU field foo on CPU n, use + $PCPU("foo", n). This works for DPCPU fields too. If the CPU ID is + omitted, and the currently selected thread is on-CPU, that CPU is + used, otherwise an error is raised. + """ + def __init__(self): + super(pcpu, self).__init__("PCPU") + + def invoke(self, field, cpuid=-1): + if cpuid == -1: + cpuid = tdfind(gdb.selected_thread().ptid[2])['td_oncpu'] + if cpuid == -1: + raise gdb.error("Currently selected thread is off-CPU") + if cpuid < 0 or cpuid > symval("mp_maxid"): + raise gdb.error(f"Currently selected on invalid CPU {cpuid}") + pcpu = symval("cpuid_to_pcpu")[cpuid] + + # Are we dealing with a PCPU or DPCPU field? + field = field.string() + for f in gdb.lookup_type("struct pcpu").fields(): + if f.name == "pc_" + field: + return pcpu["pc_" + field] + + def uintptr_t(val): + return val.cast(gdb.lookup_type("uintptr_t")) + + # We're dealing with a DPCPU field. This is handled similarly + # to VNET symbols, see vnet.py for comments. + pcpu_base = pcpu['pc_dynamic'] + pcpu_entry = symval("pcpu_entry_" + field) + pcpu_entry_addr = uintptr_t(pcpu_entry.address) + + for lf in linker_file_foreach(): + block = gdb.block_for_pc(lf['ops']['cls']['methods'][0]['func']) + elf_file_t = gdb.lookup_type("elf_file_t", block).target() + ef = lf.cast(elf_file_t) + + file_type = lf['ops']['cls']['name'].string() + if file_type == "elf64": + start = uintptr_t(ef['pcpu_start']) + if start == 0: + continue + end = uintptr_t(ef['pcpu_stop']) + base = uintptr_t(ef['pcpu_base']) + elif file_type == "elf64_obj": + for i in range(ef['nprogtab']): + pe = ef['progtab'][i] + if pe['name'].string() == "set_pcpu": + start = uintptr_t(pe['origaddr']) + end = start + uintptr_t(pe['size']) + base = uintptr_t(pe['addr']) + break + else: + continue + else: + path = lf['pathname'].string() + raise gdb.error(f"{path} has unexpected linker file type {file_type}") + + if pcpu_entry_addr >= start and pcpu_entry_addr < end: + obj = gdb.Value(pcpu_base + pcpu_entry_addr - start + base) + return obj.cast(pcpu_entry.type.pointer()).dereference() + +# Register with gdb. +pcpu() diff --git a/sys/tools/gdb/selftest.py b/sys/tools/gdb/selftest.py new file mode 100644 index 000000000000..41e9211c4bb3 --- /dev/null +++ b/sys/tools/gdb/selftest.py @@ -0,0 +1,31 @@ +# +# Copyright (c) 2025 Mark Johnston <markj@FreeBSD.org> +# +# SPDX-License-Identifier: BSD-2-Clause +# + +import gdb + +cmds = ["acttrace", + "p $V(\"tcbinfo\")", + "p $V(\"tcbinfo\", vnet0)", + "p $V(\"pf_status\")", + "p $V(\"pf_status\", \"gdbselftest\")", + "p $PCPU(\"curthread\")", + "p $PCPU(\"curthread\", 0)", + "p/x $PCPU(\"hardclocktime\", 1)", + "p $PCPU(\"pqbatch\")[0][0]", + "p $PCPU(\"ss\", 1)", + ] + +for cmd in cmds: + try: + print(f"Running command: '{cmd}'") + gdb.execute(cmd) + except gdb.error as e: + print(f"Command '{cmd}' failed: {e}") + break + +# We didn't hit any unexpected errors. This isn't as good as actually +# verifying the output, but it's better than nothing. +print("Everything seems OK") diff --git a/sys/tools/gdb/selftest.sh b/sys/tools/gdb/selftest.sh new file mode 100644 index 000000000000..252fae14af17 --- /dev/null +++ b/sys/tools/gdb/selftest.sh @@ -0,0 +1,23 @@ +# +# Copyright (c) 2025 Mark Johnston <markj@FreeBSD.org> +# +# SPDX-License-Identifier: BSD-2-Clause +# + +set -e + +n=$(sysctl -n hw.ncpu) +if [ $n -lt 2 ]; then + echo "This test requires at least 2 CPUs" + exit 1 +fi + +# Set up some things expected by selftest.py. +kldload -n pf siftr +pfctl -e || true +jail -c name=gdbselftest vnet persist + +echo "I'm about to panic your system, ctrl-C now if that's not what you want." +sleep 10 +sysctl debug.debugger_on_panic=0 +sysctl debug.kdb.panic=1 diff --git a/sys/tools/gdb/vnet.py b/sys/tools/gdb/vnet.py new file mode 100644 index 000000000000..36b4d512a3eb --- /dev/null +++ b/sys/tools/gdb/vnet.py @@ -0,0 +1,100 @@ +# +# Copyright (c) 2025 Mark Johnston <markj@FreeBSD.org> +# +# SPDX-License-Identifier: BSD-2-Clause +# + +import gdb +import traceback +from freebsd import * + +class vnet(gdb.Function): + """ + Register a function to look up VNET variables by name. + + To look at the value of a VNET variable V_foo, print $V("foo"). The + currently selected thread's VNET is used by default, but can be optionally + specified as a second parameter, e.g., $V("foo", <vnet>), where <vnet> is a + pointer to a struct vnet (e.g., vnet0 or allprison.tqh_first->pr_vnet) or a + string naming a jail. + """ + def __init__(self): + super(vnet, self).__init__("V") + + def invoke(self, sym, vnet=None): + sym = sym.string() + if sym.startswith("V_"): + sym = sym[len("V_"):] + if gdb.lookup_symbol("sysctl___kern_features_vimage")[0] is None: + return symval(sym) + + # Look up the VNET's base address. + if vnet is None: + vnet = tdfind(gdb.selected_thread().ptid[2])['td_vnet'] + if not vnet: + # If curthread->td_vnet == NULL, vnet0 is the current vnet. + vnet = symval("vnet0") + elif vnet.type.is_string_like: + vnet = vnet.string() + for prison in tailq_foreach(symval("allprison"), "pr_list"): + if prison['pr_name'].string() == vnet: + vnet = prison['pr_vnet'] + break + else: + raise gdb.error(f"No prison named {vnet}") + + def uintptr_t(val): + return val.cast(gdb.lookup_type("uintptr_t")) + + # Now the tricky part: compute the address of the symbol relative + # to the selected VNET. In the compiled kernel this is done at + # load time by applying a magic transformation to relocations + # against symbols in the vnet linker set. Here we have to apply + # the transformation manually. + vnet_data_base = vnet['vnet_data_base'] + vnet_entry = symval("vnet_entry_" + sym) + vnet_entry_addr = uintptr_t(vnet_entry.address) + + # First, which kernel module does the symbol belong to? + for lf in linker_file_foreach(): + # Find the bounds of this linker file's VNET linker set. The + # struct containing the bounds depends on the type of the linker + # file, and unfortunately both are called elf_file_t. So we use a + # PC value from the compilation unit (either link_elf.c or + # link_elf_obj.c) to disambiguate. + block = gdb.block_for_pc(lf['ops']['cls']['methods'][0]['func']) + elf_file_t = gdb.lookup_type("elf_file_t", block).target() + ef = lf.cast(elf_file_t) + + file_type = lf['ops']['cls']['name'].string() + if file_type == "elf64": + start = uintptr_t(ef['vnet_start']) + if start == 0: + # This linker file doesn't have a VNET linker set. + continue + end = uintptr_t(ef['vnet_stop']) + base = uintptr_t(ef['vnet_base']) + elif file_type == "elf64_obj": + for i in range(ef['nprogtab']): + pe = ef['progtab'][i] + if pe['name'].string() == "set_vnet": + start = uintptr_t(pe['origaddr']) + end = start + uintptr_t(pe['size']) + base = uintptr_t(pe['addr']) + break + else: + # This linker file doesn't have a VNET linker set. + continue + else: + path = lf['pathname'].string() + raise gdb.error(f"{path} has unexpected linker file type {file_type}") + + if vnet_entry_addr >= start and vnet_entry_addr < end: + # The symbol belongs to this linker file, so compute the final + # address. + obj = gdb.Value(vnet_data_base + vnet_entry_addr - start + base) + return obj.cast(vnet_entry.type.pointer()).dereference() + + +# Register with gdb. +vnet() diff --git a/sys/tools/kernel-gdb.py b/sys/tools/kernel-gdb.py new file mode 100644 index 000000000000..8a41ef6efab1 --- /dev/null +++ b/sys/tools/kernel-gdb.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2025 Mark Johnston <markj@FreeBSD.org> +# +# SPDX-License-Identifier: BSD-2-Clause +# + +import os +import sys + +sys.path.append(os.path.join(os.path.dirname(__file__), "gdb")) + +# Import FreeBSD kernel debugging commands and modules below. +import acttrace +import pcpu +import vnet diff --git a/usr.sbin/crashinfo/crashinfo.sh b/usr.sbin/crashinfo/crashinfo.sh index 3bb1e1456462..68115f09f9d4 100755 --- a/usr.sbin/crashinfo/crashinfo.sh +++ b/usr.sbin/crashinfo/crashinfo.sh @@ -217,10 +217,7 @@ echo file=`mktemp /tmp/crashinfo.XXXXXX` if [ $? -eq 0 ]; then - scriptdir=/usr/libexec/kgdb - echo "bt -full" >> $file - echo "source ${scriptdir}/acttrace.py" >> $file echo "acttrace" >> $file echo "quit" >> $file ${GDB%gdb}kgdb -q $KERNEL $VMCORE < $file
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202510031432.593EWOIA097989>