From owner-svn-src-head@freebsd.org Thu May 3 21:37:47 2018 Return-Path: Delivered-To: svn-src-head@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 78FC2FBBDBB; Thu, 3 May 2018 21:37:47 +0000 (UTC) (envelope-from kib@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 2EABB6A8D4; Thu, 3 May 2018 21:37:47 +0000 (UTC) (envelope-from kib@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 2978925779; Thu, 3 May 2018 21:37:47 +0000 (UTC) (envelope-from kib@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id w43LblSZ030899; Thu, 3 May 2018 21:37:47 GMT (envelope-from kib@FreeBSD.org) Received: (from kib@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id w43LbkOT030891; Thu, 3 May 2018 21:37:46 GMT (envelope-from kib@FreeBSD.org) Message-Id: <201805032137.w43LbkOT030891@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: kib set sender to kib@FreeBSD.org using -f From: Konstantin Belousov Date: Thu, 3 May 2018 21:37:46 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r333228 - in head/sys: amd64/amd64 i386/i386 kern sys X-SVN-Group: head X-SVN-Commit-Author: kib X-SVN-Commit-Paths: in head/sys: amd64/amd64 i386/i386 kern sys X-SVN-Commit-Revision: 333228 X-SVN-Commit-Repository: base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-head@freebsd.org X-Mailman-Version: 2.1.25 Precedence: list List-Id: SVN commit messages for the src tree for head/-current List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 03 May 2018 21:37:47 -0000 Author: kib Date: Thu May 3 21:37:46 2018 New Revision: 333228 URL: https://svnweb.freebsd.org/changeset/base/333228 Log: Implement support for ifuncs in the kernel linker. Required MD bits are only provided for x86. Reviewed by: jhb (previous version, as part of the larger patch) Sponsored by: The FreeBSD Foundation MFC after: 2 weeks Differential revision: https://reviews.freebsd.org/D13838 Modified: head/sys/amd64/amd64/elf_machdep.c head/sys/amd64/amd64/machdep.c head/sys/i386/i386/elf_machdep.c head/sys/i386/i386/machdep.c head/sys/kern/link_elf.c head/sys/kern/link_elf_obj.c head/sys/sys/linker.h Modified: head/sys/amd64/amd64/elf_machdep.c ============================================================================== --- head/sys/amd64/amd64/elf_machdep.c Thu May 3 20:43:39 2018 (r333227) +++ head/sys/amd64/amd64/elf_machdep.c Thu May 3 21:37:46 2018 (r333228) @@ -175,10 +175,13 @@ elf64_dump_thread(struct thread *td, void *dst, size_t *off = len; } +#define ERI_LOCAL 0x0001 +#define ERI_ONLYIFUNC 0x0002 + /* Process one elf relocation with addend. */ static int elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data, - int type, int local, elf_lookup_fn lookup) + int type, elf_lookup_fn lookup, int flags) { Elf64_Addr *where, val; Elf32_Addr *where32, val32; @@ -218,6 +221,9 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbas panic("unknown reloc type %d\n", type); } + if (((flags & ERI_ONLYIFUNC) == 0) ^ (rtype != R_X86_64_IRELATIVE)) + return (0); + switch (rtype) { case R_X86_64_NONE: /* none */ break; @@ -278,6 +284,13 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbas *where = val; break; + case R_X86_64_IRELATIVE: + addr = relocbase + addend; + val = ((Elf64_Addr (*)(void))addr)(); + if (*where != val) + *where = val; + break; + default: printf("kldload: unexpected relocation type %ld\n", rtype); @@ -287,11 +300,20 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbas } int +elf_reloc_ifunc(linker_file_t lf, Elf_Addr relocbase, const void *data, + int type, elf_lookup_fn lookup) +{ + + return (elf_reloc_internal(lf, relocbase, data, type, lookup, + ERI_ONLYIFUNC)); +} + +int elf_reloc(linker_file_t lf, Elf_Addr relocbase, const void *data, int type, elf_lookup_fn lookup) { - return (elf_reloc_internal(lf, relocbase, data, type, 0, lookup)); + return (elf_reloc_internal(lf, relocbase, data, type, lookup, 0)); } int @@ -299,7 +321,8 @@ elf_reloc_local(linker_file_t lf, Elf_Addr relocbase, int type, elf_lookup_fn lookup) { - return (elf_reloc_internal(lf, relocbase, data, type, 1, lookup)); + return (elf_reloc_internal(lf, relocbase, data, type, lookup, + ERI_LOCAL)); } int Modified: head/sys/amd64/amd64/machdep.c ============================================================================== --- head/sys/amd64/amd64/machdep.c Thu May 3 20:43:39 2018 (r333227) +++ head/sys/amd64/amd64/machdep.c Thu May 3 21:37:46 2018 (r333228) @@ -1566,6 +1566,8 @@ hammer_time(u_int64_t modulep, u_int64_t physfree) identify_cpu1(); identify_hypervisor(); + /* link_elf_ireloc(kmdp); */ + /* Init basic tunables, hz etc */ init_param1(); @@ -1744,6 +1746,7 @@ hammer_time(u_int64_t modulep, u_int64_t physfree) cninit(); amd64_kdb_init(); } + link_elf_ireloc(kmdp); getmemsize(kmdp, physfree); init_param2(physmem); Modified: head/sys/i386/i386/elf_machdep.c ============================================================================== --- head/sys/i386/i386/elf_machdep.c Thu May 3 20:43:39 2018 (r333227) +++ head/sys/i386/i386/elf_machdep.c Thu May 3 21:37:46 2018 (r333228) @@ -159,10 +159,13 @@ elf32_dump_thread(struct thread *td, void *dst, size_t *off = len; } +#define ERI_LOCAL 0x0001 +#define ERI_ONLYIFUNC 0x0002 + /* Process one elf relocation with addend. */ static int elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data, - int type, int local, elf_lookup_fn lookup) + int type, elf_lookup_fn lookup, int flags) { Elf_Addr *where; Elf_Addr addr; @@ -191,7 +194,10 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbas panic("unknown reloc type %d\n", type); } - if (local) { + if (((flags & ERI_ONLYIFUNC) == 0) ^ (rtype != R_386_IRELATIVE)) + return (0); + + if ((flags & ERI_LOCAL) != 0) { if (rtype == R_386_RELATIVE) { /* A + B */ addr = elf_relocaddr(lf, relocbase + addend); if (*where != addr) @@ -243,6 +249,12 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbas case R_386_RELATIVE: break; + case R_386_IRELATIVE: + addr = relocbase + addend; + addr = ((Elf_Addr (*)(void))addr)(); + if (*where != addr) + *where = addr; + break; default: printf("kldload: unexpected relocation type %d\n", rtype); @@ -252,11 +264,20 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbas } int +elf_reloc_ifunc(linker_file_t lf, Elf_Addr relocbase, const void *data, + int type, elf_lookup_fn lookup) +{ + + return (elf_reloc_internal(lf, relocbase, data, type, lookup, + ERI_ONLYIFUNC)); +} + +int elf_reloc(linker_file_t lf, Elf_Addr relocbase, const void *data, int type, elf_lookup_fn lookup) { - return (elf_reloc_internal(lf, relocbase, data, type, 0, lookup)); + return (elf_reloc_internal(lf, relocbase, data, type, lookup, 0)); } int @@ -264,7 +285,8 @@ elf_reloc_local(linker_file_t lf, Elf_Addr relocbase, int type, elf_lookup_fn lookup) { - return (elf_reloc_internal(lf, relocbase, data, type, 1, lookup)); + return (elf_reloc_internal(lf, relocbase, data, type, lookup, + ERI_LOCAL)); } int Modified: head/sys/i386/i386/machdep.c ============================================================================== --- head/sys/i386/i386/machdep.c Thu May 3 20:43:39 2018 (r333227) +++ head/sys/i386/i386/machdep.c Thu May 3 21:37:46 2018 (r333228) @@ -2302,6 +2302,7 @@ init386(int first) int gsel_tss, metadata_missing, x, pa; struct pcpu *pc; struct xstate_hdr *xhdr; + caddr_t kmdp; vm_offset_t addend; int late_console; @@ -2440,6 +2441,9 @@ init386(int first) cninit(); i386_kdb_init(); } + + kmdp = preload_search_by_type("elf kernel"); + link_elf_ireloc(kmdp); vm86_initialize(); getmemsize(first); Modified: head/sys/kern/link_elf.c ============================================================================== --- head/sys/kern/link_elf.c Thu May 3 20:43:39 2018 (r333227) +++ head/sys/kern/link_elf.c Thu May 3 21:37:46 2018 (r333228) @@ -190,6 +190,9 @@ static struct linker_class link_elf_class = { static int parse_dynamic(elf_file_t); static int relocate_file(elf_file_t); +static int relocate_file1(elf_file_t ef, int (*elf_reloc_func)( + linker_file_t lf, Elf_Addr relocbase, const void *data, + int type, elf_lookup_fn lookup)); static int link_elf_preload_parse_symbols(elf_file_t); static struct elf_set_head set_pcpu_list; @@ -1182,7 +1185,8 @@ symbol_name(elf_file_t ef, Elf_Size r_info) } static int -relocate_file(elf_file_t ef) +relocate_file1(elf_file_t ef, int (*elf_reloc_func)(linker_file_t lf, + Elf_Addr relocbase, const void *data, int type, elf_lookup_fn lookup)) { const Elf_Rel *rellim; const Elf_Rel *rel; @@ -1196,7 +1200,7 @@ relocate_file(elf_file_t ef) rellim = (const Elf_Rel *) ((const char *)ef->rel + ef->relsize); while (rel < rellim) { - if (elf_reloc(&ef->lf, (Elf_Addr)ef->address, rel, + if (elf_reloc_func(&ef->lf, (Elf_Addr)ef->address, rel, ELF_RELOC_REL, elf_lookup)) { symname = symbol_name(ef, rel->r_info); printf("link_elf: symbol %s undefined\n", symname); @@ -1212,7 +1216,7 @@ relocate_file(elf_file_t ef) relalim = (const Elf_Rela *) ((const char *)ef->rela + ef->relasize); while (rela < relalim) { - if (elf_reloc(&ef->lf, (Elf_Addr)ef->address, rela, + if (elf_reloc_func(&ef->lf, (Elf_Addr)ef->address, rela, ELF_RELOC_RELA, elf_lookup)) { symname = symbol_name(ef, rela->r_info); printf("link_elf: symbol %s undefined\n", @@ -1229,7 +1233,7 @@ relocate_file(elf_file_t ef) rellim = (const Elf_Rel *) ((const char *)ef->pltrel + ef->pltrelsize); while (rel < rellim) { - if (elf_reloc(&ef->lf, (Elf_Addr)ef->address, rel, + if (elf_reloc_func(&ef->lf, (Elf_Addr)ef->address, rel, ELF_RELOC_REL, elf_lookup)) { symname = symbol_name(ef, rel->r_info); printf("link_elf: symbol %s undefined\n", @@ -1246,7 +1250,7 @@ relocate_file(elf_file_t ef) relalim = (const Elf_Rela *) ((const char *)ef->pltrela + ef->pltrelasize); while (rela < relalim) { - if (elf_reloc(&ef->lf, (Elf_Addr)ef->address, rela, + if (elf_reloc_func(&ef->lf, (Elf_Addr)ef->address, rela, ELF_RELOC_RELA, elf_lookup)) { symname = symbol_name(ef, rela->r_info); printf("link_elf: symbol %s undefined\n", @@ -1260,6 +1264,19 @@ relocate_file(elf_file_t ef) return (0); } +static int +relocate_file(elf_file_t ef) +{ + int e; + + e = relocate_file1(ef, elf_reloc); +#if defined(__i386__) || defined(__amd64__) + if (e == 0) + e = relocate_file1(ef, elf_reloc_ifunc); +#endif + return (e); +} + /* * Hash function for symbol table lookup. Don't even think about changing * this. It is specified by the System V ABI. @@ -1317,7 +1334,8 @@ link_elf_lookup_symbol(linker_file_t lf, const char* n if (strcmp(name, strp) == 0) { if (symp->st_shndx != SHN_UNDEF || (symp->st_value != 0 && - ELF_ST_TYPE(symp->st_info) == STT_FUNC)) { + (ELF_ST_TYPE(symp->st_info) == STT_FUNC || + ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC))) { *sym = (c_linker_sym_t) symp; return (0); } @@ -1337,7 +1355,8 @@ link_elf_lookup_symbol(linker_file_t lf, const char* n if (strcmp(name, strp) == 0) { if (symp->st_shndx != SHN_UNDEF || (symp->st_value != 0 && - ELF_ST_TYPE(symp->st_info) == STT_FUNC)) { + (ELF_ST_TYPE(symp->st_info) == STT_FUNC || + ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC))) { *sym = (c_linker_sym_t) symp; return (0); } @@ -1352,12 +1371,18 @@ static int link_elf_symbol_values(linker_file_t lf, c_linker_sym_t sym, linker_symval_t *symval) { - elf_file_t ef = (elf_file_t) lf; - const Elf_Sym* es = (const Elf_Sym*) sym; + elf_file_t ef; + const Elf_Sym *es; + caddr_t val; + ef = (elf_file_t)lf; + es = (const Elf_Sym *)sym; if (es >= ef->symtab && es < (ef->symtab + ef->nchains)) { symval->name = ef->strtab + es->st_name; - symval->value = (caddr_t) ef->address + es->st_value; + val = (caddr_t)ef->address + es->st_value; + if (ELF_ST_TYPE(es->st_info) == STT_GNU_IFUNC) + val = ((caddr_t (*)(void))val)(); + symval->value = val; symval->size = es->st_size; return (0); } @@ -1365,7 +1390,10 @@ link_elf_symbol_values(linker_file_t lf, c_linker_sym_ return (ENOENT); if (es >= ef->ddbsymtab && es < (ef->ddbsymtab + ef->ddbsymcnt)) { symval->name = ef->ddbstrtab + es->st_name; - symval->value = (caddr_t) ef->address + es->st_value; + val = (caddr_t)ef->address + es->st_value; + if (ELF_ST_TYPE(es->st_info) == STT_GNU_IFUNC) + val = ((caddr_t (*)(void))val)(); + symval->value = val; symval->size = es->st_size; return (0); } @@ -1475,7 +1503,8 @@ link_elf_each_function_name(linker_file_t file, /* Exhaustive search */ for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) { if (symp->st_value != 0 && - ELF_ST_TYPE(symp->st_info) == STT_FUNC) { + (ELF_ST_TYPE(symp->st_info) == STT_FUNC || + ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC)) { error = callback(ef->ddbstrtab + symp->st_name, opaque); if (error != 0) return (error); @@ -1496,7 +1525,8 @@ link_elf_each_function_nameval(linker_file_t file, /* Exhaustive search */ for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) { if (symp->st_value != 0 && - ELF_ST_TYPE(symp->st_info) == STT_FUNC) { + (ELF_ST_TYPE(symp->st_info) == STT_FUNC || + ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC)) { error = link_elf_symbol_values(file, (c_linker_sym_t) symp, &symval); if (error != 0) @@ -1655,3 +1685,21 @@ link_elf_strtab_get(linker_file_t lf, caddr_t *strtab) return (ef->ddbstrcnt); } + +#if defined(__i386__) || defined(__amd64__) +void +link_elf_ireloc(caddr_t kmdp) +{ + struct elf_file eff; + elf_file_t ef; + + ef = &eff; + bzero(ef, sizeof(*ef)); + ef->modptr = kmdp; + ef->dynamic = (Elf_Dyn *)&_DYNAMIC; + parse_dynamic(ef); + ef->address = 0; + link_elf_preload_parse_symbols(ef); + relocate_file1(ef, elf_reloc_ifunc); +} +#endif Modified: head/sys/kern/link_elf_obj.c ============================================================================== --- head/sys/kern/link_elf_obj.c Thu May 3 20:43:39 2018 (r333227) +++ head/sys/kern/link_elf_obj.c Thu May 3 21:37:46 2018 (r333228) @@ -1201,12 +1201,19 @@ static int link_elf_symbol_values(linker_file_t lf, c_linker_sym_t sym, linker_symval_t *symval) { - elf_file_t ef = (elf_file_t) lf; - const Elf_Sym *es = (const Elf_Sym*) sym; + elf_file_t ef; + const Elf_Sym *es; + caddr_t val; + ef = (elf_file_t) lf; + es = (const Elf_Sym*) sym; + val = (caddr_t)es->st_value; if (es >= ef->ddbsymtab && es < (ef->ddbsymtab + ef->ddbsymcnt)) { symval->name = ef->ddbstrtab + es->st_name; - symval->value = (caddr_t)es->st_value; + val = (caddr_t)es->st_value; + if (ELF_ST_TYPE(es->st_info) == STT_GNU_IFUNC) + val = ((caddr_t (*)(void))val)(); + symval->value = val; symval->size = es->st_size; return 0; } @@ -1291,7 +1298,8 @@ link_elf_each_function_name(linker_file_t file, /* Exhaustive search */ for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) { if (symp->st_value != 0 && - ELF_ST_TYPE(symp->st_info) == STT_FUNC) { + (ELF_ST_TYPE(symp->st_info) == STT_FUNC || + ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC)) { error = callback(ef->ddbstrtab + symp->st_name, opaque); if (error) return (error); @@ -1312,8 +1320,10 @@ link_elf_each_function_nameval(linker_file_t file, /* Exhaustive search */ for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) { if (symp->st_value != 0 && - ELF_ST_TYPE(symp->st_info) == STT_FUNC) { - error = link_elf_symbol_values(file, (c_linker_sym_t) symp, &symval); + (ELF_ST_TYPE(symp->st_info) == STT_FUNC || + ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC)) { + error = link_elf_symbol_values(file, + (c_linker_sym_t)symp, &symval); if (error) return (error); error = callback(file, i, &symval, opaque); Modified: head/sys/sys/linker.h ============================================================================== --- head/sys/sys/linker.h Thu May 3 20:43:39 2018 (r333227) +++ head/sys/sys/linker.h Thu May 3 21:37:46 2018 (r333228) @@ -272,11 +272,16 @@ extern int kld_debug; typedef int elf_lookup_fn(linker_file_t, Elf_Size, int, Elf_Addr *); /* Support functions */ -int elf_reloc(linker_file_t _lf, Elf_Addr base, const void *_rel, int _type, elf_lookup_fn _lu); -int elf_reloc_local(linker_file_t _lf, Elf_Addr base, const void *_rel, int _type, elf_lookup_fn _lu); +int elf_reloc(linker_file_t _lf, Elf_Addr base, const void *_rel, + int _type, elf_lookup_fn _lu); +int elf_reloc_ifunc(linker_file_t _lf, Elf_Addr base, const void *_rel, + int _type, elf_lookup_fn _lu); +int elf_reloc_local(linker_file_t _lf, Elf_Addr base, const void *_rel, + int _type, elf_lookup_fn _lu); Elf_Addr elf_relocaddr(linker_file_t _lf, Elf_Addr addr); const Elf_Sym *elf_get_sym(linker_file_t _lf, Elf_Size _symidx); const char *elf_get_symname(linker_file_t _lf, Elf_Size _symidx); +void link_elf_ireloc(caddr_t kmdp); typedef struct linker_ctf { const uint8_t *ctftab; /* Decompressed CTF data. */