Date: Wed, 13 May 2015 16:31:21 +0000 From: "zbb (Zbigniew Bodek)" <phabric-noreply@FreeBSD.org> To: freebsd-arm@freebsd.org Subject: [Differential] [Updated, 1, 030 lines] D2377: Introduce ARM GICv3 support Message-ID: <aa86d2b102d65d0b7ec96760f1e6406e@localhost.localdomain> In-Reply-To: <differential-rev-PHID-DREV-piql4a7htwxjub743777-req@FreeBSD.org>
index | next in thread | previous in thread | raw e-mail
[-- Attachment #1 --] zbb updated this revision to Diff 5366. This revision now requires review to proceed. CHANGES SINCE LAST UPDATE https://reviews.freebsd.org/D2377?vs=5364&id=5366 REVISION DETAIL https://reviews.freebsd.org/D2377 AFFECTED FILES sys/arm64/arm64/gic_v3.c sys/arm64/arm64/gic_v3_fdt.c sys/arm64/arm64/gic_v3_reg.h sys/arm64/arm64/gic_v3_var.h sys/arm64/arm64/locore.S sys/arm64/include/armreg.h sys/conf/files.arm64 EMAIL PREFERENCES https://reviews.freebsd.org/settings/panel/emailpreferences/ To: zbb, ian, andrew, emaste, imp, brueffer, joel, wblock Cc: kostikbel, emaste, andrew, imp, freebsd-arm [-- Attachment #2 --] diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64 --- a/sys/conf/files.arm64 +++ b/sys/conf/files.arm64 @@ -19,6 +19,8 @@ arm64/arm64/elf_machdep.c standard arm64/arm64/exception.S standard arm64/arm64/gic.c standard +arm64/arm64/gic_v3.c standard +arm64/arm64/gic_v3_fdt.c optional fdt arm64/arm64/identcpu.c standard arm64/arm64/intr_machdep.c standard arm64/arm64/in_cksum.c optional inet | inet6 diff --git a/sys/arm64/include/armreg.h b/sys/arm64/include/armreg.h --- a/sys/arm64/include/armreg.h +++ b/sys/arm64/include/armreg.h @@ -96,6 +96,24 @@ #define EXCP_WATCHPT_EL1 0x35 /* Watchpoint, from same EL */ #define EXCP_BRK 0x3c /* Breakpoint */ +/* ICC_CTLR_EL1 */ +#define ICC_CTLR_EL1_EOIMODE (1U << 1) + +/* ICC_IAR1_EL1 */ +#define ICC_IAR1_EL1_SPUR (0x03ff) + +/* ICC_IGRPEN0_EL1 */ +#define ICC_IGRPEN0_EL1_EN (1U << 0) + +/* ICC_PMR_EL1 */ +#define ICC_PMR_EL1_PRIO_MASK (0xFFUL) + +/* ICC_SRE_EL1 */ +#define ICC_SRE_EL1_SRE (1U << 0) + +/* ICC_SRE_EL2 */ +#define ICC_SRE_EL2_EN (1U << 3) + /* ID_AA64PFR0_EL1 */ #define ID_AA64PFR0_EL0_MASK (0xf << 0) #define ID_AA64PFR0_EL1_MASK (0xf << 4) @@ -105,7 +123,10 @@ #define ID_AA64PFR0_FP_IMPL (0x0 << 16) /* Floating-point implemented */ #define ID_AA64PFR0_FP_NONE (0xf << 16) /* Floating-point not implemented */ #define ID_AA64PFR0_ADV_SIMD_MASK (0xf << 20) -#define ID_AA64PFR0_GIC_MASK (0xf << 24) +#define ID_AA64PFR0_GIC_SHIFT (24) +#define ID_AA64PFR0_GIC_BITS (0x4) /* Number of bits in GIC field */ +#define ID_AA64PFR0_GIC_MASK (0xf << ID_AA64PFR0_GIC_SHIFT) +#define ID_AA64PFR0_GIC_CPUIF_EN (0x1 << ID_AA64PFR0_GIC_SHIFT) /* MAIR_EL1 - Memory Attribute Indirection Register */ #define MAIR_ATTR_MASK(idx) (0xff << ((n)* 8)) diff --git a/sys/arm64/arm64/locore.S b/sys/arm64/arm64/locore.S --- a/sys/arm64/arm64/locore.S +++ b/sys/arm64/arm64/locore.S @@ -187,6 +187,20 @@ mov x2, #(PSR_F | PSR_I | PSR_A | PSR_D | PSR_M_EL1h) msr spsr_el2, x2 + /* Configure GICv3 CPU interface */ + mrs x2, id_aa64pfr0_el1 + /* Extract GIC bits from the register */ + ubfx x2, x2, #ID_AA64PFR0_GIC_SHIFT, #ID_AA64PFR0_GIC_BITS + /* GIC[3:0] == 0001 - GIC CPU interface via special regs. supported */ + cmp x2, #(ID_AA64PFR0_GIC_CPUIF_EN >> ID_AA64PFR0_GIC_SHIFT) + b.ne 2f + + mrs x2, icc_sre_el2 + orr x2, x2, #ICC_SRE_EL2_EN /* Enable access from insecure EL1 */ + msr icc_sre_el2, x2 + isb +2: + /* Set the address to return to our return address */ msr elr_el2, x30 diff --git a/sys/arm64/arm64/gic_v3_var.h b/sys/arm64/arm64/gic_v3_var.h new file mode 100644 --- /dev/null +++ b/sys/arm64/arm64/gic_v3_var.h @@ -0,0 +1,104 @@ +/*- + * Copyright (c) 2015 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Semihalf under + * the sponsorship of the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _GIC_V3_VAR_H_ +#define _GIC_V3_VAR_H_ + +#define GIC_V3_DEVSTR "ARM Generic Interrupt Controller v3.0" + +DECLARE_CLASS(gic_v3_driver); + +struct gic_redists { + /* + * Re-Distributor region description. + * We will have few of those depending + * on the #redistributor-regions property in FDT. + */ + struct resource ** regions; + /* Number of Re-Distributor regions */ + u_int nregions; + /* Per-CPU Re-Distributor handler */ + struct resource * pcpu[MAXCPU]; +}; + +struct gic_v3_softc { + device_t dev; + struct resource ** gic_res; + struct mtx gic_mtx; + /* Distributor */ + struct resource * gic_dist; + /* Re-Distributors */ + struct gic_redists gic_redists; + + u_int gic_nirqs; + u_int gic_idbits; + + boolean_t gic_registered; +}; + +MALLOC_DECLARE(M_GIC_V3); + +/* Device methods */ +int gic_v3_attach(device_t dev); +int gic_v3_detach(device_t dev); + +/* + * GIC Distributor accessors. + * Notice that only GIC sofc can be passed. + */ +#define gic_d_read(sc, len, reg) \ +({ \ + bus_read_##len(sc->gic_dist, reg); \ +}) + +#define gic_d_write(sc, len, reg, val) \ +({ \ + bus_write_##len(sc->gic_dist, reg, val);\ +}) + +/* GIC Re-Distributor accessors (per-CPU) */ +#define gic_r_read(sc, len, reg) \ +({ \ + u_int cpu = PCPU_GET(cpuid); \ + \ + bus_read_##len( \ + sc->gic_redists.pcpu[cpu], \ + reg); \ +}) + +#define gic_r_write(sc, len, reg, val) \ +({ \ + u_int cpu = PCPU_GET(cpuid); \ + \ + bus_write_##len( \ + sc->gic_redists.pcpu[cpu], \ + reg, val); \ +}) + +#endif /* _GIC_V3_VAR_H_ */ diff --git a/sys/arm64/arm64/gic_v3_reg.h b/sys/arm64/arm64/gic_v3_reg.h new file mode 100644 --- /dev/null +++ b/sys/arm64/arm64/gic_v3_reg.h @@ -0,0 +1,183 @@ +/*- + * Copyright (c) 2015 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Semihalf under + * the sponsorship of the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _GIC_V3_REG_H_ +#define _GIC_V3_REG_H_ + +/* + * Maximum number of interrupts + * supported by GIC (including SGIs, PPIs and SPIs) + */ +#define GIC_I_NUM_MAX (1020) +/* + * Priority MAX/MIN values + */ +#define GIC_PRIORITY_MAX (0x00UL) +/* Upper value is determined by LPI max priority */ +#define GIC_PRIORITY_MIN (0xFCUL) + +/* Numbers for software generated interrupts */ +#define GIC_FIRST_SGI (0) +#define GIC_LAST_SGI (15) +/* Numbers for private peripheral interrupts */ +#define GIC_FIRST_PPI (16) +#define GIC_LAST_PPI (31) +/* Numbers for spared peripheral interrupts */ +#define GIC_FIRST_SPI (32) +#define GIC_LAST_SPI (1019) +/* Numbers for local peripheral interrupts */ +#define GIC_FIRST_LPI (8192) + +/* + * Registers (v2/v3) + */ +#define GICD_CTLR (0x0000) +#define GICD_CTLR_G1 (1 << 0) +#define GICD_CTLR_G1A (1 << 1) +#define GICD_CTLR_ARE_NS (1 << 4) +#define GICD_CTLR_RWP (1 << 31) + +#define GICD_TYPER (0x0004) +#define GICD_TYPER_IDBITS(n) ((((n) >> 19) & 0x1F) + 1) +#define GICD_TYPER_I_NUM(n) ((((n) & 0xF1) + 1) * 32) + +#define GICD_ISENABLER(n) (0x0100 + (((n) >> 5) * 4)) +#define GICD_I_PER_ISENABLERn (32) + +#define GICD_ICENABLER(n) (0x0180 + (((n) >> 5) * 4)) +#define GICD_IPRIORITYR(n) (0x0400 + (((n) >> 2) * 4)) +#define GICD_I_PER_IPRIORITYn (4) + +#define GICD_I_MASK(n) (1 << ((n) % 32)) + +#define GICD_ICFGR(n) (0x0C00 + (((n) >> 4) * 4)) +/* First bit is a polarity bit (0 - low, 1 - high) */ +#define GICD_ICFGR_POL_LOW (0 << 0) +#define GICD_ICFGR_POL_HIGH (1 << 0) +#define GICD_ICFGR_POL_MASK (0x1) +/* Second bit is a trigger bit (0 - level, 1 - edge) */ +#define GICD_ICFGR_TRIG_LVL (0 << 1) +#define GICD_ICFGR_TRIG_EDGE (1 << 1) +#define GICD_ICFGR_TRIG_MASK (0x2) + +#define GICD_I_PER_ICFGRn (16) + +/* + * Registers (v3) + */ +#define GICD_IROUTER(n) (0x6000 + ((n) * 8)) +#define GICD_PIDR2 (0xFFE8) + +#define GICR_PIDR2_ARCH_MASK (0xF0) +#define GICR_PIDR2_ARCH_GICv3 (0x30) +#define GICR_PIDR2_ARCH_GICv4 (0x40) + +/* Redistributor registers */ +#define GICR_PIDR2 GICD_PIDR2 + +#define GICR_TYPER (0x0008) +#define GICR_TYPER_VLPIS (1 << 1) +#define GICR_TYPER_LAST (1 << 4) +#define GICR_TYPER_AFF_SHIFT (32) + +#define GICR_WAKER (0x0014) +#define GICR_WAKER_PS (1 << 1) /* Processor sleep */ +#define GICR_WAKER_CA (1 << 2) /* Children asleep */ + +/* Re-distributor registers for SGIs and PPIs */ +#define GICR_RD_BASE_SIZE PAGE_SIZE_64K +#define GICR_SGI_BASE_SIZE PAGE_SIZE_64K +#define GICR_VLPI_BASE_SIZE PAGE_SIZE_64K +#define GICR_RESERVED_SIZE PAGE_SIZE_64K + +#define GICR_ISENABLER0 (0x0100) +#define GICR_ICENABLER0 (0x0180) +#define GICR_I_ENABLER_SGI_MASK (0x0000FFFF) +#define GICR_I_ENABLER_PPI_MASK (0xFFFF0000) + +#define GICR_I_PER_IPRIORITYn (GICD_I_PER_IPRIORITYn) + +/* + * CPU interface + */ + +/* + * Registers list (ICC_xyz_EL1): + * + * PMR - Priority Mask Register + * * interrupts of priority higher than specified + * in this mask will be signalled to the CPU. + * (0xff - lowest possible prio., 0x00 - highest prio.) + * + * CTLR - Control Register + * * controls behavior of the CPU interface and displays + * implemented features. + * + * IGRPEN1 - Interrupt Group 1 Enable Register + * + * IAR1 - Interrupt Acknowledge Register Group 1 + * * contains number of the highest priority pending + * interrupt from the Group 1. + * + * EOIR1 - End of Interrupt Register Group 1 + * * Writes inform CPU interface about completed Group 1 + * interrupts processing. + */ + +#define gic_icc_write(reg, val) \ +do { \ + WRITE_SPECIALREG(ICC_ ##reg ##_EL1, val); \ + isb(); \ +} while (0) + +#define gic_icc_read(reg) \ +({ \ + uint64_t val; \ + \ + val = READ_SPECIALREG(ICC_ ##reg ##_EL1); \ + (val); \ +}) + +#define gic_icc_set(reg, mask) \ +do { \ + uint64_t val; \ + val = gic_icc_read(reg); \ + val |= (mask); \ + gic_icc_write(reg, val); \ +} while (0) + +#define gic_icc_clear(reg, mask) \ +do { \ + uint64_t val; \ + val = gic_icc_read(reg); \ + val &= ~(mask); \ + gic_icc_write(reg, val); \ +} while (0) + +#endif /* _GIC_V3_REG_H_ */ diff --git a/sys/arm64/arm64/gic_v3_fdt.c b/sys/arm64/arm64/gic_v3_fdt.c new file mode 100644 --- /dev/null +++ b/sys/arm64/arm64/gic_v3_fdt.c @@ -0,0 +1,124 @@ +/*- + * Copyright (c) 2015 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Semihalf under + * the sponsorship of the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/bus.h> +#include <sys/kernel.h> +#include <sys/module.h> + +#include <dev/fdt/fdt_common.h> +#include <dev/ofw/openfirm.h> +#include <dev/ofw/ofw_bus.h> +#include <dev/ofw/ofw_bus_subr.h> + +#include "pic_if.h" + +#include "gic_v3_reg.h" +#include "gic_v3_var.h" + +/* + * FDT glue. + */ +static int gic_v3_fdt_probe(device_t); +static int gic_v3_fdt_attach(device_t); + +static device_method_t gic_v3_fdt_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, gic_v3_fdt_probe), + DEVMETHOD(device_attach, gic_v3_fdt_attach), + + /* End */ + DEVMETHOD_END +}; + +DEFINE_CLASS_1(gic_v3, gic_v3_fdt_driver, gic_v3_fdt_methods, + sizeof(struct gic_v3_softc), gic_v3_driver); + +static devclass_t gic_v3_fdt_devclass; + +EARLY_DRIVER_MODULE(gic_v3, simplebus, gic_v3_fdt_driver, gic_v3_fdt_devclass, + 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); +EARLY_DRIVER_MODULE(gic_v3, ofwbus, gic_v3_fdt_driver, gic_v3_fdt_devclass, + 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); + +/* + * Device interface. + */ +static int +gic_v3_fdt_probe(device_t dev) +{ + + if (!ofw_bus_status_okay(dev)) + return (ENXIO); + + if (!ofw_bus_is_compatible(dev, "arm,gic-v3")) + return (ENXIO); + + device_set_desc(dev, GIC_V3_DEVSTR); + return (BUS_PROBE_DEFAULT); +} + +static int +gic_v3_fdt_attach(device_t dev) +{ + struct gic_v3_softc *sc; + pcell_t redist_regions; + int err; + + sc = device_get_softc(dev); + sc->dev = dev; + + /* + * Recover number of the Re-Distributor regions. + */ + if (OF_getencprop(ofw_bus_get_node(dev), "#redistributor-regions", + &redist_regions, sizeof(redist_regions)) <= 0) + sc->gic_redists.nregions = 1; + else + sc->gic_redists.nregions = redist_regions; + + err = gic_v3_attach(dev); + if (err) + goto error; + + return (err); + +error: + if (bootverbose) { + device_printf(dev, + "Failed to attach. Error %d\n", err); + } + /* Failure so free resources */ + gic_v3_detach(dev); + + return (err); +} diff --git a/sys/arm64/arm64/gic_v3.c b/sys/arm64/arm64/gic_v3.c new file mode 100644 --- /dev/null +++ b/sys/arm64/arm64/gic_v3.c @@ -0,0 +1,580 @@ +/*- + * Copyright (c) 2015 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Semihalf under + * the sponsorship of the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/bus.h> +#include <sys/kernel.h> +#include <sys/ktr.h> +#include <sys/malloc.h> +#include <sys/module.h> +#include <sys/rman.h> +#include <sys/pcpu.h> +#include <sys/proc.h> +#include <sys/cpuset.h> +#include <sys/lock.h> +#include <sys/mutex.h> + +#include <vm/vm.h> +#include <vm/pmap.h> + +#include <machine/bus.h> +#include <machine/intr.h> + +#include "pic_if.h" + +#include "gic_v3_reg.h" +#include "gic_v3_var.h" + +/* Device and PIC methods */ +static void gic_v3_dispatch(device_t, struct trapframe *); +static void gic_v3_eoi(device_t, u_int); +static void gic_v3_mask_irq(device_t, u_int); +static void gic_v3_unmask_irq(device_t, u_int); + +static device_method_t gic_v3_methods[] = { + /* Device interface */ + DEVMETHOD(device_detach, gic_v3_detach), + + /* PIC interface */ + DEVMETHOD(pic_dispatch, gic_v3_dispatch), + DEVMETHOD(pic_eoi, gic_v3_eoi), + DEVMETHOD(pic_mask, gic_v3_mask_irq), + DEVMETHOD(pic_unmask, gic_v3_unmask_irq), + + /* End */ + DEVMETHOD_END +}; + +DEFINE_CLASS_0(gic_v3, gic_v3_driver, gic_v3_methods, + sizeof(struct gic_v3_softc)); + +/* + * Driver-specific definitions. + */ +MALLOC_DEFINE(M_GIC_V3, "GICv3", GIC_V3_DEVSTR); + +/* + * Helper functions and definitions. + */ +/* Destination registers, either Distributor or Re-Distributor */ +enum gic_v3_xdist { + DIST = 0, + REDIST, +}; + +/* Helper routines starting with gic_v3_ */ +static int gic_v3_dist_init(struct gic_v3_softc *); +static int gic_v3_redist_find(struct gic_v3_softc *); +static int gic_v3_redist_init(struct gic_v3_softc *); +static int gic_v3_cpu_init(struct gic_v3_softc *); +static void gic_v3_wait_for_rwp(struct gic_v3_softc *, enum gic_v3_xdist); + +/* A sequence of init functions for primary (boot) CPU */ +typedef int (*gic_v3_initseq_t) (struct gic_v3_softc *); +/* Primary CPU initialization sequence */ +static gic_v3_initseq_t gic_v3_primary_init[] = { + gic_v3_dist_init, + gic_v3_redist_init, + gic_v3_cpu_init, + NULL +}; + +/* + * Device interface. + */ +int +gic_v3_attach(device_t dev) +{ + struct gic_v3_softc *sc; + gic_v3_initseq_t *init_func; + uint32_t typer; + int rid; + int err; + size_t i; + + sc = device_get_softc(dev); + sc->gic_registered = FALSE; + sc->dev = dev; + err = 0; + + /* Initialize mutex */ + mtx_init(&sc->gic_mtx, "GICv3 lock", NULL, MTX_SPIN); + + /* + * Allocate array of struct resource. + * One entry for Distributor and all remaining for Re-Distributor. + */ + sc->gic_res = malloc( + sizeof(sc->gic_res) * (sc->gic_redists.nregions + 1), + M_GIC_V3, M_WAITOK); + + /* Now allocate corresponding resources */ + for (i = 0, rid = 0; i < (sc->gic_redists.nregions + 1); i++, rid++) { + sc->gic_res[rid] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, + &rid, RF_ACTIVE); + if (sc->gic_res[rid] == NULL) + return (ENXIO); + } + + /* + * Distributor interface + */ + sc->gic_dist = sc->gic_res[0]; + + /* + * Re-Dristributor interface + */ + /* Allocate space under region descriptions */ + sc->gic_redists.regions = malloc( + sizeof(*sc->gic_redists.regions) * sc->gic_redists.nregions, + M_GIC_V3, M_WAITOK); + + /* Fill-up bus_space information for each region. */ + for (i = 0, rid = 1; i < sc->gic_redists.nregions; i++, rid++) + sc->gic_redists.regions[i] = sc->gic_res[rid]; + + /* Get the number of supported SPI interrupts */ + typer = gic_d_read(sc, 4, GICD_TYPER); + sc->gic_nirqs = GICD_TYPER_I_NUM(typer); + if (sc->gic_nirqs > GIC_I_NUM_MAX) + sc->gic_nirqs = GIC_I_NUM_MAX; + + /* Get the number of supported interrupt identifier bits */ + sc->gic_idbits = GICD_TYPER_IDBITS(typer); + + if (bootverbose) { + device_printf(dev, "SPIs: %u, IDs: %u\n", + sc->gic_nirqs, (1 << sc->gic_idbits) - 1); + } + + /* Train init sequence for boot CPU */ + for (init_func = gic_v3_primary_init; *init_func != NULL; init_func++) { + err = (*init_func)(sc); + if (err != 0) + return (err); + } + /* + * Full success. + * Now register PIC to the interrupts handling layer. + */ + arm_register_root_pic(dev, sc->gic_nirqs); + sc->gic_registered = TRUE; + + return (0); +} + +int +gic_v3_detach(device_t dev) +{ + struct gic_v3_softc *sc; + size_t i; + int rid; + + sc = device_get_softc(dev); + + if (device_is_attached(dev)) { + /* + * XXX: We should probably deregister PIC + */ + if (sc->gic_registered) + panic("Trying to detach registered PIC"); + } + for (rid = 0; rid < (sc->gic_redists.nregions + 1); rid++) + bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->gic_res[rid]); + + for (i = 0; i < MAXCPU; i++) + free(sc->gic_redists.pcpu[i], M_GIC_V3); + + free(sc->gic_res, M_GIC_V3); + free(sc->gic_redists.regions, M_GIC_V3); + + return (0); +} + +/* + * PIC interface. + */ +static void +gic_v3_dispatch(device_t dev, struct trapframe *frame) +{ + uint64_t active_irq; + + while (1) { + active_irq = gic_icc_read(IAR1); + + if (__predict_false(active_irq == ICC_IAR1_EL1_SPUR)) + break; + + if (__predict_true((active_irq >= GIC_FIRST_PPI && + active_irq <= GIC_LAST_SPI))) { + arm_dispatch_intr(active_irq, frame); + continue; + } + + if (active_irq <= GIC_LAST_SGI || active_irq >= GIC_FIRST_LPI) { + /* + * TODO: Implement proper SGI/LPI handling. + * Mask it if such is received for some reason. + */ + device_printf(dev, + "Received unsupported interrupt type: %s\n", + active_irq >= GIC_FIRST_LPI ? "LPI" : "SGI"); + PIC_MASK(dev, active_irq); + } + } +} + +static void +gic_v3_eoi(device_t dev, u_int irq) +{ + + gic_icc_write(EOIR1, (uint64_t)irq); +} + +static void +gic_v3_mask_irq(device_t dev, u_int irq) +{ + struct gic_v3_softc *sc; + + sc = device_get_softc(dev); + + if (irq >= GIC_FIRST_PPI && irq <= GIC_LAST_PPI) { /* PPIs in corresponding Re-Distributor */ + gic_r_write(sc, 4, + GICR_SGI_BASE_SIZE + GICD_ICENABLER(irq), GICD_I_MASK(irq)); + gic_v3_wait_for_rwp(sc, REDIST); + } else if (irq >= GIC_FIRST_SPI && irq <= GIC_LAST_SPI) { /* SPIs in distributor */ + gic_r_write(sc, 4, GICD_ICENABLER(irq), GICD_I_MASK(irq)); + gic_v3_wait_for_rwp(sc, DIST); + } else + panic("%s: Unsupported IRQ number %u", __func__, irq); +} + +static void +gic_v3_unmask_irq(device_t dev, u_int irq) +{ + struct gic_v3_softc *sc; + + sc = device_get_softc(dev); + + if (irq >= GIC_FIRST_PPI && irq <= GIC_LAST_PPI) { /* PPIs in corresponding Re-Distributor */ + gic_r_write(sc, 4, + GICR_SGI_BASE_SIZE + GICD_ISENABLER(irq), GICD_I_MASK(irq)); + gic_v3_wait_for_rwp(sc, REDIST); + } else if (irq >= GIC_FIRST_SPI && irq <= GIC_LAST_SPI) { /* SPIs in distributor */ + gic_d_write(sc, 4, GICD_ISENABLER(irq), GICD_I_MASK(irq)); + gic_v3_wait_for_rwp(sc, DIST); + } else + panic("%s: Unsupported IRQ number %u", __func__, irq); +} + +/* + * Helper routines + */ +static void +gic_v3_wait_for_rwp(struct gic_v3_softc *sc, enum gic_v3_xdist xdist) +{ + struct resource *res; + u_int cpuid; + size_t us_left = 1000000; + + cpuid = PCPU_GET(cpuid); + + switch (xdist) { + case DIST: + res = sc->gic_dist; + break; + case REDIST: + res = sc->gic_redists.pcpu[cpuid]; + break; + default: + KASSERT(0, ("%s: Attempt to wait for unknown RWP", __func__)); + return; + } + + while ((bus_read_4(res, GICD_CTLR) & GICD_CTLR_RWP) != 0) { + DELAY(1); + if (us_left-- == 0) + panic("GICD Register write pending for too long"); + } +} + +/* CPU interface. */ +static __inline void +gic_v3_cpu_priority(uint64_t mask) +{ + + /* Set prority mask */ + gic_icc_write(PMR, mask & ICC_PMR_EL1_PRIO_MASK); +} + +static int +gic_v3_cpu_enable_sre(struct gic_v3_softc *sc) +{ + uint64_t sre; + u_int cpuid; + + cpuid = PCPU_GET(cpuid); + /* + * Set the SRE bit to enable access to GIC CPU interface + * via system registers. + */ + sre = READ_SPECIALREG(icc_sre_el1); + sre |= ICC_SRE_EL1_SRE; + WRITE_SPECIALREG(icc_sre_el1, sre); + isb(); + /* + * Now ensure that the bit is set. + */ + sre = READ_SPECIALREG(icc_sre_el1); + if ((sre & ICC_SRE_EL1_SRE) == 0) { + /* We are done. This was disabled in EL2 */ + device_printf(sc->dev, "ERROR: CPU%u cannot enable CPU interface " + "via system registers\n", cpuid); + return (ENXIO); + } else if (bootverbose) { + device_printf(sc->dev, + "CPU%u enabled CPU interface via system registers\n", + cpuid); + } + + return (0); +} + +static int +gic_v3_cpu_init(struct gic_v3_softc *sc) +{ + int err; + + /* Enable access to CPU interface via system registers */ + err = gic_v3_cpu_enable_sre(sc); + if (err != 0) + return (err); + /* Priority mask to minimum - accept all interrupts */ + gic_v3_cpu_priority(GIC_PRIORITY_MIN); + /* Disable EOI mode */ + gic_icc_clear(CTLR, ICC_CTLR_EL1_EOIMODE); + /* Enable group 1 (insecure) interrups */ + gic_icc_set(IGRPEN1, ICC_IGRPEN0_EL1_EN); + + return (0); +} + +/* Distributor */ +static int +gic_v3_dist_init(struct gic_v3_softc *sc) +{ + uint64_t aff; + u_int i; + + /* + * 1. Disable the Distributor + */ + gic_d_write(sc, 4, GICD_CTLR, 0); + gic_v3_wait_for_rwp(sc, DIST); + + /* + * 2. Configure the Distributor + */ + /* Set all global interrupts to be level triggered, active low. */ + for (i = GIC_FIRST_SPI; i < sc->gic_nirqs; i += GICD_I_PER_ICFGRn) + gic_d_write(sc, 4, GICD_ICFGR(i), 0x00000000); + + /* Set priority to all shared interrupts */ + for (i = GIC_FIRST_SPI; + i < sc->gic_nirqs; i += GICD_I_PER_IPRIORITYn) { + /* Set highest priority */ + gic_d_write(sc, 4, GICD_IPRIORITYR(i), GIC_PRIORITY_MAX); + } + + /* + * Disable all interrupts. Leave PPI and SGIs as they are enabled in + * Re-Distributor registers. + */ + for (i = GIC_FIRST_SPI; i < sc->gic_nirqs; i += GICD_I_PER_ISENABLERn) + gic_d_write(sc, 4, GICD_ICENABLER(i), 0xFFFFFFFF); + + gic_v3_wait_for_rwp(sc, DIST); + + /* + * 3. Enable Distributor + */ + /* Enable Distributor with ARE, Group 1 */ + gic_d_write(sc, 4, GICD_CTLR, GICD_CTLR_ARE_NS | GICD_CTLR_G1A | + GICD_CTLR_G1); + + /* + * 4. Route all interrupts to boot CPU. + */ + aff = CPU_AFFINITY(PCPU_GET(cpuid)); + for (i = GIC_FIRST_SPI; i < sc->gic_nirqs; i++) + gic_d_write(sc, 4, GICD_IROUTER(i), aff); + + return (0); +} + +/* Re-Distributor */ +static int +gic_v3_redist_find(struct gic_v3_softc *sc) +{ + struct resource r_res; + bus_space_handle_t r_bsh; + uint64_t aff; + uint64_t typer; + uint32_t pidr2; + u_int cpuid; + size_t i; + + cpuid = PCPU_GET(cpuid); + + /* Allocate struct resource for this CPU's Re-Distributor registers */ + sc->gic_redists.pcpu[cpuid] = + malloc(sizeof(*sc->gic_redists.pcpu[0]), M_GIC_V3, M_WAITOK); + + aff = CPU_AFFINITY(cpuid); + /* Affinity in format for comparison with typer */ + aff = (CPU_AFF3(aff) << 24) | (CPU_AFF2(aff) << 16) | + (CPU_AFF1(aff) << 8) | CPU_AFF0(aff); + + if (bootverbose) { + device_printf(sc->dev, + "Start searching for Re-Distributor\n"); + } + /* Iterate through Re-Distributor regions */ + for (i = 0; i < sc->gic_redists.nregions; i++) { + /* Take a copy of the region's resource */ + r_res = *sc->gic_redists.regions[i]; + r_bsh = rman_get_bushandle(&r_res); + + pidr2 = bus_read_4(&r_res, GICR_PIDR2); + switch (pidr2 & GICR_PIDR2_ARCH_MASK) { + case GICR_PIDR2_ARCH_GICv3: /* fall through */ + case GICR_PIDR2_ARCH_GICv4: + break; + default: + device_printf(sc->dev, + "No Re-Distributor found for CPU%u\n", cpuid); + free(sc->gic_redists.pcpu[cpuid], M_GIC_V3); + return (ENODEV); + } + + do { + typer = bus_read_8(&r_res, GICR_TYPER); + if ((typer >> GICR_TYPER_AFF_SHIFT) == aff) { + KASSERT(sc->gic_redists.pcpu[cpuid] != NULL, + ("Invalid pointer to per-CPU redistributor")); + /* Copy res contents to its final destination */ + *sc->gic_redists.pcpu[cpuid] = r_res; + if (bootverbose) { + device_printf(sc->dev, + "CPU%u Re-Distributor has been found\n", + cpuid); + } + return (0); + } + + r_bsh += (GICR_RD_BASE_SIZE + GICR_SGI_BASE_SIZE); + if ((typer & GICR_TYPER_VLPIS) != 0) { + r_bsh += + (GICR_VLPI_BASE_SIZE + GICR_RESERVED_SIZE); + } + + rman_set_bushandle(&r_res, r_bsh); + } while ((typer & GICR_TYPER_LAST) == 0); + } + + free(sc->gic_redists.pcpu[cpuid], M_GIC_V3); + device_printf(sc->dev, "No Re-Distributor found for CPU%u\n", cpuid); + return (ENXIO); +} + +static int +gic_v3_redist_wake(struct gic_v3_softc *sc) +{ + uint32_t waker; + size_t us_left = 1000000; + + waker = gic_r_read(sc, 4, GICR_WAKER); + /* Wake up Re-Distributor for this CPU */ + waker &= ~GICR_WAKER_PS; + gic_r_write(sc, 4, GICR_WAKER, waker); + /* + * When clearing ProcessorSleep bit it is required to wait for + * ChildrenAsleep to become zero following the processor power-on. + */ + while ((gic_r_read(sc, 4, GICR_WAKER) & GICR_WAKER_CA) != 0) { + DELAY(1); + if (us_left-- == 0) { + panic("Could not wake Re-Distributor for CPU%u", + PCPU_GET(cpuid)); + } + } + + if (bootverbose) { + device_printf(sc->dev, "CPU%u Re-Distributor woke up\n", + PCPU_GET(cpuid)); + } + + return (0); +} + +static int +gic_v3_redist_init(struct gic_v3_softc *sc) +{ + int err; + size_t i; + + err = gic_v3_redist_find(sc); + if (err != 0) + return (err); + + err = gic_v3_redist_wake(sc); + if (err != 0) + return (err); + + /* Disable SPIs */ + gic_r_write(sc, 4, GICR_SGI_BASE_SIZE + GICR_ICENABLER0, + GICR_I_ENABLER_PPI_MASK); + /* Enable SGIs */ + gic_r_write(sc, 4, GICR_SGI_BASE_SIZE + GICR_ISENABLER0, + GICR_I_ENABLER_SGI_MASK); + + /* Set priority for SGIs and PPIs */ + for (i = 0; i <= GIC_LAST_PPI; i += GICR_I_PER_IPRIORITYn) { + gic_r_write(sc, 4, GICR_SGI_BASE_SIZE + GICD_IPRIORITYR(i), + GIC_PRIORITY_MAX); + } + + gic_v3_wait_for_rwp(sc, REDIST); + + return (0); +}help
Want to link to this message? Use this
URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?aa86d2b102d65d0b7ec96760f1e6406e>