Date: Mon, 27 Apr 2015 13:52:25 +0000 From: "zbb (Zbigniew Bodek)" <phabric-noreply@FreeBSD.org> To: freebsd-arm@freebsd.org Subject: [Differential] [Request, 2, 082 lines] D2378: Introduce ITS support for ARM64 Message-ID: <differential-rev-PHID-DREV-xmqmcfafyun443233vly-req@FreeBSD.org>
next in thread | raw e-mail | index | archive | help
[-- Attachment #1 --] zbb created this revision. zbb added reviewers: emaste, imp, ian, andrew. zbb added a subscriber: freebsd-arm. zbb set the repository for this revision to rS FreeBSD src repository. Herald added subscribers: imp, andrew, emaste. Herald added a reviewer: manpages. REVISION SUMMARY Add ARM ITS (Interrupt Translation Services) support required to bring-up message signaled interrupts on some ARM64 platforms. Obtained from: Semihalf Sponsored by: The FreeBSD Foundation REPOSITORY rS FreeBSD src repository REVISION DETAIL https://reviews.freebsd.org/D2378 AFFECTED FILES sys/arm64/arm64/gic_v3.c sys/arm64/arm64/gic_v3_fdt.c sys/arm64/arm64/gic_v3_its.c sys/arm64/arm64/gic_v3_reg.h sys/arm64/arm64/gic_v3_var.h EMAIL PREFERENCES https://reviews.freebsd.org/settings/panel/emailpreferences/ To: zbb, emaste, imp, ian, andrew, brueffer, joel, wblock Cc: emaste, andrew, imp, freebsd-arm [-- Attachment #2 --] diff --git a/sys/arm64/arm64/gic_v3_var.h b/sys/arm64/arm64/gic_v3_var.h --- a/sys/arm64/arm64/gic_v3_var.h +++ b/sys/arm64/arm64/gic_v3_var.h @@ -32,6 +32,17 @@ #define GIC_V3_DEVSTR "ARM Generic Interrupt Controller v3.0" +#define LPI_FLAGS_CONF_FLUSH (1UL << 0) +#define LPI_CONFTAB_SIZE PAGE_SIZE_64K +/* 1 bit per LPI + 1 KB more for the obligatory PPI, SGI, SPI stuff */ +#define LPI_PENDTAB_SIZE ((LPI_CONFTAB_SIZE / 8) + 0x400) + +struct redist_lpis { + vm_offset_t conf_base; + vm_offset_t pend_base[MAXCPU]; + uint64_t flags; +}; + struct gic_redists { /* * Re-Distributor region description. @@ -43,6 +54,8 @@ u_int nregions; /* Per-CPU Re-Distributor handler */ struct resource * pcpu[MAXCPU]; + /* LPIs data */ + struct redist_lpis lpis; }; struct gic_v3_softc { @@ -72,6 +85,166 @@ void gic_v3_unmask_irq(device_t, u_int); /* + * ITS + */ +#define GIC_V3_ITS_DEVSTR "ARM GIC Interrupt Translation Service" +#define GIC_V3_ITS_COMPSTR "arm,gic-v3-its" + +/* LPI chunk owned by ITS device */ +struct lpi_chunk { + u_int lpi_base; + u_int lpi_num; + u_int lpi_free; /* First free LPI in set */ +}; + +/* ITS device */ +struct its_dev { + TAILQ_ENTRY(its_dev) entry; + /* PCI device */ + device_t pci_dev; + /* Device ID (i.e. PCI device ID) */ + uint32_t devid; + /* List of assigned LPIs */ + struct lpi_chunk lpis; + /* Virtual address of ITT */ + vm_offset_t itt; + /* Interrupt collection */ + struct its_col * col; +}; +TAILQ_HEAD(its_dev_list, its_dev); + +/* ITS private table description */ +struct its_ptab { + vm_offset_t ptab_vaddr; /* Virtual Address of table */ + size_t ptab_pgsz; /* Page size */ + size_t ptab_npages; /* Number of pages */ +}; + +/* ITS collection description. */ +struct its_col { + uint64_t col_target; /* Target Re-Distributor */ + uint64_t col_id; /* Collection ID */ +}; + +/* ITS command. Each command is 32 bytes long */ +struct its_cmd { + uint64_t cmd_dword[4]; /* ITS command double word */ +}; + +/* ITS commands encoding */ +#define ITS_CMD_SYNC (0x05) +#define ITS_CMD_MAPD (0x08) +#define ITS_CMD_MAPC (0x09) +#define ITS_CMD_MAPVI (0x0a) +#define ITS_CMD_MAPI (0x0b) +#define ITS_CMD_INV (0x0c) +#define ITS_CMD_INVALL (0x0d) +/* Command */ +#define CMD_COMMAND_MASK (0xFFUL) +/* PCI device ID */ +#define CMD_DEVID_SHIFT (32) +#define CMD_DEVID_MASK (0xFFFFFFFFUL << CMD_DEVID_SHIFT) +/* Size of IRQ ID bitfield */ +#define CMD_SIZE_MASK (0xFFUL) +/* Virtual LPI ID */ +#define CMD_ID_MASK (0xFFFFFFFFUL) +/* Physical LPI ID */ +#define CMD_PID_SHIFT (32) +#define CMD_PID_MASK (0xFFFFFFFFUL << CMD_PID_SHIFT) +/* Collection */ +#define CMD_COL_MASK (0xFFFFUL) +/* Target (CPU or Re-Distributor) */ +#define CMD_TARGET_SHIFT (16) +#define CMD_TARGET_MASK (0xFFFFFFFFUL << CMD_TARGET_SHIFT) +/* Interrupt Translation Table address */ +#define CMD_ITT_MASK (0xFFFFFFFFFF00UL) +/* Valid command bit */ +#define CMD_VALID_SHIFT (63) +#define CMD_VALID_MASK (1UL << CMD_VALID_SHIFT) + +/* + * ITS command descriptor. + * Idea for command description passing taken from Linux. + */ +struct its_cmd_desc { + uint8_t cmd_type; + + union { + struct { + struct its_col *col; + } cmd_desc_sync; + + struct { + struct its_col *col; + uint8_t valid; + } cmd_desc_mapc; + + struct { + struct its_dev *its_dev; + uint32_t pid; + uint32_t id; + } cmd_desc_mapvi; + + struct { + struct its_dev *its_dev; + uint32_t lpinum; + } cmd_desc_mapi; + + struct { + struct its_dev *its_dev; + uint8_t valid; + } cmd_desc_mapd; + + struct { + struct its_dev *its_dev; + uint32_t lpinum; + } cmd_desc_inv; + + struct { + struct its_col *col; + } cmd_desc_invall; + }; +}; + +#define ITS_CMDQ_SIZE PAGE_SIZE_64K +#define ITS_CMDQ_NENTRIES (ITS_CMDQ_SIZE / sizeof(struct its_cmd)) + +#define ITS_FLAGS_CMDQ_FLUSH (1UL << 0) + +#define ITS_TARGET_NONE 0xFBADBEEF + +struct gic_v3_its_softc { + device_t dev; + struct resource * its_res; + + struct its_cmd * its_cmdq_base; /* ITS command queue base */ + struct its_cmd * its_cmdq_write; /* ITS command queue write ptr */ + struct its_ptab its_ptabs[GITS_BASER_NUM];/* ITS private tables */ + struct its_col * its_cols; /* Per-CPU collections */ + + uint64_t its_flags; + + struct its_dev_list its_dev_list; + + unsigned long * its_lpi_bitmap; + uint32_t its_lpi_maxid; + + struct mtx its_mtx; + struct mtx its_spin_mtx; +}; + +extern devclass_t gic_v3_its_devclass; + +int gic_v3_its_attach(device_t); +int gic_v3_its_detach(device_t); + +int gic_v3_its_alloc_msix(device_t, device_t, int *); +int gic_v3_its_alloc_msi(device_t, device_t, int, int *); +int gic_v3_its_map_msix(device_t, device_t, int, uint64_t *, uint32_t *); + +void lpi_unmask_irq(device_t, uint32_t); +void lpi_mask_irq(device_t, uint32_t); +/* * GIC Distributor accessors. * Notice that only GIC sofc can be passed. */ @@ -104,4 +277,28 @@ reg, val); \ }) +#define PCI_DEVID(pci_dev) \ +({ \ + (((pci_get_domain(pci_dev) >> 2) << 19) | \ + ((pci_get_domain(pci_dev) % 4) << 16) | \ + (pci_get_bus(pci_dev) << 8) | \ + (pci_get_slot(pci_dev) << 3) | \ + (pci_get_function(pci_dev) << 0)); \ +}) + +/* + * Request number of maximum MSI-X vectors for this device. + * Device can ask for less vectors than maximum supported but not more. + */ +#define PCI_MSIX_NUM(pci_dev) \ +({ \ + struct pci_devinfo *dinfo; \ + pcicfgregs *cfg; \ + \ + dinfo = device_get_ivars(pci_dev); \ + cfg = &dinfo->cfg; \ + \ + cfg->msix.msix_msgnum; \ +}) + #endif /* _GIC_V3_VAR_H_ */ diff --git a/sys/arm64/arm64/gic_v3_reg.h b/sys/arm64/arm64/gic_v3_reg.h --- a/sys/arm64/arm64/gic_v3_reg.h +++ b/sys/arm64/arm64/gic_v3_reg.h @@ -86,20 +86,248 @@ #define GICR_PIDR2_ARCH_GICv4 (0x40) /* Redistributor registers */ +#define GICR_CTLR GICD_CTLR +#define GICR_CTLR_LPI_ENABLE (1 << 0) + #define GICR_PIDR2 GICD_PIDR2 #define GICR_TYPER (0x0008) +#define GICR_TYPER_PLPIS (1 << 0) #define GICR_TYPER_VLPIS (1 << 1) #define GICR_TYPER_LAST (1 << 4) +#define GICR_TYPER_CPUNUM_SHIFT 8 +#define GICR_TYPER_CPUNUM_MASK (0xFFFUL << GICR_TYPER_CPUNUM_SHIFT) +#define GICR_TYPER_CPUNUM(x) \ + (((x) & GICR_TYPER_CPUNUM_MASK) >> GICR_TYPER_CPUNUM_SHIFT) #define GICR_WAKER (0x0014) #define GICR_WAKER_PS (1 << 1) /* Processor sleep */ #define GICR_WAKER_CA (1 << 2) /* Children asleep */ +#define GICR_PROPBASER (0x0070) +#define GICR_PROPBASER_IDBITS_MASK 0x1FUL +/* + * Cacheability + * 0x0 - Device-nGnRnE + * 0x1 - Normal Inner Non-cacheable + * 0x2 - Normal Inner Read-allocate, Write-through + * 0x3 - Normal Inner Read-allocate, Write-back + * 0x4 - Normal Inner Write-allocate, Write-through + * 0x5 - Normal Inner Write-allocate, Write-back + * 0x6 - Normal Inner Read-allocate, Write-allocate, Write-through + * 0x7 - Normal Inner Read-allocate, Write-allocate, Write-back + */ +#define GICR_PROPBASER_CACHE_SHIFT 7 +#define GICR_PROPBASER_CACHE_DnGnRnE 0x0UL +#define GICR_PROPBASER_CACHE_NIN 0x1UL +#define GICR_PROPBASER_CACHE_NIRAWT 0x2UL +#define GICR_PROPBASER_CACHE_NIRAWB 0x3UL +#define GICR_PROPBASER_CACHE_NIWAWT 0x4UL +#define GICR_PROPBASER_CACHE_NIWAWB 0x5UL +#define GICR_PROPBASER_CACHE_NIRAWAWT 0x6UL +#define GICR_PROPBASER_CACHE_NIRAWAWB 0x7UL + +/* + * Shareability + * 0x0 - Non-shareable + * 0x1 - Inner-shareable + * 0x2 - Outer-shareable + * 0x3 - Reserved. Threated as 0x0 + */ +#define GICR_PROPBASER_SHARE_SHIFT 10 +#define GICR_PROPBASER_SHARE_NS 0x0UL +#define GICR_PROPBASER_SHARE_IS 0x1UL +#define GICR_PROPBASER_SHARE_OS 0x2UL +#define GICR_PROPBASER_SHARE_RES 0x3UL +#define GICR_PROPBASER_SHARE_MASK \ + (0x3UL << GICR_PROPBASER_SHARE_SHIFT) + +#define GICR_PENDBASER (0x0078) +/* + * Cacheability + * 0x0 - Device-nGnRnE + * 0x1 - Normal Inner Non-cacheable + * 0x2 - Normal Inner Read-allocate, Write-through + * 0x3 - Normal Inner Read-allocate, Write-back + * 0x4 - Normal Inner Write-allocate, Write-through + * 0x5 - Normal Inner Write-allocate, Write-back + * 0x6 - Normal Inner Read-allocate, Write-allocate, Write-through + * 0x7 - Normal Inner Read-allocate, Write-allocate, Write-back + */ +#define GICR_PENDBASER_CACHE_SHIFT 7 +#define GICR_PENDBASER_CACHE_DnGnRnE 0x0UL +#define GICR_PENDBASER_CACHE_NIN 0x1UL +#define GICR_PENDBASER_CACHE_NIRAWT 0x2UL +#define GICR_PENDBASER_CACHE_NIRAWB 0x3UL +#define GICR_PENDBASER_CACHE_NIWAWT 0x4UL +#define GICR_PENDBASER_CACHE_NIWAWB 0x5UL +#define GICR_PENDBASER_CACHE_NIRAWAWT 0x6UL +#define GICR_PENDBASER_CACHE_NIRAWAWB 0x7UL + +/* + * Shareability + * 0x0 - Non-shareable + * 0x1 - Inner-shareable + * 0x2 - Outer-shareable + * 0x3 - Reserved. Threated as 0x0 + */ +#define GICR_PENDBASER_SHARE_SHIFT 10 +#define GICR_PENDBASER_SHARE_NS 0x0UL +#define GICR_PENDBASER_SHARE_IS 0x1UL +#define GICR_PENDBASER_SHARE_OS 0x2UL +#define GICR_PENDBASER_SHARE_RES 0x3UL +#define GICR_PENDBASER_SHARE_MASK \ + (0x3UL << GICR_PENDBASER_SHARE_SHIFT) + /* Re-distributor registers for SGIs and PPIs */ #define GICR_ISENABLER0 (0x0100) #define GICR_ICENABLER0 (0x0180) +/* ITS registers */ +#define GITS_PIDR2 GICR_PIDR2 +#define GITS_PIDR2_ARCH_MASK GICR_PIDR2_ARCH_MASK +#define GITS_PIDR2_ARCH_GICv3 GICR_PIDR2_ARCH_GICv3 +#define GITS_PIDR2_ARCH_GICv4 GICR_PIDR2_ARCH_GICv4 + +#define GITS_CTLR (0x0000) +#define GITS_CTLR_EN (1 << 0) + +#define GITS_CBASER (0x0080) +#define GITS_CBASER_VALID (1UL << 63) +/* + * Cacheability + * 0x0 - Device-nGnRnE + * 0x1 - Normal Inner Non-cacheable + * 0x2 - Normal Inner Read-allocate, Write-through + * 0x3 - Normal Inner Read-allocate, Write-back + * 0x4 - Normal Inner Write-allocate, Write-through + * 0x5 - Normal Inner Write-allocate, Write-back + * 0x6 - Normal Inner Read-allocate, Write-allocate, Write-through + * 0x7 - Normal Inner Read-allocate, Write-allocate, Write-back + */ +#define GITS_CBASER_CACHE_SHIFT 59 +#define GITS_CBASER_CACHE_DnGnRnE 0x0UL +#define GITS_CBASER_CACHE_NIN 0x1UL +#define GITS_CBASER_CACHE_NIRAWT 0x2UL +#define GITS_CBASER_CACHE_NIRAWB 0x3UL +#define GITS_CBASER_CACHE_NIWAWT 0x4UL +#define GITS_CBASER_CACHE_NIWAWB 0x5UL +#define GITS_CBASER_CACHE_NIRAWAWT 0x6UL +#define GITS_CBASER_CACHE_NIRAWAWB 0x7UL +#define GITS_CBASER_CACHE_MASK (0x7UL << GITS_CBASER_TYPE_SHIFT) +/* + * Shareability + * 0x0 - Non-shareable + * 0x1 - Inner-shareable + * 0x2 - Outer-shareable + * 0x3 - Reserved. Threated as 0x0 + */ +#define GITS_CBASER_SHARE_SHIFT 10 +#define GITS_CBASER_SHARE_NS 0x0UL +#define GITS_CBASER_SHARE_IS 0x1UL +#define GITS_CBASER_SHARE_OS 0x2UL +#define GITS_CBASER_SHARE_RES 0x3UL +#define GITS_CBASER_SHARE_MASK \ + (0x3UL << GITS_CBASER_SHARE_SHIFT) + +#define GITS_CBASER_PA_SHIFT 12 +#define GITS_CBASER_PA_MASK (0xFFFFFFFFFUL << GITS_CBASER_PA_SHIFT) + +#define GITS_CWRITER (0x0088) +#define GITS_CREADR (0x0090) + +#define GITS_BASER_BASE (0x0100) +#define GITS_BASER(x) (GITS_BASER_BASE + (x) * 8) + +#define GITS_BASER_VALID (1UL << 63) + +#define GITS_BASER_TYPE_SHIFT 56 +#define GITS_BASER_TYPE(x) \ + (((x) & GITS_BASER_TYPE_MASK) >> GITS_BASER_TYPE_SHIFT) +#define GITS_BASER_TYPE_UNIMPL 0x0UL /* Unimplemented */ +#define GITS_BASER_TYPE_DEV 0x1UL /* Devices */ +#define GITS_BASER_TYPE_VP 0x2UL /* Virtual Processors */ +#define GITS_BASER_TYPE_PP 0x3UL /* Physical Processors */ +#define GITS_BASER_TYPE_IC 0x4UL /* Interrupt Collections */ +#define GITS_BASER_TYPE_RES5 0x5UL /* Reserved */ +#define GITS_BASER_TYPE_RES6 0x6UL /* Reserved */ +#define GITS_BASER_TYPE_RES7 0x7UL /* Reserved */ +#define GITS_BASER_TYPE_MASK (0x7UL << GITS_BASER_TYPE_SHIFT) +/* + * Cacheability + * 0x0 - Non-cacheable, non-bufferable + * 0x1 - Non-cacheable + * 0x2 - Read-allocate, Write-through + * 0x3 - Read-allocate, Write-back + * 0x4 - Write-allocate, Write-through + * 0x5 - Write-allocate, Write-back + * 0x6 - Read-allocate, Write-allocate, Write-through + * 0x7 - Read-allocate, Write-allocate, Write-back + */ +#define GITS_BASER_CACHE_SHIFT 59 +#define GITS_BASER_CACHE_NCNB 0x0UL +#define GITS_BASER_CACHE_NC 0x1UL +#define GITS_BASER_CACHE_RAWT 0x2UL +#define GITS_BASER_CACHE_RAWB 0x3UL +#define GITS_BASER_CACHE_WAWT 0x4UL +#define GITS_BASER_CACHE_WAWB 0x5UL +#define GITS_BASER_CACHE_RAWAWT 0x6UL +#define GITS_BASER_CACHE_RAWAWB 0x7UL +#define GITS_BASER_CACHE_MASK (0x7UL << GITS_BASER_CACHE_SHIFT) + +#define GITS_BASER_ESIZE_SHIFT 48 +#define GITS_BASER_ESIZE_MASK (0x1FUL << GITS_BASER_ESIZE_SHIFT) +#define GITS_BASER_ESIZE(x) \ + ((((x) & GITS_BASER_ESIZE_MASK) >> GITS_BASER_ESIZE_SHIFT) + 1) + +#define GITS_BASER_PA_SHIFT 12 +#define GITS_BASER_PA_MASK (0xFFFFFFFFFUL << GITS_BASER_PA_SHIFT) + +/* + * Shareability + * 0x0 - Non-shareable + * 0x1 - Inner-shareable + * 0x2 - Outer-shareable + * 0x3 - Reserved. Threated as 0x0 + */ +#define GITS_BASER_SHARE_SHIFT 10 +#define GITS_BASER_SHARE_NS 0x0UL +#define GITS_BASER_SHARE_IS 0x1UL +#define GITS_BASER_SHARE_OS 0x2UL +#define GITS_BASER_SHARE_RES 0x3UL +#define GITS_BASER_SHARE_MASK (0x3UL << GITS_BASER_SHARE_SHIFT) + +#define GITS_BASER_PSZ_SHIFT 8 +#define GITS_BASER_PSZ_4K 0x0UL +#define GITS_BASER_PSZ_16K 0x1UL +#define GITS_BASER_PSZ_64K 0x2UL +#define GITS_BASER_PSZ_MASK (0x3UL << GITS_BASER_PSZ_SHIFT) + +#define GITS_BASER_SIZE_MASK 0xFFUL + +#define GITS_BASER_NUM 8 + +#define GITS_TYPER (0x0008) +#define GITS_TYPER_PTA (1UL << 19) +#define GITS_TYPER_DEVB_SHIFT 13 +#define GITS_TYPER_DEVB_MASK (0x1FUL << GITS_TYPER_DEVB_SHIFT) +/* Number of device identifiers implemented */ +#define GITS_TYPER_DEVB(x) \ + ((((x) & GITS_TYPER_DEVB_MASK) >> GITS_TYPER_DEVB_SHIFT) + 1) +#define GITS_TYPER_ITTES_SHIFT 4 +#define GITS_TYPER_ITTES_MASK (0xFUL << GITS_TYPER_ITTES_SHIFT) +/* Number of bytes per ITT Entry */ +#define GITS_TYPER_ITTES(x) \ + ((((x) & GITS_TYPER_ITTES_MASK) >> GITS_TYPER_ITTES_SHIFT) + 1) + +#define GITS_TRANSLATER (0x10040) +/* + * LPI related + */ +#define LPI_CONF_PRIO_MASK (0xFC) +#define LPI_CONF_GROUP1 (1 << 1) +#define LPI_CONF_ENABLE (1 << 0) + /* * CPU interface */ diff --git a/sys/arm64/arm64/gic_v3_its.c b/sys/arm64/arm64/gic_v3_its.c new file mode 100644 --- /dev/null +++ b/sys/arm64/arm64/gic_v3_its.c @@ -0,0 +1,1447 @@ +/*- + * Copyright (c) 2015 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Semihalf under + * the sponsorship of the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/bitset.h> +#include <sys/bitstring.h> +#include <sys/bus.h> +#include <sys/endian.h> +#include <sys/kernel.h> +#include <sys/ktr.h> +#include <sys/malloc.h> +#include <sys/module.h> +#include <sys/rman.h> +#include <sys/pciio.h> +#include <sys/pcpu.h> +#include <sys/proc.h> +#include <sys/cpuset.h> +#include <sys/lock.h> +#include <sys/mutex.h> + +#include <dev/pci/pcivar.h> + +#include <vm/vm.h> +#include <vm/pmap.h> + +#include <machine/bus.h> +#include <machine/cpu.h> +#include <machine/cpufunc.h> +#include <machine/intr.h> + +#include "gic_v3_reg.h" +#include "gic_v3_var.h" + +MALLOC_DEFINE(M_GIC_V3_ITS, "GICv3 ITS", GIC_V3_ITS_DEVSTR); + +devclass_t gic_v3_its_devclass; + +static int its_alloc_tables(struct gic_v3_its_softc *); +static void its_free_tables(struct gic_v3_its_softc *); +static int its_init_commandq(struct gic_v3_its_softc *); +static int its_init_cpu(struct gic_v3_its_softc *); +static int its_init_cpu_collection(struct gic_v3_its_softc *); + +static int its_cmd_send(struct gic_v3_its_softc *, struct its_cmd_desc *); + +static void its_cmd_mapc(struct gic_v3_its_softc *, struct its_col *, uint8_t); +static void its_cmd_mapvi(struct gic_v3_its_softc *, struct its_dev *, uint32_t, + uint32_t); +static void its_cmd_mapi(struct gic_v3_its_softc *, struct its_dev *, uint32_t); +static void its_cmd_inv(struct gic_v3_its_softc *, struct its_dev *, uint32_t); +static void its_cmd_invall(struct gic_v3_its_softc *, struct its_col *); + +static int lpi_init_conftable(struct gic_v3_its_softc *); +static int lpi_bitmap_init(struct gic_v3_its_softc *); +static int lpi_init_cpu(struct gic_v3_its_softc *); +static int lpi_config_cpu(struct gic_v3_its_softc *); + +const char *its_ptab_cache[] = { + [GITS_BASER_CACHE_NCNB] = "(NC,NB)", + [GITS_BASER_CACHE_NC] = "(NC)", + [GITS_BASER_CACHE_RAWT] = "(RA,WT)", + [GITS_BASER_CACHE_RAWB] = "(RA,WB)", + [GITS_BASER_CACHE_WAWT] = "(WA,WT)", + [GITS_BASER_CACHE_WAWB] = "(WA,WB)", + [GITS_BASER_CACHE_RAWAWT] = "(RAWA,WT)", + [GITS_BASER_CACHE_RAWAWB] = "(RAWA,WB)", +}; + +const char *its_ptab_share[] = { + [GITS_BASER_SHARE_NS] = "none", + [GITS_BASER_SHARE_IS] = "inner", + [GITS_BASER_SHARE_OS] = "outer", + [GITS_BASER_SHARE_RES] = "none", +}; + +const char *its_ptab_type[] = { + [GITS_BASER_TYPE_UNIMPL] = "Unimplemented", + [GITS_BASER_TYPE_DEV] = "Devices", + [GITS_BASER_TYPE_VP] = "Virtual Processors", + [GITS_BASER_TYPE_PP] = "Physical Processors", + [GITS_BASER_TYPE_IC] = "Interrupt Collections", + [GITS_BASER_TYPE_RES5] = "Reserved (5)", + [GITS_BASER_TYPE_RES6] = "Reserved (6)", + [GITS_BASER_TYPE_RES7] = "Reserved (7)", +}; + +static struct gic_v3_its_softc *its_sc; + +#define gic_its_read(sc, len, reg) \ +({ \ + bus_read_##len(&sc->its_res[0], \ + reg); \ +}) + +#define gic_its_write(sc, len, reg, val) \ +({ \ + bus_write_##len(&sc->its_res[0], \ + reg, val); \ +}) + +int +gic_v3_its_attach(device_t dev) +{ + struct gic_v3_its_softc *sc; + uint64_t gits_tmp; + uint32_t gits_pidr2; + int rid; + int ret; + + sc = device_get_softc(dev); + + /* + * Initialize sleep & spin mutex for ITS + */ + /* Protects ITS device list and assigned LPIs bitmaps. */ + mtx_init(&sc->its_mtx, "ITS sleep lock", NULL, MTX_DEF); + /* Protects access to ITS command circular buffer. */ + mtx_init(&sc->its_spin_mtx, "ITS spin lock", NULL, MTX_SPIN); + + rid = 0; + sc->its_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, + RF_ACTIVE); + if (sc->its_res == NULL) + return (ENXIO); + + sc->dev = dev; + + gits_pidr2 = gic_its_read(sc, 4, GITS_PIDR2); + switch (gits_pidr2 & GITS_PIDR2_ARCH_MASK) { + case GITS_PIDR2_ARCH_GICv3: /* fall through */ + case GITS_PIDR2_ARCH_GICv4: + if (bootverbose) { + device_printf(dev, "ITS found. Architecture rev. %u\n", + (u_int)(gits_pidr2 & GITS_PIDR2_ARCH_MASK) >> 4); + } + break; + default: + device_printf(dev, "No ITS found in the system\n"); + ret = ENODEV; + goto error; + } + + /* 1. Initialize commands queue */ + ret = its_init_commandq(sc); + if (ret) + goto error; + + /* 2. Provide memory for any private ITS tables */ + ret = its_alloc_tables(sc); + if (ret) + goto error; + + /* 3. Allocate collections. One per-CPU */ + sc->its_cols = malloc(sizeof(*sc->its_cols) * MAXCPU, + M_GIC_V3_ITS, M_WAITOK | M_ZERO); + + /* 4. Enable ITS in GITS_CTLR */ + gits_tmp = gic_its_read(sc, 4, GITS_CTLR); + gic_its_write(sc, 4, GITS_CTLR, gits_tmp | GITS_CTLR_EN); + + /* 5. Initialize LPIs configuration table */ + ret = lpi_init_conftable(sc); + if (ret) + goto error; + + /* 6. LPIs bitmap init */ + ret = lpi_bitmap_init(sc); + if (ret) + goto error; + + /* 7. CPU init */ + (void)its_init_cpu(sc); + + /* 8. Init ITS devices list */ + TAILQ_INIT(&sc->its_dev_list); + + arm_register_msi_pic(dev); + + /* + * XXX: We need to have ITS software context when. + * Being called by the interrupt code (mask/unmask). + * This may be used only when one ITS is present in + * the system and eventually should be removed. + */ + its_sc = sc; + + return (0); + +error: + gic_v3_its_detach(dev); + return (ret); +} + +/* Will not detach but use it for convenience */ +int +gic_v3_its_detach(device_t dev) +{ + device_t parent; + struct gic_v3_softc *gic_sc; + struct gic_v3_its_softc *sc; + u_int cpuid; + int rid = 0; + + sc = device_get_softc(dev); + cpuid = PCPU_GET(cpuid); + + /* Release what's possible */ + + /* Command queue */ + if ((void *)sc->its_cmdq_base != NULL) { + contigfree((void *)sc->its_cmdq_base, + ITS_CMDQ_SIZE, M_GIC_V3_ITS); + } + /* ITTs */ + its_free_tables(sc); + /* Collections */ + free(sc->its_cols, M_GIC_V3_ITS); + /* LPI config table */ + parent = device_get_parent(sc->dev); + gic_sc = device_get_softc(parent); + if ((void *)gic_sc->gic_redists.lpis.conf_base != NULL) { + contigfree((void *)gic_sc->gic_redists.lpis.conf_base, + LPI_CONFTAB_SIZE, M_GIC_V3_ITS); + } + if ((void *)gic_sc->gic_redists.lpis.pend_base[cpuid] != NULL) { + contigfree((void *)gic_sc->gic_redists.lpis.pend_base[cpuid], + roundup2(LPI_PENDTAB_SIZE, PAGE_SIZE_64K), M_GIC_V3_ITS); + } + + /* Resource... */ + bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->its_res); + + return (0); +} + +static int +its_alloc_tables(struct gic_v3_its_softc *sc) +{ + uint64_t gits_baser, gits_tmp; + uint64_t type, esize, cache, share, psz; + uint64_t gits_typer; + size_t page_size, npages, nitspages, nidents, tn; + size_t its_tbl_size; + vm_offset_t ptab_vaddr; + vm_paddr_t ptab_paddr; + boolean_t first = TRUE; + + page_size = PAGE_SIZE_64K; + + /* Read features first */ + gits_typer = gic_its_read(sc, 8, GITS_TYPER); + + for (tn = 0; tn < GITS_BASER_NUM; tn++) { + gits_baser = gic_its_read(sc, 8, GITS_BASER(tn)); + type = GITS_BASER_TYPE(gits_baser); + /* Get the Table Entry size */ + esize = GITS_BASER_ESIZE(gits_baser); + + switch (type) { + case GITS_BASER_TYPE_UNIMPL: /* fall through */ + case GITS_BASER_TYPE_RES5: + case GITS_BASER_TYPE_RES6: + case GITS_BASER_TYPE_RES7: + continue; + case GITS_BASER_TYPE_DEV: + nidents = (1 << GITS_TYPER_DEVB(gits_typer)); + its_tbl_size = esize * nidents; + its_tbl_size = roundup2(its_tbl_size, page_size); + npages = howmany(its_tbl_size, PAGE_SIZE); + break; + default: + npages = howmany(page_size, PAGE_SIZE); + break; + } + + /* Allocate required space */ + ptab_vaddr = (vm_offset_t)contigmalloc(npages * PAGE_SIZE, + M_GIC_V3_ITS, M_WAITOK | M_ZERO, 0, ~0UL, PAGE_SIZE, 0); + + sc->its_ptabs[tn].ptab_vaddr = ptab_vaddr; + sc->its_ptabs[tn].ptab_pgsz = PAGE_SIZE; + sc->its_ptabs[tn].ptab_npages = npages; + + ptab_paddr = vtophys(ptab_vaddr); + KASSERT((ptab_paddr & GITS_BASER_PA_MASK) == ptab_paddr, + ("%s: Unaligned PA for Interrupt Translation Table", + device_get_name(sc->dev))); + + /* Set defaults: WAWB, IS */ + cache = GITS_BASER_CACHE_WAWB; + share = GITS_BASER_SHARE_IS; + + while (1) { + nitspages = howmany(its_tbl_size, page_size); + + switch (page_size) { + case (1 << 12): /* 4KB */ + psz = GITS_BASER_PSZ_4K; + break; + case (1 << 14): /* 16KB */ + psz = GITS_BASER_PSZ_4K; + break; + case (1 << 16): /* 64KB */ + psz = GITS_BASER_PSZ_64K; + break; + default: + /* XXX: Other page sizes are currently not supported */ + device_printf(sc->dev, "Unsupported page size: %zuKB\n", + page_size / 1024); + its_free_tables(sc); + return (ENXIO); + } + + /* Clear fields under modification first */ + gits_baser &= ~(GITS_BASER_VALID | + GITS_BASER_CACHE_MASK | GITS_BASER_TYPE_MASK | + GITS_BASER_ESIZE_MASK | GITS_BASER_PA_MASK | + GITS_BASER_SHARE_MASK | GITS_BASER_PSZ_MASK | + GITS_BASER_SIZE_MASK); + /* Construct register value */ + gits_baser |= + (type << GITS_BASER_TYPE_SHIFT) | + ((esize - 1) << GITS_BASER_ESIZE_SHIFT) | + (cache << GITS_BASER_CACHE_SHIFT) | + (share << GITS_BASER_SHARE_SHIFT) | + (psz << GITS_BASER_PSZ_SHIFT) | + ptab_paddr | (nitspages - 1) | + GITS_BASER_VALID; + + gic_its_write(sc, 8, GITS_BASER(tn), gits_baser); + /* + * Verify. + * Depending on implementation we may encounter + * shareability and page size mismatch. + */ + gits_tmp = gic_its_read(sc, 8, GITS_BASER(tn)); + if ((gits_tmp ^ gits_baser) & GITS_BASER_SHARE_MASK) { + share = gits_tmp & GITS_BASER_SHARE_MASK; + share >>= GITS_BASER_SHARE_SHIFT; + continue; + } + + if ((gits_tmp ^ gits_baser) & GITS_BASER_PSZ_MASK) { + switch (page_size) { + case (1 << 14): + page_size = (1 << 12); + continue; + case (1 << 16): + page_size = (1 << 14); + continue; + } + } + /* We did what we could */ + break; + } + /* + * Do not compare Cacheability field since + * it is implementation defined. + */ + gits_tmp &= ~GITS_BASER_CACHE_MASK; + gits_baser &= ~GITS_BASER_CACHE_MASK; + + if (gits_tmp != gits_baser) { + device_printf(sc->dev, + "Could not allocate ITS tables\n"); + its_free_tables(sc); + return (ENXIO); + } + + if (bootverbose) { + if (first) { + device_printf(sc->dev, + "Allocated ITS private tables:\n"); + first = FALSE; + } + device_printf(sc->dev, + "\tPTAB%zu for %s: PA 0x%lx, %lu entries," + " cache policy %s, %s shareable," + " page size %zuKB\n", tn, its_ptab_type[type], + ptab_paddr, (page_size * nitspages) / esize, + its_ptab_cache[cache], its_ptab_share[share], + page_size / 1024); + } + } + + return (0); +} + +static void +its_free_tables(struct gic_v3_its_softc *sc) +{ + vm_offset_t ptab_vaddr; + size_t size; + size_t tn; + + for (tn = 0; tn < GITS_BASER_NUM; tn++) { + ptab_vaddr = sc->its_ptabs[tn].ptab_vaddr; + if (!ptab_vaddr) + continue; + size = sc->its_ptabs[tn].ptab_pgsz; + size *= sc->its_ptabs[tn].ptab_npages; + + if ((void *)ptab_vaddr != NULL) + contigfree((void *)ptab_vaddr, size, M_GIC_V3_ITS); + + /* Clear the table description */ + memset(&sc->its_ptabs[tn], 0, + sizeof(sc->its_ptabs[tn])); + } +} + +static int +its_init_commandq(struct gic_v3_its_softc *sc) +{ + uint64_t gits_cbaser, gits_tmp; + uint64_t cache, share; + vm_paddr_t cmdq_paddr; + device_t dev; + + dev = sc->dev; + /* Allocate memory for command queue */ + sc->its_cmdq_base = contigmalloc(ITS_CMDQ_SIZE, M_GIC_V3_ITS, + (M_WAITOK | M_ZERO), + 0, ~0UL, ITS_CMDQ_SIZE, 0); + /* Set command queue write pointer (command queue empty) */ + sc->its_cmdq_write = sc->its_cmdq_base; + + /* Save command queue pointer and attributes */ + cmdq_paddr = vtophys(sc->its_cmdq_base); + KASSERT((cmdq_paddr & GITS_CBASER_PA_MASK) == cmdq_paddr, + ("%s: Unaligned PA for ITS Commands Queue", device_get_name(dev))); + + /* Set defaults: Normal Inner WAWB, IS */ + cache = GITS_CBASER_CACHE_NIWAWB; + share = GITS_CBASER_SHARE_IS; + + gits_cbaser = (cmdq_paddr | + (cache << GITS_CBASER_CACHE_SHIFT) | + (share << GITS_CBASER_SHARE_SHIFT) | + /* Number of 4KB pages - 1 */ + (ITS_CMDQ_SIZE / (1 << 12) - 1) | + /* Valid bit */ + GITS_CBASER_VALID); + + gic_its_write(sc, 8, GITS_CBASER, gits_cbaser); + gits_tmp = gic_its_read(sc, 8, GITS_CBASER); + + if ((gits_tmp ^ gits_cbaser) & GITS_CBASER_SHARE_MASK) { + if (bootverbose) { + device_printf(dev, + "Will use cache flushing for commands queue\n"); + } + /* Command queue needs cache flushing */ + sc->its_flags |= ITS_FLAGS_CMDQ_FLUSH; + } + + gic_its_write(sc, 8, GITS_CWRITER, 0x0); + + return (0); +} + +static int +its_init_cpu(struct gic_v3_its_softc *sc) +{ + device_t parent; + struct gic_v3_softc *gic_sc; + + /* + * Check for LPIs support on this Re-Distributor. + */ + parent = device_get_parent(sc->dev); + gic_sc = device_get_softc(parent); + if (!(gic_r_read(gic_sc, 4, GICR_TYPER) & GICR_TYPER_PLPIS)) { + if (bootverbose) { + device_printf(sc->dev, + "LPIs not supported on CPU%u\n", PCPU_GET(cpuid)); + } + return (ENXIO); + } + + if (lpi_init_cpu(sc)) + return (ENXIO); + + /* Init collections */ + its_init_cpu_collection(sc); + + return (0); +} + +static int +its_init_cpu_collection(struct gic_v3_its_softc *sc) +{ + device_t parent; + struct gic_v3_softc *gic_sc; + uint64_t typer; + uint64_t target; + vm_offset_t redist_base; + u_int cpuid; + + cpuid = PCPU_GET(cpuid); + parent = device_get_parent(sc->dev); + gic_sc = device_get_softc(parent); + + typer = gic_its_read(sc, 8, GITS_TYPER); + if (typer & GITS_TYPER_PTA) { + redist_base = + rman_get_bushandle(gic_sc->gic_redists.pcpu[cpuid]); + /* + * Target Address correspond to the base physical + * address of Re-Distributors. + */ + target = vtophys(redist_base); + } else { + /* Target Address correspond to unique processor numbers */ + typer = gic_r_read(gic_sc, 8, GICR_TYPER); + target = GICR_TYPER_CPUNUM(typer); + } + + sc->its_cols[cpuid].col_target = target; + sc->its_cols[cpuid].col_id = cpuid; + + its_cmd_mapc(sc, &sc->its_cols[cpuid], 1); + its_cmd_invall(sc, &sc->its_cols[cpuid]); + + return (0); +} + +static int +lpi_init_conftable(struct gic_v3_its_softc *sc) +{ + device_t parent; + struct gic_v3_softc *gic_sc; + vm_offset_t conf_base; + uint8_t prio_default; + + parent = device_get_parent(sc->dev); + gic_sc = device_get_softc(parent); + /* + * LPI Configuration Table settings. + * Notice that Configuration Table is shared among all + * Re-Distributors, so this is going to be created just once. + */ + conf_base = (vm_offset_t)contigmalloc(LPI_CONFTAB_SIZE, + M_GIC_V3_ITS, M_WAITOK | M_ZERO, 0, ~0UL, PAGE_SIZE_64K, 0); + + KASSERT((vtophys(conf_base) & PAGE_MASK_64K) == 0, + ("LPI Configuration Table not aligned to 64 KB")); + + if (!conf_base) { + if (bootverbose) { + device_printf(sc->dev, + "Could not allocate memory for LPI " + "Configuration Table\n"); + } + return (ENOMEM); + } + + if (bootverbose) { + device_printf(sc->dev, + "LPI Configuration Table at PA: 0x%lx\n", + vtophys(conf_base)); + } + + /* + * Let the default priority be aligned with all other + * interrupts assuming that each interrupt is assigned + * MAX priority at startup. MAX priority on the other + * hand cannot be higher than 0xFC for LPIs. + */ + prio_default = GIC_PRIORITY_MAX; + + /* Write each settings byte to LPI configuration table */ + memset((void *)conf_base, + (prio_default & LPI_CONF_PRIO_MASK) | LPI_CONF_GROUP1, + LPI_CONFTAB_SIZE); + + cpu_dcache_wb_range((vm_offset_t)conf_base, roundup2(LPI_CONFTAB_SIZE, + PAGE_SIZE_64K)); + + gic_sc->gic_redists.lpis.conf_base = conf_base; + + return (0); +} + +static int +lpi_init_cpu(struct gic_v3_its_softc *sc) +{ + device_t parent; + struct gic_v3_softc *gic_sc; + vm_offset_t pend_base; + u_int cpuid; + + parent = device_get_parent(sc->dev); + gic_sc = device_get_softc(parent); + + /* + * LPI Pending Table settings. + * This has to be done for each Re-Distributor, hence for each CPU. + */ + cpuid = PCPU_GET(cpuid); + + pend_base = (vm_offset_t)contigmalloc( + roundup2(LPI_PENDTAB_SIZE, PAGE_SIZE_64K), M_GIC_V3_ITS, + (M_WAITOK | M_ZERO), 0, ~0UL, PAGE_SIZE_64K, 0); + + KASSERT((vtophys(pend_base) & PAGE_MASK_64K) == 0, + ("LPI Pending Table not aligned to 64 KB")); + + /* Clean D-cache so that ITS can see zeroed pages */ + cpu_dcache_wb_range((vm_offset_t)pend_base, + roundup2(LPI_PENDTAB_SIZE, PAGE_SIZE_64K)); + + if (!pend_base) { + if (bootverbose) { + device_printf(sc->dev, + "Could not allocate memory for LPI " + "Pending Table on CPU%u\n", cpuid); + } + return (ENOMEM); + } + if (bootverbose) { + device_printf(sc->dev, + "LPI Pending Table for CPU%u at PA: 0x%lx\n", + cpuid, vtophys(pend_base)); + } + + gic_sc->gic_redists.lpis.pend_base[cpuid] = pend_base; + + lpi_config_cpu(sc); + + return (0); +} + +static int +lpi_config_cpu(struct gic_v3_its_softc *sc) +{ + device_t parent; + struct gic_v3_softc *gic_sc; + vm_offset_t conf_base, pend_base; + uint64_t gicr_xbaser, gicr_temp; + uint64_t cache, share, idbits; + uint32_t gicr_ctlr; + u_int cpuid; + + parent = device_get_parent(sc->dev); + gic_sc = device_get_softc(parent); + cpuid = PCPU_GET(cpuid); + + conf_base = gic_sc->gic_redists.lpis.conf_base; + pend_base = gic_sc->gic_redists.lpis.pend_base[cpuid]; + + /* Disable LPIs */ + gicr_ctlr = gic_r_read(gic_sc, 4, GICR_CTLR); + gicr_ctlr &= ~GICR_CTLR_LPI_ENABLE; + gic_r_write(gic_sc, 4, GICR_CTLR, gicr_ctlr); + /* Perform full system barrier */ + dsb(sy); + + /* + * Set GICR_PROPBASER + */ + + /* + * Find out how many bits do we need for LPI identifiers. + * Remark 1.: Even though we have (LPI_CONFTAB_SIZE / 8) LPIs + * the notified LPI ID still starts from 8192 + * (GIC_FIRST_LPI). + * Remark 2.: This could be done on compilation time but there + * seems to be no sufficient macro. + */ + idbits = flsl(LPI_CONFTAB_SIZE + GIC_FIRST_LPI) - 1; + + /* Set defaults: Normal Inner WAWB, IS */ + cache = GICR_PROPBASER_CACHE_NIWAWB; + share = GICR_PROPBASER_SHARE_IS; + + gicr_xbaser = vtophys(conf_base) | + ((idbits - 1) & GICR_PROPBASER_IDBITS_MASK) | + (cache << GICR_PROPBASER_CACHE_SHIFT) | + (share << GICR_PROPBASER_SHARE_SHIFT); + + gic_r_write(gic_sc, 8, GICR_PROPBASER, gicr_xbaser); + gicr_temp = gic_r_read(gic_sc, 8, GICR_PROPBASER); + + if ((gicr_xbaser ^ gicr_temp) & GICR_PROPBASER_SHARE_MASK) { + if (bootverbose) { + device_printf(sc->dev, + "Will use cache flushing for LPI " + "Configuration Table\n"); + } + gic_sc->gic_redists.lpis.flags |= LPI_FLAGS_CONF_FLUSH; + } + + /* + * Set GICR_PENDBASER + */ + + /* Set defaults: Normal Inner WAWB, IS */ + cache = GICR_PENDBASER_CACHE_NIWAWB; + share = GICR_PENDBASER_SHARE_IS; + + gicr_xbaser = vtophys(pend_base) | + (cache << GICR_PENDBASER_CACHE_SHIFT) | + (share << GICR_PENDBASER_SHARE_SHIFT); + + gic_r_write(gic_sc, 8, GICR_PENDBASER, gicr_xbaser); + + /* Enable LPIs */ + gicr_ctlr = gic_r_read(gic_sc, 4, GICR_CTLR); + gicr_ctlr |= GICR_CTLR_LPI_ENABLE; + gic_r_write(gic_sc, 4, GICR_CTLR, gicr_ctlr); + + dsb(sy); + + return (0); +} + +static int +lpi_bitmap_init(struct gic_v3_its_softc *sc) +{ + device_t parent; + struct gic_v3_softc *gic_sc; + uint32_t lpi_id_num; + size_t lpi_chunks_num; + size_t bits_in_chunk; + + parent = device_get_parent(sc->dev); + gic_sc = device_get_softc(parent); + + lpi_id_num = (1 << gic_sc->gic_idbits) - 1; + /* Substract IDs dedicated for SGIs, PPIs and SPIs */ + lpi_id_num -= GIC_FIRST_LPI; + + sc->its_lpi_maxid = lpi_id_num; + + bits_in_chunk = sizeof(*sc->its_lpi_bitmap) * NBBY; + + /* + * Round up to the number of bits in chunk. + * We will need to take care to avoid using invalid LPI IDs later. + */ + lpi_id_num = roundup2(lpi_id_num, bits_in_chunk); + lpi_chunks_num = lpi_id_num / bits_in_chunk; + + sc->its_lpi_bitmap = + contigmalloc((lpi_chunks_num * sizeof(*sc->its_lpi_bitmap)), + M_GIC_V3_ITS, M_WAITOK | M_ZERO, 0, ~0UL, + sizeof(*sc->its_lpi_bitmap), 0); + + return (0); +} + +static int +lpi_alloc_chunk(struct gic_v3_its_softc *sc, struct lpi_chunk *lpic, + u_int nvecs) +{ + int fclr; /* First cleared bit */ + uint8_t *bitmap; + size_t nb, i; + + bitmap = (uint8_t *)sc->its_lpi_bitmap; + + fclr = 0; +retry: + /* Check other bits - sloooow */ + for (i = 0, nb = fclr; i < nvecs; i++, nb++) { + if (nb > sc->its_lpi_maxid) + return (EINVAL); + + if (isset(bitmap, nb)) { + /* To little free bits in this area. Move on. */ + fclr = nb + 1; + goto retry; + } + } + /* This area is free. Take it. */ + bit_nset(bitmap, fclr, fclr + nvecs - 1); + lpic->lpi_base = fclr + GIC_FIRST_LPI; + lpic->lpi_num = nvecs; + lpic->lpi_free = lpic->lpi_num; + + return (0); +} + +static void +lpi_configure(struct gic_v3_its_softc *sc, struct its_dev *its_dev, + uint32_t lpinum, boolean_t unmask) +{ + device_t parent; + struct gic_v3_softc *gic_sc; + uint8_t *conf_byte; + + parent = device_get_parent(sc->dev); + gic_sc = device_get_softc(parent); + + conf_byte = (uint8_t *)gic_sc->gic_redists.lpis.conf_base; + conf_byte += (lpinum - GIC_FIRST_LPI); + + if (unmask) + *conf_byte |= LPI_CONF_ENABLE; + else + *conf_byte &= ~LPI_CONF_ENABLE; + + if (gic_sc->gic_redists.lpis.flags & LPI_FLAGS_CONF_FLUSH) { + /* Clean D-cache under configuration byte */ + cpu_dcache_wb_range((vm_offset_t)conf_byte, sizeof(*conf_byte)); + } else { + /* DSB inner shareable, store */ + dsb(ishst); + } + + its_cmd_inv(sc, its_dev, lpinum); +} + +static void +lpi_map_to_device(struct gic_v3_its_softc *sc, struct its_dev *its_dev, + uint32_t id, uint32_t pid) +{ + + KASSERT((pid >= its_dev->lpis.lpi_base) && + (pid < (its_dev->lpis.lpi_base + its_dev->lpis.lpi_num)), + ("Trying to map ivalid LPI %u for this device\n", pid)); + + its_cmd_mapvi(sc, its_dev, id, pid); +} + +static void +lpi_xmask_irq(device_t parent, uint32_t irq, boolean_t unmask) +{ + struct its_dev *its_dev; + + TAILQ_FOREACH(its_dev, &its_sc->its_dev_list, entry) { + if (irq >= its_dev->lpis.lpi_base && + irq < (its_dev->lpis.lpi_base + its_dev->lpis.lpi_num)) { + lpi_configure(its_sc, its_dev, irq, unmask); + return; + } + } + + KASSERT(0, ("Trying to %s not existing LPI: %u\n", + (unmask == TRUE) ? "unmask" : "mask", irq)); +} + +void +lpi_unmask_irq(device_t parent, uint32_t irq) +{ + + lpi_xmask_irq(parent, irq, 1); +} + +void +lpi_mask_irq(device_t parent, uint32_t irq) +{ + + lpi_xmask_irq(parent, irq, 0); +} + +/* + * Commands handling. + */ + +static __inline void +cmd_format_command(struct its_cmd *cmd, uint8_t cmd_type) +{ + /* Command field: DW0 [7:0] */ + cmd->cmd_dword[0] &= ~CMD_COMMAND_MASK; + cmd->cmd_dword[0] |= cmd_type; +} + +static __inline void +cmd_format_devid(struct its_cmd *cmd, uint32_t devid) +{ + /* Device ID field: DW0 [63:32] */ + cmd->cmd_dword[0] &= ~CMD_DEVID_MASK; + cmd->cmd_dword[0] |= ((uint64_t)devid << CMD_DEVID_SHIFT); +} + +static __inline void +cmd_format_size(struct its_cmd *cmd, uint16_t size) +{ + /* Size field: DW1 [4:0] */ + cmd->cmd_dword[1] &= ~CMD_SIZE_MASK; + cmd->cmd_dword[1] |= (size & CMD_SIZE_MASK); +} + +static __inline void +cmd_format_id(struct its_cmd *cmd, uint32_t id) +{ + /* ID field: DW1 [31:0] */ + cmd->cmd_dword[1] &= ~CMD_ID_MASK; + cmd->cmd_dword[1] |= id; +} + +static __inline void +cmd_format_pid(struct its_cmd *cmd, uint32_t pid) +{ + /* Physical ID field: DW1 [63:32] */ + cmd->cmd_dword[1] &= ~CMD_PID_MASK; + cmd->cmd_dword[1] |= ((uint64_t)pid << CMD_PID_SHIFT); +} + +static __inline void +cmd_format_col(struct its_cmd *cmd, uint16_t col_id) +{ + /* Collection field: DW2 [16:0] */ + cmd->cmd_dword[2] &= ~CMD_COL_MASK; + cmd->cmd_dword[2] |= col_id; +} + +static __inline void +cmd_format_target(struct its_cmd *cmd, uint64_t target) +{ + /* Target Address field: DW2 [47:16] */ + cmd->cmd_dword[2] &= ~CMD_TARGET_MASK; + cmd->cmd_dword[2] |= (target & CMD_TARGET_MASK); +} + +static __inline void +cmd_format_itt(struct its_cmd *cmd, uint64_t itt) +{ + /* ITT Address field: DW2 [47:8] */ + cmd->cmd_dword[2] &= ~CMD_ITT_MASK; + cmd->cmd_dword[2] |= (itt & CMD_ITT_MASK); +} + +static __inline void +cmd_format_valid(struct its_cmd *cmd, uint8_t valid) +{ + /* Valid field: DW2 [63] */ + cmd->cmd_dword[2] &= ~CMD_VALID_MASK; + cmd->cmd_dword[2] |= ((uint64_t)valid << CMD_VALID_SHIFT); +} + +static __inline void +cmd_fix_endian(struct its_cmd *cmd) +{ + size_t i; + + for (i = 0; i < nitems(cmd->cmd_dword); i++) + cmd->cmd_dword[i] = htole64(cmd->cmd_dword[i]); +} + +static void +its_cmd_mapc(struct gic_v3_its_softc *sc, struct its_col *col, uint8_t valid) +{ + struct its_cmd_desc desc; + + desc.cmd_type = ITS_CMD_MAPC; + desc.cmd_desc_mapc.col = col; + /* + * Valid bit set - map the collection. + * Valid bit cleared - unmap the collection. + */ + desc.cmd_desc_mapc.valid = valid; + + its_cmd_send(sc, &desc); +} + +static void +its_cmd_mapvi(struct gic_v3_its_softc *sc, struct its_dev *its_dev, + uint32_t id, uint32_t pid) +{ + struct its_cmd_desc desc; + + desc.cmd_type = ITS_CMD_MAPVI; + desc.cmd_desc_mapvi.its_dev = its_dev; + desc.cmd_desc_mapvi.id = id; + desc.cmd_desc_mapvi.pid = pid; + + its_cmd_send(sc, &desc); +} + +static void __unused +its_cmd_mapi(struct gic_v3_its_softc *sc, struct its_dev *its_dev, + uint32_t lpinum) +{ + struct its_cmd_desc desc; + + desc.cmd_type = ITS_CMD_MAPI; + desc.cmd_desc_mapi.its_dev = its_dev; + desc.cmd_desc_mapi.lpinum = lpinum; + + its_cmd_send(sc, &desc); +} + +static void +its_cmd_mapd(struct gic_v3_its_softc *sc, struct its_dev *its_dev, + uint8_t valid) +{ + struct its_cmd_desc desc; + + desc.cmd_type = ITS_CMD_MAPD; + desc.cmd_desc_mapd.its_dev = its_dev; + desc.cmd_desc_mapd.valid = valid; + + its_cmd_send(sc, &desc); +} + +static void +its_cmd_inv(struct gic_v3_its_softc *sc, struct its_dev *its_dev, + uint32_t lpinum) +{ + struct its_cmd_desc desc; + + desc.cmd_type = ITS_CMD_INV; + desc.cmd_desc_inv.lpinum = lpinum - its_dev->lpis.lpi_base; + desc.cmd_desc_inv.its_dev = its_dev; + + its_cmd_send(sc, &desc); +} + +static void +its_cmd_invall(struct gic_v3_its_softc *sc, struct its_col *col) +{ + struct its_cmd_desc desc; + + desc.cmd_type = ITS_CMD_INVALL; + desc.cmd_desc_invall.col = col; + + its_cmd_send(sc, &desc); +} + +/* + * Helper routines for commands processing. + */ +static __inline boolean_t +its_cmd_queue_full(struct gic_v3_its_softc *sc) +{ + size_t read_idx, write_idx; + + write_idx = (size_t)(sc->its_cmdq_write - sc->its_cmdq_base); + read_idx = gic_its_read(sc, 4, GITS_CREADR) / sizeof(struct its_cmd); + + /* + * The queue is full when the write offset points + * at the command before the current read offset. + */ + if (((write_idx + 1) % ITS_CMDQ_NENTRIES) == read_idx) + return (TRUE); + + return (FALSE); +} + +static __inline void +its_cmd_sync(struct gic_v3_its_softc *sc, struct its_cmd *cmd) +{ + + if (sc->its_flags & ITS_FLAGS_CMDQ_FLUSH) { + /* Clean D-cache under command. */ + cpu_dcache_wb_range((vm_offset_t)cmd, sizeof(*cmd)); + } else { + /* DSB inner shareable, store */ + dsb(ishst); + } + +} + +static struct its_cmd * +its_cmd_alloc_locked(struct gic_v3_its_softc *sc) +{ + struct its_cmd *cmd; + size_t us_left = 1000000; + + mtx_assert(&sc->its_spin_mtx, MA_OWNED); + while (its_cmd_queue_full(sc)) { + if (us_left-- == 0) { + /* Timeout while waiting for free command */ + device_printf(sc->dev, + "Timeout while waiting for free command\n"); + return (NULL); + } + DELAY(1); + } + + cmd = sc->its_cmdq_write; + sc->its_cmdq_write++; + + if (sc->its_cmdq_write == (sc->its_cmdq_base + ITS_CMDQ_NENTRIES)) { + /* Wrap the queue */ + sc->its_cmdq_write = sc->its_cmdq_base; + } + + return (cmd); +} + +static uint64_t +its_cmd_prepare(struct its_cmd *cmd, struct its_cmd_desc *desc) +{ + uint64_t target; + uint8_t cmd_type; + u_int size; + boolean_t error; + + error = FALSE; + cmd_type = desc->cmd_type; + target = ITS_TARGET_NONE; + + switch (cmd_type) { + case ITS_CMD_SYNC: /* Wait for previous commands completion */ + target = desc->cmd_desc_sync.col->col_target; + cmd_format_command(cmd, ITS_CMD_SYNC); + cmd_format_target(cmd, target); + break; + case ITS_CMD_MAPD: /* Assign ITT to device */ + target = desc->cmd_desc_mapd.its_dev->col->col_target; + cmd_format_command(cmd, ITS_CMD_MAPD); + cmd_format_itt(cmd, vtophys(desc->cmd_desc_mapd.its_dev->itt)); + /* + * Size describes number of bits to encode interrupt IDs + * supported by the device minus one. + * When V (valid) bit is zero, this field should be written + * as zero. + */ + if (desc->cmd_desc_mapd.valid) { + size = + MAX(1, flsl(desc->cmd_desc_mapd.its_dev->lpis.lpi_num)) - 1; + } else + size = 0; + + cmd_format_size(cmd, size); + cmd_format_devid(cmd, desc->cmd_desc_mapd.its_dev->devid); + cmd_format_valid(cmd, desc->cmd_desc_mapd.valid); + break; + case ITS_CMD_MAPC: /* Map collection to Re-Distributor */ + target = desc->cmd_desc_mapc.col->col_target; + cmd_format_command(cmd, ITS_CMD_MAPC); + cmd_format_col(cmd, desc->cmd_desc_mapc.col->col_id); + cmd_format_valid(cmd, desc->cmd_desc_mapc.valid); + cmd_format_target(cmd, target); + break; + case ITS_CMD_MAPVI: + target = desc->cmd_desc_mapvi.its_dev->col->col_target; + cmd_format_command(cmd, ITS_CMD_MAPVI); + cmd_format_devid(cmd, desc->cmd_desc_mapvi.its_dev->devid); + cmd_format_id(cmd, desc->cmd_desc_mapvi.id); + cmd_format_pid(cmd, desc->cmd_desc_mapvi.pid); + cmd_format_col(cmd, desc->cmd_desc_mapvi.its_dev->col->col_id); + break; + case ITS_CMD_MAPI: + target = desc->cmd_desc_mapi.its_dev->col->col_target; + cmd_format_command(cmd, ITS_CMD_MAPI); + cmd_format_devid(cmd, desc->cmd_desc_mapi.its_dev->devid); + cmd_format_id(cmd, desc->cmd_desc_mapi.lpinum); + cmd_format_col(cmd, desc->cmd_desc_mapi.its_dev->col->col_id); + break; + case ITS_CMD_INV: + target = desc->cmd_desc_inv.its_dev->col->col_target; + cmd_format_command(cmd, ITS_CMD_INV); + cmd_format_devid(cmd, desc->cmd_desc_inv.its_dev->devid); + cmd_format_id(cmd, desc->cmd_desc_inv.lpinum); + break; + case ITS_CMD_INVALL: + cmd_format_command(cmd, ITS_CMD_INVALL); + cmd_format_col(cmd, desc->cmd_desc_invall.col->col_id); + break; + default: + error = TRUE; + break; + } + + if (!error) + cmd_fix_endian(cmd); + + return (target); +} + +static __inline uint64_t +its_cmd_cwriter_offset(struct gic_v3_its_softc *sc, struct its_cmd *cmd) +{ + uint64_t off; + + off = (cmd - sc->its_cmdq_base) * sizeof(*cmd); + + return (off); +} + +static void +its_cmd_wait_completion(struct gic_v3_its_softc *sc, struct its_cmd *cmd_first, + struct its_cmd *cmd_last) +{ + uint64_t first, last, read; + size_t us_left = 1000000; + + first = its_cmd_cwriter_offset(sc, cmd_first); + last = its_cmd_cwriter_offset(sc, cmd_last); + + while (1) { + read = gic_its_read(sc, 8, GITS_CREADR); + if (read < first || read >= last) + break; + + if (us_left-- == 0) { + /* This means timeout */ + device_printf(sc->dev, + "Timeout while waiting for CMD completion.\n"); + return; + } + DELAY(1); + } +} + +static int +its_cmd_send(struct gic_v3_its_softc *sc, struct its_cmd_desc *desc) +{ + struct its_cmd *cmd, *cmd_sync; + struct its_col col_sync; + struct its_cmd_desc desc_sync; + uint64_t target, cwriter; + + mtx_lock_spin(&sc->its_spin_mtx); + cmd = its_cmd_alloc_locked(sc); + mtx_unlock_spin(&sc->its_spin_mtx); + if (!cmd) { + device_printf(sc->dev, "no memory for cmd queue\n"); + return (EBUSY); + } + + target = its_cmd_prepare(cmd, desc); + its_cmd_sync(sc, cmd); + + if (target != ITS_TARGET_NONE) { + mtx_lock_spin(&sc->its_spin_mtx); + cmd_sync = its_cmd_alloc_locked(sc); + mtx_unlock_spin(&sc->its_spin_mtx); + if (!cmd_sync) + goto end; + desc_sync.cmd_type = ITS_CMD_SYNC; + col_sync.col_target = target; + desc_sync.cmd_desc_sync.col = &col_sync; + its_cmd_prepare(cmd_sync, &desc_sync); + its_cmd_sync(sc, cmd_sync); + } +end: + /* Update GITS_CWRITER */ + mtx_lock_spin(&sc->its_spin_mtx); + cwriter = its_cmd_cwriter_offset(sc, sc->its_cmdq_write); + gic_its_write(sc, 8, GITS_CWRITER, cwriter); + mtx_unlock_spin(&sc->its_spin_mtx); + + its_cmd_wait_completion(sc, cmd, sc->its_cmdq_write); + + return (0); +} + +static struct its_dev * +its_device_find_locked(struct gic_v3_its_softc *sc, device_t pci_dev) +{ + struct its_dev *its_dev; + + mtx_assert(&sc->its_mtx, MA_OWNED); + /* Find existing device if any */ + TAILQ_FOREACH(its_dev, &sc->its_dev_list, entry) { + if (its_dev->pci_dev == pci_dev) + return (its_dev); + } + + return (NULL); +} + +static struct its_dev * +its_device_alloc_locked(struct gic_v3_its_softc *sc, device_t pci_dev, + u_int nvecs) +{ + struct its_dev *newdev; + uint64_t typer; + uint32_t devid; + u_int cpuid; + size_t esize; + + mtx_assert(&sc->its_mtx, MA_OWNED); + /* Find existing device if any */ + newdev = its_device_find_locked(sc, pci_dev); + if (newdev != NULL) + return (newdev); + + devid = PCI_DEVID(pci_dev); + + /* There was no previously created device. Create one now */ + newdev = malloc(sizeof(*newdev), M_GIC_V3_ITS, M_WAITOK | M_ZERO); + newdev->pci_dev = pci_dev; + newdev->devid = devid; + + if (lpi_alloc_chunk(sc, &newdev->lpis, nvecs)) { + free(newdev, M_GIC_V3_ITS); + return (NULL); + } + + /* Get ITT entry size */ + typer = gic_its_read(sc, 8, GITS_TYPER); + esize = GITS_TYPER_ITTES(typer); + /* + * Allocate ITT for this device. + * PA has to be 256 B aligned. At least two entries for device. + */ + newdev->itt = (vm_offset_t)contigmalloc( + roundup2(roundup2(nvecs, 2) * esize, 0x100), M_GIC_V3_ITS, + (M_WAITOK | M_ZERO), 0, ~0UL, 0x100, 0); + + /* XXX: Bind device interrupts to this CPU */ + cpuid = PCPU_GET(cpuid); + newdev->col = &sc->its_cols[cpuid]; + + TAILQ_INSERT_TAIL(&sc->its_dev_list, newdev, entry); + + /* Map device to its ITT */ + its_cmd_mapd(sc, newdev, 1); + + return (newdev); +} + +static __inline void +its_device_asign_lpi_locked(struct gic_v3_its_softc *sc, + struct its_dev *its_dev, u_int *irq) +{ + + mtx_assert(&sc->its_mtx, MA_OWNED); + KASSERT((its_dev->lpis.lpi_free > 0), + ("Cannot provide more LPIs for this device. LPI num: %u, free %u", + its_dev->lpis.lpi_num, its_dev->lpis.lpi_free)); + *irq = its_dev->lpis.lpi_base + (its_dev->lpis.lpi_num - + its_dev->lpis.lpi_free); + its_dev->lpis.lpi_free--; +} +/* + * Message signalled interrupts handling. + */ + +/* + * XXX ARM64TODO: Watch out for "irq" type. + * + * In theory GIC can handle up to (2^32 - 1) interrupt IDs whereas + * we pass "irq" pointer of type integer. This is obviously wrong but + * is determined by the way as PCI layer wants it to be done. + * WARNING: devid must be PCI BUS+FUNCTION pair. + */ +int +gic_v3_its_alloc_msix(device_t dev, device_t pci_dev, int *irq) +{ + struct gic_v3_its_softc *sc; + struct its_dev *its_dev; + u_int nvecs; + + sc = device_get_softc(dev); + + mtx_lock(&sc->its_mtx); + nvecs = PCI_MSIX_NUM(pci_dev); + + /* + * TODO: Allocate device as seen by ITS if not already available. + * Notice that MSI-X interrupts are allocated on one-by-one basis. + */ + its_dev = its_device_alloc_locked(sc, pci_dev, nvecs); + if (its_dev == NULL) { + mtx_unlock(&sc->its_mtx); + return (ENOMEM); + } + + its_device_asign_lpi_locked(sc, its_dev, irq); + mtx_unlock(&sc->its_mtx); + + return (0); +} + +int +gic_v3_its_alloc_msi(device_t dev, device_t pci_dev, int count, int *irqs) +{ + struct gic_v3_its_softc *sc; + struct its_dev *its_dev; + + sc = device_get_softc(dev); + + /* Allocate device as seen by ITS if not already available. */ + mtx_lock(&sc->its_mtx); + its_dev = its_device_alloc_locked(sc, pci_dev, count); + if (its_dev == NULL) { + mtx_unlock(&sc->its_mtx); + return (ENOMEM); + } + + for (; count > 0; count--) { + its_device_asign_lpi_locked(sc, its_dev, irqs); + irqs++; + } + mtx_unlock(&sc->its_mtx); + + return (0); +} + +int +gic_v3_its_map_msix(device_t dev, device_t pci_dev, int irq, uint64_t *addr, + uint32_t *data) +{ + struct gic_v3_its_softc *sc; + bus_space_handle_t its_bsh; + struct its_dev *its_dev; + uint64_t its_pa; + uint32_t id; + + sc = device_get_softc(dev); + /* Verify that this device is allocated and owns this LPI */ + mtx_lock(&sc->its_mtx); + its_dev = its_device_find_locked(sc, pci_dev); + mtx_unlock(&sc->its_mtx); + if (its_dev == NULL) + return (EINVAL); + + id = irq - its_dev->lpis.lpi_base; + lpi_map_to_device(sc, its_dev, id, irq); + + its_bsh = rman_get_bushandle(&sc->its_res[0]); + its_pa = vtophys(its_bsh); + + *addr = (its_pa + GITS_TRANSLATER); + *data = id; + + return (0); +} diff --git a/sys/arm64/arm64/gic_v3_fdt.c b/sys/arm64/arm64/gic_v3_fdt.c --- a/sys/arm64/arm64/gic_v3_fdt.c +++ b/sys/arm64/arm64/gic_v3_fdt.c @@ -65,6 +65,9 @@ static int gic_v3_fdt_attach(device_t); static int gic_v3_fdt_detach(device_t); +static struct resource * gic_v3_bus_alloc_res(device_t, + device_t, int, int *, u_long, u_long, u_long, u_int); + static const struct ofw_bus_devinfo * gic_v3_ofw_get_devinfo(device_t __unused, device_t); @@ -80,6 +83,18 @@ DEVMETHOD(pic_mask, gic_v3_mask_irq), DEVMETHOD(pic_unmask, gic_v3_unmask_irq), + /* Bus interface */ + DEVMETHOD(bus_alloc_resource, gic_v3_bus_alloc_res), + DEVMETHOD(bus_activate_resource, bus_generic_activate_resource), + + /* ofw_bus interface */ + DEVMETHOD(ofw_bus_get_devinfo, gic_v3_ofw_get_devinfo), + DEVMETHOD(ofw_bus_get_compat, ofw_bus_gen_get_compat), + DEVMETHOD(ofw_bus_get_model, ofw_bus_gen_get_model), + DEVMETHOD(ofw_bus_get_name, ofw_bus_gen_get_name), + DEVMETHOD(ofw_bus_get_node, ofw_bus_gen_get_node), + DEVMETHOD(ofw_bus_get_type, ofw_bus_gen_get_type), + /* End */ DEVMETHOD_END }; @@ -96,6 +111,11 @@ BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); /* + * Helper functions declarations. + */ +static int gic_v3_fdt_bus_attach(device_t); + +/* * Device interface. */ static int @@ -134,6 +154,17 @@ err = gic_v3_attach(dev); if (err) goto error; + /* + * Try to register ITS to this GIC. + * GIC will act as a bus in that case. + * Failure here will not affect main GIC functionality. + */ + if (gic_v3_fdt_bus_attach(dev)) { + if (bootverbose) { + device_printf(dev, + "Failed to attach ITS to this GIC\n"); + } + } return (err); @@ -154,3 +185,169 @@ return (gic_v3_detach(dev)); } + +/* ofw_bus interface */ +static const struct ofw_bus_devinfo * +gic_v3_ofw_get_devinfo(device_t bus __unused, device_t child) +{ + struct gic_v3_ofw_devinfo *di; + + di = device_get_ivars(child); + return (&di->di_dinfo); +} + +/* Bus interface */ +static struct resource * +gic_v3_bus_alloc_res(device_t bus, device_t child, int type, int *rid, + u_long start, u_long end, u_long count, u_int flags) +{ + struct gic_v3_ofw_devinfo *di; + struct resource_list_entry *rle; + int ranges_len; + + if ((start == 0UL) && (end == ~0UL)) { + if ((di = device_get_ivars(child)) == NULL) + return (NULL); + if (type != SYS_RES_MEMORY) + return (NULL); + + /* Find defaults for this rid */ + rle = resource_list_find(&di->di_rl, type, *rid); + if (rle == NULL) + return (NULL); + + start = rle->start; + end = rle->end; + count = rle->count; + } + /* + * XXX: No ranges remap! + * Absolute address is expected. + */ + if (ofw_bus_has_prop(bus, "ranges")) { + ranges_len = OF_getproplen(ofw_bus_get_node(bus), "ranges"); + if (ranges_len != 0) { + if (bootverbose) { + device_printf(child, + "Ranges remap not supported\n"); + } + return (NULL); + } + } + return (bus_generic_alloc_resource(bus, child, type, rid, start, end, + count, flags)); +} + +/* Helper functions */ + +/* + * Bus capability support for GICv3. + * Collects and configures device informations and finally + * adds ITS device as a child of GICv3 in Newbus hierarchy. + */ +static int +gic_v3_fdt_bus_attach(device_t dev) +{ + struct gic_v3_ofw_devinfo *di; + device_t child; + phandle_t parent, node; + pcell_t addr_cells, size_cells; + + parent = ofw_bus_get_node(dev); + if (parent > 0) { + addr_cells = 2; + OF_getencprop(parent, "#address-cells", &addr_cells, + sizeof(addr_cells)); + size_cells = 2; + OF_getencprop(parent, "#size-cells", &size_cells, + sizeof(size_cells)); + /* Iterate through all GIC subordinates */ + for (node = OF_child(parent); node > 0; node = OF_peer(node)) { + /* Allocate and populate devinfo. */ + di = malloc(sizeof(*di), M_GIC_V3, M_WAITOK | M_ZERO); + if (ofw_bus_gen_setup_devinfo(&di->di_dinfo, node)) { + if (bootverbose) { + device_printf(dev, + "Could not set up devinfo for ITS\n"); + } + free(di, M_GIC_V3); + continue; + } + + /* Initialize and populate resource list. */ + resource_list_init(&di->di_rl); + ofw_bus_reg_to_rl(dev, node, addr_cells, size_cells, + &di->di_rl); + + /* Should not have any interrupts, so don't add any */ + + /* Add newbus device for this FDT node */ + child = device_add_child(dev, NULL, -1); + if (!child) { + if (bootverbose) { + device_printf(dev, + "Could not add child: %s\n", + di->di_dinfo.obd_name); + } + resource_list_free(&di->di_rl); + ofw_bus_gen_destroy_devinfo(&di->di_dinfo); + free(di, M_GIC_V3); + continue; + } + + device_set_ivars(child, di); + } + } + + return (bus_generic_attach(dev)); +} + +static int gic_v3_its_fdt_probe(device_t dev); +static int gic_v3_its_fdt_attach(device_t dev); + +static device_method_t gic_v3_its_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, gic_v3_its_fdt_probe), + DEVMETHOD(device_attach, gic_v3_its_fdt_attach), + + /* PIC interface */ + /* MSI-X */ + DEVMETHOD(pic_alloc_msix, gic_v3_its_alloc_msix), + DEVMETHOD(pic_map_msix, gic_v3_its_map_msix), + /* MSI */ + DEVMETHOD(pic_alloc_msi, gic_v3_its_alloc_msi), + DEVMETHOD(pic_map_msi, gic_v3_its_map_msix), + + /* End */ + DEVMETHOD_END +}; + +static driver_t gic_v3_its_driver = { + "gic-its", + gic_v3_its_methods, + sizeof(struct gic_v3_its_softc), +}; + +EARLY_DRIVER_MODULE(gic_v3_its, gic, gic_v3_its_driver, gic_v3_its_devclass, 0, 0, + BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); + +static int +gic_v3_its_fdt_probe(device_t dev) +{ + + if (!ofw_bus_status_okay(dev)) + return (ENXIO); + + if (!ofw_bus_is_compatible(dev, GIC_V3_ITS_COMPSTR)) + return (ENXIO); + + device_set_desc(dev, GIC_V3_ITS_DEVSTR); + return (BUS_PROBE_DEFAULT); +} + +static int +gic_v3_its_fdt_attach(device_t dev) +{ + + return (gic_v3_its_attach(dev)); +} diff --git a/sys/arm64/arm64/gic_v3.c b/sys/arm64/arm64/gic_v3.c --- a/sys/arm64/arm64/gic_v3.c +++ b/sys/arm64/arm64/gic_v3.c @@ -219,19 +219,18 @@ break; if (__predict_true((active_irq >= GIC_FIRST_PPI && - active_irq <= GIC_LAST_SPI))) { + active_irq <= GIC_LAST_SPI) || active_irq >= GIC_FIRST_LPI)) { arm_dispatch_intr(active_irq, frame); continue; } - if (active_irq <= GIC_LAST_SGI || active_irq >= GIC_FIRST_LPI) { + if (active_irq <= GIC_LAST_SGI) { /* - * TODO: Implement proper SGI/LPI handling. + * TODO: Implement proper SGI handling. * Mask it if such is received for some reason. */ device_printf(dev, - "Received unsupported interrupt type: %s\n", - sctive_irq >= GIC_FIRST_LPI ? "LPI" : "SGI"); + "Received unsupported interrupt type: SGI\n"); PIC_MASK(dev, active_irq); } } @@ -261,6 +260,8 @@ } else if (irq >= GIC_FIRST_SPI && irq <= GIC_LAST_SPI) { /* SPIs in distributor */ gic_r_write(sc, 4, GICD_ICENABLER(irq >> 5), mask); gic_v3_wait_for_rwp(sc, DIST); + } else if (irq >= GIC_FIRST_LPI) { /* LPIs */ + lpi_mask_irq(dev, irq); } else { KASSERT(0, ("%s: Unsupported IRQ number %u", __func__, irq)); } @@ -284,6 +285,8 @@ } else if (irq >= GIC_FIRST_SPI && irq <= GIC_LAST_SPI) { /* SPIs in distributor */ gic_d_write(sc, 4, GICD_ISENABLER(irq >> 5), mask); gic_v3_wait_for_rwp(sc, DIST); + } else if (irq >= GIC_FIRST_LPI) { /* LPIs */ + lpi_unmask_irq(dev, irq); } else { KASSERT(0, ("%s: Unsupported IRQ number %u", __func__, irq)); }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?differential-rev-PHID-DREV-xmqmcfafyun443233vly-req>