From owner-svn-src-stable@freebsd.org Wed Jun 15 01:42:54 2016 Return-Path: Delivered-To: svn-src-stable@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id E26D4B72872; Wed, 15 Jun 2016 01:42:54 +0000 (UTC) (envelope-from mav@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id A1EC12F93; Wed, 15 Jun 2016 01:42:54 +0000 (UTC) (envelope-from mav@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id u5F1grwX018524; Wed, 15 Jun 2016 01:42:53 GMT (envelope-from mav@FreeBSD.org) Received: (from mav@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id u5F1grIs018523; Wed, 15 Jun 2016 01:42:53 GMT (envelope-from mav@FreeBSD.org) Message-Id: <201606150142.u5F1grIs018523@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: mav set sender to mav@FreeBSD.org using -f From: Alexander Motin Date: Wed, 15 Jun 2016 01:42:53 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-10@freebsd.org Subject: svn commit: r301904 - stable/10/sys/dev/ntb/ntb_hw X-SVN-Group: stable-10 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-stable@freebsd.org X-Mailman-Version: 2.1.22 Precedence: list List-Id: SVN commit messages for all the -stable branches of the src tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 15 Jun 2016 01:42:55 -0000 Author: mav Date: Wed Jun 15 01:42:53 2016 New Revision: 301904 URL: https://svnweb.freebsd.org/changeset/base/301904 Log: MFC r301293: When negotiating NTB_SB01BASE_LOCKUP workaround, don't try to limit the BAR size to 1MB. According to Xeon v3 specifications and my tests, that size register is write-once and so not writeable after BIOS written it. Instead of that, make the code work with BAR of any sufficient size, properly calculating offset within its base. It also simplifies the code. Sponsored by: iXsystems, Inc. Modified: stable/10/sys/dev/ntb/ntb_hw/ntb_hw.c Directory Properties: stable/10/ (props changed) Modified: stable/10/sys/dev/ntb/ntb_hw/ntb_hw.c ============================================================================== --- stable/10/sys/dev/ntb/ntb_hw/ntb_hw.c Wed Jun 15 01:41:43 2016 (r301903) +++ stable/10/sys/dev/ntb/ntb_hw/ntb_hw.c Wed Jun 15 01:42:53 2016 (r301904) @@ -75,7 +75,6 @@ __FBSDID("$FreeBSD$"); #define NTB_MSIX_VER_GUARD 0xaabbccdd #define NTB_MSIX_RECEIVED 0xe0f0e0f0 -#define ONE_MB (1024u * 1024) /* * PCI constants could be somewhere more generic, but aren't defined/used in @@ -239,6 +238,7 @@ struct ntb_softc { /* Memory window used to access peer bar0 */ #define B2B_MW_DISABLED UINT8_MAX uint8_t b2b_mw_idx; + uint32_t msix_xlat; uint8_t msix_mw_idx; uint8_t mw_count; @@ -1375,12 +1375,12 @@ ntb_get_msix_info(struct ntb_softc *ntb) laddr = bus_read_4(msix->msix_table_res, offset + PCI_MSIX_ENTRY_LOWER_ADDR); - ntb_printf(2, "local lower MSIX addr(%u): 0x%x\n", i, laddr); + ntb_printf(2, "local MSIX addr(%u): 0x%x\n", i, laddr); KASSERT((laddr & MSI_INTEL_ADDR_BASE) == MSI_INTEL_ADDR_BASE, ("local MSIX addr 0x%x not in MSI base 0x%x", laddr, MSI_INTEL_ADDR_BASE)); - ntb->msix_data[i].nmd_ofs = laddr & ~MSI_INTEL_ADDR_BASE; + ntb->msix_data[i].nmd_ofs = laddr; data = bus_read_4(msix->msix_table_res, offset + PCI_MSIX_ENTRY_DATA); @@ -1674,15 +1674,6 @@ xeon_reset_sbar_size(struct ntb_softc *n bar_sz--; else bar_sz = 0; - } else if (HAS_FEATURE(NTB_SB01BASE_LOCKUP) && - ntb_mw_to_bar(ntb, ntb->msix_mw_idx) == idx) { - /* Restrict LAPIC BAR to 1MB */ - pci_write_config(ntb->device, bar->psz_off, 20, 1); - pci_write_config(ntb->device, bar->ssz_off, 20, 1); - bar_sz = pci_read_config(ntb->device, bar->psz_off, 1); - bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1); - (void)bar_sz; - return; } pci_write_config(ntb->device, bar->ssz_off, bar_sz, 1); bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1); @@ -1693,24 +1684,19 @@ static void xeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t bar_addr, enum ntb_bar idx, enum ntb_bar regbar) { - uint64_t reg_val, lmt_addr; + uint64_t reg_val; uint32_t base_reg, lmt_reg; bar_get_xlat_params(ntb, idx, &base_reg, NULL, &lmt_reg); if (idx == regbar) bar_addr += ntb->b2b_off; - lmt_addr = bar_addr; - - if (HAS_FEATURE(NTB_SB01BASE_LOCKUP) && - ntb_mw_to_bar(ntb, ntb->msix_mw_idx) == idx) - lmt_addr += ONE_MB; /* * Set limit registers first to avoid an errata where setting the base * registers locks the limit registers. */ if (!bar_is_64bit(ntb, idx)) { - ntb_reg_write(4, lmt_reg, lmt_addr); + ntb_reg_write(4, lmt_reg, bar_addr); reg_val = ntb_reg_read(4, lmt_reg); (void)reg_val; @@ -1718,7 +1704,7 @@ xeon_set_sbar_base_and_limit(struct ntb_ reg_val = ntb_reg_read(4, base_reg); (void)reg_val; } else { - ntb_reg_write(8, lmt_reg, lmt_addr); + ntb_reg_write(8, lmt_reg, bar_addr); reg_val = ntb_reg_read(8, lmt_reg); (void)reg_val; @@ -1747,31 +1733,13 @@ xeon_set_pbar_xlat(struct ntb_softc *ntb static int xeon_setup_msix_bar(struct ntb_softc *ntb) { - struct ntb_pci_bar_info *lapic_bar; enum ntb_bar bar_num; - int rc; if (!HAS_FEATURE(NTB_SB01BASE_LOCKUP)) return (0); bar_num = ntb_mw_to_bar(ntb, ntb->msix_mw_idx); - lapic_bar = &ntb->bar_info[bar_num]; - - /* Restrict LAPIC BAR to 1MB */ - if (lapic_bar->size > ONE_MB) { - rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, - lapic_bar->pci_resource, lapic_bar->pbase, - lapic_bar->pbase + ONE_MB - 1); - if (rc == 0) - lapic_bar->size = ONE_MB; - else { - ntb_printf(0, "Failed to shrink LAPIC BAR resource to " - "1 MB: %d\n", rc); - /* Ignore error */ - } - } - - ntb->peer_lapic_bar = lapic_bar; + ntb->peer_lapic_bar = &ntb->bar_info[bar_num]; return (0); } @@ -1882,10 +1850,13 @@ xeon_setup_b2b_mw(struct ntb_softc *ntb, * We point the chosen MSIX MW BAR xlat to remote LAPIC for * workaround */ - if (size == 4) + if (size == 4) { ntb_reg_write(4, xlatoffset, MSI_INTEL_ADDR_BASE); - else + ntb->msix_xlat = ntb_reg_read(4, xlatoffset); + } else { ntb_reg_write(8, xlatoffset, MSI_INTEL_ADDR_BASE); + ntb->msix_xlat = ntb_reg_read(8, xlatoffset); + } } (void)ntb_reg_read(8, XEON_SBAR2XLAT_OFFSET); (void)ntb_reg_read(8, XEON_SBAR4XLAT_OFFSET); @@ -2829,7 +2800,7 @@ ntb_exchange_msix(void *ctx) ntb_peer_spad_write(ntb, NTB_MSIX_DATA0 + i, ntb->msix_data[i].nmd_data); ntb_peer_spad_write(ntb, NTB_MSIX_OFS0 + i, - ntb->msix_data[i].nmd_ofs); + ntb->msix_data[i].nmd_ofs - ntb->msix_xlat); } ntb_peer_spad_write(ntb, NTB_MSIX_GUARD, NTB_MSIX_VER_GUARD); @@ -2839,8 +2810,10 @@ ntb_exchange_msix(void *ctx) for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { ntb_spad_read(ntb, NTB_MSIX_DATA0 + i, &val); + ntb_printf(2, "remote MSIX data(%u): 0x%x\n", i, val); ntb->peer_msix_data[i].nmd_data = val; ntb_spad_read(ntb, NTB_MSIX_OFS0 + i, &val); + ntb_printf(2, "remote MSIX addr(%u): 0x%x\n", i, val); ntb->peer_msix_data[i].nmd_ofs = val; }