From owner-svn-src-head@FreeBSD.ORG Sun Jan 11 20:27:16 2015 Return-Path: Delivered-To: svn-src-head@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [8.8.178.115]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by hub.freebsd.org (Postfix) with ESMTPS id EEEA17F6; Sun, 11 Jan 2015 20:27:16 +0000 (UTC) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id D08F8D05; Sun, 11 Jan 2015 20:27:16 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.9/8.14.9) with ESMTP id t0BKRGSC028262; Sun, 11 Jan 2015 20:27:16 GMT (envelope-from kib@FreeBSD.org) Received: (from kib@localhost) by svn.freebsd.org (8.14.9/8.14.9/Submit) id t0BKRF0A028255; Sun, 11 Jan 2015 20:27:15 GMT (envelope-from kib@FreeBSD.org) Message-Id: <201501112027.t0BKRF0A028255@svn.freebsd.org> X-Authentication-Warning: svn.freebsd.org: kib set sender to kib@FreeBSD.org using -f From: Konstantin Belousov Date: Sun, 11 Jan 2015 20:27:15 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r277023 - head/sys/x86/iommu X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-head@freebsd.org X-Mailman-Version: 2.1.18-1 Precedence: list List-Id: SVN commit messages for the src tree for head/-current List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 11 Jan 2015 20:27:17 -0000 Author: kib Date: Sun Jan 11 20:27:15 2015 New Revision: 277023 URL: https://svnweb.freebsd.org/changeset/base/277023 Log: Right now, for non-coherent DMARs, page table update code flushes the cache for whole page containing modified pte, and more, only last page in the series of the consequtive pages is flushed (i.e. the affected mappings should be larger than 2MB). Avoid excessive flushing and do missed neccessary flushing, by splitting invalidation and unmapping. For now, flush exactly the range of the changed pte. This is still somewhat bigger than neccessary, since pte is 8 bytes, while cache flush line is at least 32 bytes. The originator of the issue reports that after the change, 'dmar_bus_dmamap_unload went from 13,288 cycles down to 3,257. dmar_bus_dmamap_load_buffer went from 9,686 cycles down to 3,517. and I am now able to get line 1GbE speed with Netperf TCP (even with 1K message size).' Diagnosed and tested by: Nadav Amit Sponsored by: The FreeBSD Foundation MFC after: 1 week Modified: head/sys/x86/iommu/intel_ctx.c head/sys/x86/iommu/intel_dmar.h head/sys/x86/iommu/intel_idpgtbl.c head/sys/x86/iommu/intel_utils.c Modified: head/sys/x86/iommu/intel_ctx.c ============================================================================== --- head/sys/x86/iommu/intel_ctx.c Sun Jan 11 20:22:12 2015 (r277022) +++ head/sys/x86/iommu/intel_ctx.c Sun Jan 11 20:27:15 2015 (r277023) @@ -97,7 +97,8 @@ dmar_ensure_ctx_page(struct dmar_unit *d re += bus; dmar_pte_store(&re->r1, DMAR_ROOT_R1_P | (DMAR_ROOT_R1_CTP_MASK & VM_PAGE_TO_PHYS(ctxm))); - dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(dmar)); + dmar_flush_root_to_ram(dmar, re); + dmar_unmap_pgtbl(sf); TD_PINNED_ASSERT; } @@ -158,6 +159,7 @@ ctx_id_entry_init(struct dmar_ctx *ctx, (DMAR_CTX1_ASR_MASK & VM_PAGE_TO_PHYS(ctx_root)) | DMAR_CTX1_P); } + dmar_flush_ctx_to_ram(unit, ctxp); } static int @@ -364,7 +366,7 @@ dmar_get_ctx(struct dmar_unit *dmar, dev ctx->domain = alloc_unrl(dmar->domids); if (ctx->domain == -1) { DMAR_UNLOCK(dmar); - dmar_unmap_pgtbl(sf, true); + dmar_unmap_pgtbl(sf); dmar_ctx_dtr(ctx, true, true); TD_PINNED_ASSERT; return (NULL); @@ -389,7 +391,7 @@ dmar_get_ctx(struct dmar_unit *dmar, dev } else { dmar_ctx_dtr(ctx1, true, true); } - dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(dmar)); + dmar_unmap_pgtbl(sf); } ctx->refs++; if ((ctx->flags & DMAR_CTX_RMRR) != 0) @@ -480,7 +482,7 @@ dmar_free_ctx_locked(struct dmar_unit *d if (ctx->refs > 1) { ctx->refs--; DMAR_UNLOCK(dmar); - dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(dmar)); + dmar_unmap_pgtbl(sf); TD_PINNED_ASSERT; return; } @@ -496,6 +498,7 @@ dmar_free_ctx_locked(struct dmar_unit *d */ dmar_pte_clear(&ctxp->ctx1); ctxp->ctx2 = 0; + dmar_flush_ctx_to_ram(dmar, ctxp); dmar_inv_ctx_glob(dmar); if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0) { if (dmar->qi_enabled) @@ -513,7 +516,7 @@ dmar_free_ctx_locked(struct dmar_unit *d taskqueue_drain(dmar->delayed_taskqueue, &ctx->unload_task); KASSERT(TAILQ_EMPTY(&ctx->unload_entries), ("unfinished unloads %p", ctx)); - dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(dmar)); + dmar_unmap_pgtbl(sf); free_unr(dmar->domids, ctx->domain); dmar_ctx_dtr(ctx, true, true); TD_PINNED_ASSERT; Modified: head/sys/x86/iommu/intel_dmar.h ============================================================================== --- head/sys/x86/iommu/intel_dmar.h Sun Jan 11 20:22:12 2015 (r277022) +++ head/sys/x86/iommu/intel_dmar.h Sun Jan 11 20:27:15 2015 (r277023) @@ -228,11 +228,14 @@ struct vm_page *dmar_pgalloc(vm_object_t void dmar_pgfree(vm_object_t obj, vm_pindex_t idx, int flags); void *dmar_map_pgtbl(vm_object_t obj, vm_pindex_t idx, int flags, struct sf_buf **sf); -void dmar_unmap_pgtbl(struct sf_buf *sf, bool coherent); +void dmar_unmap_pgtbl(struct sf_buf *sf); int dmar_load_root_entry_ptr(struct dmar_unit *unit); int dmar_inv_ctx_glob(struct dmar_unit *unit); int dmar_inv_iotlb_glob(struct dmar_unit *unit); int dmar_flush_write_bufs(struct dmar_unit *unit); +void dmar_flush_pte_to_ram(struct dmar_unit *unit, dmar_pte_t *dst); +void dmar_flush_ctx_to_ram(struct dmar_unit *unit, dmar_ctx_entry_t *dst); +void dmar_flush_root_to_ram(struct dmar_unit *unit, dmar_root_entry_t *dst); int dmar_enable_translation(struct dmar_unit *unit); int dmar_disable_translation(struct dmar_unit *unit); bool dmar_barrier_enter(struct dmar_unit *dmar, u_int barrier_id); Modified: head/sys/x86/iommu/intel_idpgtbl.c ============================================================================== --- head/sys/x86/iommu/intel_idpgtbl.c Sun Jan 11 20:22:12 2015 (r277022) +++ head/sys/x86/iommu/intel_idpgtbl.c Sun Jan 11 20:27:15 2015 (r277023) @@ -146,7 +146,7 @@ ctx_idmap_nextlvl(struct idpgtbl *tbl, i } } /* ctx_get_idmap_pgtbl flushes CPU cache if needed. */ - dmar_unmap_pgtbl(sf, true); + dmar_unmap_pgtbl(sf); VM_OBJECT_WLOCK(tbl->pgtbl_obj); } @@ -361,7 +361,7 @@ ctx_pgtbl_map_pte(struct dmar_ctx *ctx, pte = (dmar_pte_t *)sf_buf_kva(*sf); } else { if (*sf != NULL) - dmar_unmap_pgtbl(*sf, DMAR_IS_COHERENT(ctx->dmar)); + dmar_unmap_pgtbl(*sf); *idxp = idx; retry: pte = dmar_map_pgtbl(ctx->pgtbl_obj, idx, flags, sf); @@ -397,9 +397,10 @@ retry: } dmar_pte_store(&ptep->pte, DMAR_PTE_R | DMAR_PTE_W | VM_PAGE_TO_PHYS(m)); + dmar_flush_pte_to_ram(ctx->dmar, ptep); sf_buf_page(sfp)->wire_count += 1; m->wire_count--; - dmar_unmap_pgtbl(sfp, DMAR_IS_COHERENT(ctx->dmar)); + dmar_unmap_pgtbl(sfp); /* Only executed once. */ goto retry; } @@ -467,20 +468,19 @@ ctx_map_buf_locked(struct dmar_ctx *ctx, if (pte == NULL) { KASSERT((flags & DMAR_PGF_WAITOK) == 0, ("failed waitable pte alloc %p", ctx)); - if (sf != NULL) { - dmar_unmap_pgtbl(sf, - DMAR_IS_COHERENT(ctx->dmar)); - } + if (sf != NULL) + dmar_unmap_pgtbl(sf); ctx_unmap_buf_locked(ctx, base1, base - base1, flags); TD_PINNED_ASSERT; return (ENOMEM); } dmar_pte_store(&pte->pte, VM_PAGE_TO_PHYS(ma[pi]) | pflags | (superpage ? DMAR_PTE_SP : 0)); + dmar_flush_pte_to_ram(ctx->dmar, pte); sf_buf_page(sf)->wire_count += 1; } if (sf != NULL) - dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(ctx->dmar)); + dmar_unmap_pgtbl(sf); TD_PINNED_ASSERT; return (0); } @@ -567,9 +567,10 @@ ctx_unmap_clear_pte(struct dmar_ctx *ctx vm_page_t m; dmar_pte_clear(&pte->pte); + dmar_flush_pte_to_ram(ctx->dmar, pte); m = sf_buf_page(*sf); if (free_sf) { - dmar_unmap_pgtbl(*sf, DMAR_IS_COHERENT(ctx->dmar)); + dmar_unmap_pgtbl(*sf); *sf = NULL; } m->wire_count--; @@ -651,7 +652,7 @@ ctx_unmap_buf_locked(struct dmar_ctx *ct (uintmax_t)base, (uintmax_t)size, (uintmax_t)pg_sz)); } if (sf != NULL) - dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(ctx->dmar)); + dmar_unmap_pgtbl(sf); /* * See 11.1 Write Buffer Flushing for an explanation why RWBF * can be ignored there. Modified: head/sys/x86/iommu/intel_utils.c ============================================================================== --- head/sys/x86/iommu/intel_utils.c Sun Jan 11 20:22:12 2015 (r277022) +++ head/sys/x86/iommu/intel_utils.c Sun Jan 11 20:27:15 2015 (r277023) @@ -354,20 +354,46 @@ dmar_map_pgtbl(vm_object_t obj, vm_pinde } void -dmar_unmap_pgtbl(struct sf_buf *sf, bool coherent) +dmar_unmap_pgtbl(struct sf_buf *sf) { - vm_page_t m; - m = sf_buf_page(sf); sf_buf_free(sf); sched_unpin(); +} + +static void +dmar_flush_transl_to_ram(struct dmar_unit *unit, void *dst, size_t sz) +{ + if (DMAR_IS_COHERENT(unit)) + return; /* * If DMAR does not snoop paging structures accesses, flush * CPU cache to memory. */ - if (!coherent) - pmap_invalidate_cache_pages(&m, 1); + pmap_invalidate_cache_range((uintptr_t)dst, (uintptr_t)dst + sz, + TRUE); +} + +void +dmar_flush_pte_to_ram(struct dmar_unit *unit, dmar_pte_t *dst) +{ + + dmar_flush_transl_to_ram(unit, dst, sizeof(*dst)); +} + +void +dmar_flush_ctx_to_ram(struct dmar_unit *unit, dmar_ctx_entry_t *dst) +{ + + dmar_flush_transl_to_ram(unit, dst, sizeof(*dst)); +} + +void +dmar_flush_root_to_ram(struct dmar_unit *unit, dmar_root_entry_t *dst) +{ + + dmar_flush_transl_to_ram(unit, dst, sizeof(*dst)); } /*