Date: Tue, 28 Jul 2015 21:47:38 +0000 (UTC) From: Jean-Sebastien Pedron <dumbbell@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r285988 - head/sys/dev/drm2/i915 Message-ID: <201507282147.t6SLlcNG050640@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: dumbbell Date: Tue Jul 28 21:47:37 2015 New Revision: 285988 URL: https://svnweb.freebsd.org/changeset/base/285988 Log: drm/i915: Sort functions in i915_gem.c to match Linux 3.8's ordering While here, reduce the style diff with Linux. There is no functional change. The goal is to ease the future update to Linux 3.8's i915 driver. MFC after: 2 months Modified: head/sys/dev/drm2/i915/i915_drv.h head/sys/dev/drm2/i915/i915_gem.c head/sys/dev/drm2/i915/i915_gem_gtt.c Modified: head/sys/dev/drm2/i915/i915_drv.h ============================================================================== --- head/sys/dev/drm2/i915/i915_drv.h Tue Jul 28 21:43:23 2015 (r285987) +++ head/sys/dev/drm2/i915/i915_drv.h Tue Jul 28 21:47:37 2015 (r285988) @@ -1151,8 +1151,6 @@ void i915_disable_pipestat(drm_i915_priv void i915_destroy_error_state(struct drm_device *dev); /* i915_gem.c */ -int i915_gem_create(struct drm_file *file, struct drm_device *dev, uint64_t size, - uint32_t *handle_p); int i915_gem_init_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_create_ioctl(struct drm_device *dev, void *data, @@ -1243,7 +1241,6 @@ int i915_gem_object_finish_gpu(struct dr int i915_gem_flush_ring(struct intel_ring_buffer *ring, uint32_t invalidate_domains, uint32_t flush_domains); void i915_gem_release_mmap(struct drm_i915_gem_object *obj); -int i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj); int i915_gem_object_sync(struct drm_i915_gem_object *obj, struct intel_ring_buffer *to); int i915_gem_object_put_fence(struct drm_i915_gem_object *obj); Modified: head/sys/dev/drm2/i915/i915_gem.c ============================================================================== --- head/sys/dev/drm2/i915/i915_gem.c Tue Jul 28 21:43:23 2015 (r285987) +++ head/sys/dev/drm2/i915/i915_gem.c Tue Jul 28 21:47:37 2015 (r285988) @@ -60,6 +60,7 @@ __FBSDID("$FreeBSD$"); #include <dev/drm2/i915/i915_drv.h> #include <dev/drm2/i915/intel_drv.h> #include <dev/drm2/i915/intel_ringbuffer.h> + #include <sys/resourcevar.h> #include <sys/sched.h> #include <sys/sf_buf.h> @@ -69,38 +70,40 @@ __FBSDID("$FreeBSD$"); #include <machine/md_var.h> -static void i915_gem_object_flush_cpu_write_domain( - struct drm_i915_gem_object *obj); -static uint32_t i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, - int tiling_mode); -static uint32_t i915_gem_get_gtt_alignment(struct drm_device *dev, - uint32_t size, int tiling_mode); -static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, - unsigned alignment, bool map_and_fenceable); -static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj, - int flags); -static void i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj); -static void i915_gem_object_put_pages_range(struct drm_i915_gem_object *obj, - off_t start, off_t end); +#define __user +#define __force +#define __iomem +#define __must_check +#define to_user_ptr(x) ((void *)(uintptr_t)(x)) +#define offset_in_page(x) ((x) & PAGE_MASK) +#define page_to_phys(x) VM_PAGE_TO_PHYS(x) + +static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj); +static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj); +static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, + unsigned alignment, + bool map_and_fenceable); +static int i915_gem_phys_pwrite(struct drm_device *dev, + struct drm_i915_gem_object *obj, + struct drm_i915_gem_pwrite *args, + struct drm_file *file); + +static void i915_gem_write_fence(struct drm_device *dev, int reg, + struct drm_i915_gem_object *obj); +static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, + struct drm_i915_fence_reg *fence, + bool enable); + +static void i915_gem_lowmem(void *arg); +static void i915_gem_object_truncate(struct drm_i915_gem_object *obj); + static int i915_gem_object_get_pages_range(struct drm_i915_gem_object *obj, off_t start, off_t end); -static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj); -static void i915_gem_object_truncate(struct drm_i915_gem_object *obj); -static int i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj); -static bool i915_gem_object_is_inactive(struct drm_i915_gem_object *obj); -static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj); +static void i915_gem_object_put_pages_range(struct drm_i915_gem_object *obj, + off_t start, off_t end); + static vm_page_t i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex, bool *fresh); -static void i915_gem_process_flushing_list(struct intel_ring_buffer *ring, - uint32_t flush_domains); -static void i915_gem_reset_fences(struct drm_device *dev); -static void i915_gem_retire_task_handler(void *arg, int pending); -static void i915_gem_lowmem(void *arg); -static void i915_gem_write_fence(struct drm_device *dev, int reg, - struct drm_i915_gem_object *obj); -static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno, - bool interruptible); -static int i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno); MALLOC_DEFINE(DRM_I915_GEM, "i915gem", "Allocations from i915 gem"); long i915_gem_wired_pages_cnt; @@ -131,18 +134,17 @@ static inline void i915_gem_object_fence obj->fence_reg = I915_FENCE_REG_NONE; } -static void -i915_gem_info_add_obj(struct drm_i915_private *dev_priv, size_t size) +/* some bookkeeping */ +static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv, + size_t size) { - dev_priv->mm.object_count++; dev_priv->mm.object_memory += size; } -static void -i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, size_t size) +static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, + size_t size) { - dev_priv->mm.object_count--; dev_priv->mm.object_memory -= size; } @@ -150,10 +152,9 @@ i915_gem_info_remove_obj(struct drm_i915 static int i915_gem_wait_for_error(struct drm_device *dev) { - struct drm_i915_private *dev_priv; + struct drm_i915_private *dev_priv = dev->dev_private; int ret; - dev_priv = dev->dev_private; if (!atomic_load_acq_int(&dev_priv->mm.wedged)) return (0); @@ -171,150 +172,61 @@ i915_gem_wait_for_error(struct drm_devic mtx_unlock(&dev_priv->error_completion_lock); if (atomic_load_acq_int(&dev_priv->mm.wedged)) { + /* GPU is hung, bump the completion count to account for + * the token we just consumed so that we never hit zero and + * end up waiting upon a subsequent completion event that + * will never happen. + */ mtx_lock(&dev_priv->error_completion_lock); dev_priv->error_completion++; mtx_unlock(&dev_priv->error_completion_lock); } - return (0); + return 0; } -int -i915_mutex_lock_interruptible(struct drm_device *dev) +int i915_mutex_lock_interruptible(struct drm_device *dev) { - struct drm_i915_private *dev_priv; int ret; - dev_priv = dev->dev_private; ret = i915_gem_wait_for_error(dev); - if (ret != 0) - return (ret); + if (ret) + return ret; /* * interruptible shall it be. might indeed be if dev_lock is * changed to sx */ - ret = sx_xlock_sig(&dev->dev_struct_lock); - if (ret != 0) - return (-ret); - - return (0); -} - - -void -i915_gem_free_object(struct drm_gem_object *gem_obj) -{ - struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); - struct drm_device *dev; - drm_i915_private_t *dev_priv; - - dev = obj->base.dev; - dev_priv = dev->dev_private; - - CTR1(KTR_DRM, "object_destroy_tail %p", obj); - - if (obj->phys_obj) - i915_gem_detach_phys_object(dev, obj); - - obj->pin_count = 0; - if (i915_gem_object_unbind(obj) == -ERESTARTSYS) { - bool was_interruptible; - - was_interruptible = dev_priv->mm.interruptible; - dev_priv->mm.interruptible = false; - - if (i915_gem_object_unbind(obj)) - printf("i915_gem_free_object: unbind\n"); - - dev_priv->mm.interruptible = was_interruptible; - } - - drm_gem_free_mmap_offset(&obj->base); - drm_gem_object_release(&obj->base); - i915_gem_info_remove_obj(dev_priv, obj->base.size); - - free(obj->bit_17, DRM_I915_GEM); - free(obj, DRM_I915_GEM); -} - -static void -init_ring_lists(struct intel_ring_buffer *ring) -{ + ret = -sx_xlock_sig(&dev->dev_struct_lock); + if (ret) + return ret; - INIT_LIST_HEAD(&ring->active_list); - INIT_LIST_HEAD(&ring->request_list); - INIT_LIST_HEAD(&ring->gpu_write_list); + return 0; } -void -i915_gem_load(struct drm_device *dev) +static bool +i915_gem_object_is_inactive(struct drm_i915_gem_object *obj) { - drm_i915_private_t *dev_priv; - int i; - - dev_priv = dev->dev_private; - - INIT_LIST_HEAD(&dev_priv->mm.active_list); - INIT_LIST_HEAD(&dev_priv->mm.flushing_list); - INIT_LIST_HEAD(&dev_priv->mm.inactive_list); - INIT_LIST_HEAD(&dev_priv->mm.fence_list); - INIT_LIST_HEAD(&dev_priv->mm.gtt_list); - for (i = 0; i < I915_NUM_RINGS; i++) - init_ring_lists(&dev_priv->rings[i]); - for (i = 0; i < I915_MAX_NUM_FENCES; i++) - INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); - TIMEOUT_TASK_INIT(dev_priv->tq, &dev_priv->mm.retire_task, 0, - i915_gem_retire_task_handler, dev_priv); - dev_priv->error_completion = 0; - - /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ - if (IS_GEN3(dev)) { - I915_WRITE(MI_ARB_STATE, - _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE)); - } - - dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL; - - /* Old X drivers will take 0-2 for front, back, depth buffers */ - if (!drm_core_check_feature(dev, DRIVER_MODESET)) - dev_priv->fence_reg_start = 3; - - if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || - IS_G33(dev)) - dev_priv->num_fence_regs = 16; - else - dev_priv->num_fence_regs = 8; - - /* Initialize fence registers to zero */ - i915_gem_reset_fences(dev); - i915_gem_detect_bit_6_swizzle(dev); - dev_priv->mm.interruptible = true; - - dev_priv->mm.i915_lowmem = EVENTHANDLER_REGISTER(vm_lowmem, - i915_gem_lowmem, dev, EVENTHANDLER_PRI_ANY); + return !obj->active; } int i915_gem_init_ioctl(struct drm_device *dev, void *data, - struct drm_file *file) + struct drm_file *file) { - struct drm_i915_gem_init *args; - drm_i915_private_t *dev_priv; - int error; + struct drm_i915_gem_init *args = data; + drm_i915_private_t *dev_priv = dev->dev_private; + int ret; if (drm_core_check_feature(dev, DRIVER_MODESET)) return -ENODEV; - dev_priv = dev->dev_private; - args = data; - if (args->gtt_start >= args->gtt_end || (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1)) - return (-EINVAL); + return -EINVAL; if (mtx_initialized(&dev_priv->mm.gtt_space.unused_lock)) - return (-EBUSY); + return -EBUSY; /* GEM with user mode setting was never supported on ilk and later. */ if (INTEL_INFO(dev)->gen >= 5) @@ -325,423 +237,444 @@ i915_gem_init_ioctl(struct drm_device *d * against. */ DRM_LOCK(dev); - error = i915_gem_init_global_gtt(dev, args->gtt_start, - args->gtt_end, args->gtt_end); + ret = i915_gem_init_global_gtt(dev, args->gtt_start, + args->gtt_end, args->gtt_end); DRM_UNLOCK(dev); - return (error); + + return ret; } int -i915_gem_idle(struct drm_device *dev) +i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, + struct drm_file *file) { - drm_i915_private_t *dev_priv; - int ret; + struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_gem_get_aperture *args = data; + struct drm_i915_gem_object *obj; + size_t pinned; + pinned = 0; DRM_LOCK(dev); + list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) + if (obj->pin_count) + pinned += obj->gtt_space->size; + DRM_UNLOCK(dev); - dev_priv = dev->dev_private; - if (dev_priv->mm.suspended) { - DRM_UNLOCK(dev); - return (0); - } + args->aper_size = dev_priv->mm.gtt_total; + args->aper_available_size = args->aper_size - pinned; - ret = i915_gpu_idle(dev); - if (ret != 0) { - DRM_UNLOCK(dev); - return (ret); - } - i915_gem_retire_requests(dev); + return 0; +} - /* Under UMS, be paranoid and evict. */ - if (!drm_core_check_feature(dev, DRIVER_MODESET)) { - ret = i915_gem_evict_everything(dev, false); - if (ret != 0) { - DRM_UNLOCK(dev); - return ret; - } - } +static int +i915_gem_create(struct drm_file *file, + struct drm_device *dev, + uint64_t size, + uint32_t *handle_p) +{ + struct drm_i915_gem_object *obj; + int ret; + u32 handle; - i915_gem_reset_fences(dev); + size = roundup(size, PAGE_SIZE); + if (size == 0) + return -EINVAL; - /* Hack! Don't let anybody do execbuf while we don't control the chip. - * We need to replace this with a semaphore, or something. - * And not confound mm.suspended! - */ - dev_priv->mm.suspended = 1; - callout_stop(&dev_priv->hangcheck_timer); + /* Allocate the new object */ + obj = i915_gem_alloc_object(dev, size); + if (obj == NULL) + return -ENOMEM; - i915_kernel_lost_context(dev); - i915_gem_cleanup_ringbuffer(dev); + ret = drm_gem_handle_create(file, &obj->base, &handle); + if (ret) { + drm_gem_object_release(&obj->base); + i915_gem_info_remove_obj(dev->dev_private, obj->base.size); + free(obj, DRM_I915_GEM); + return ret; + } - DRM_UNLOCK(dev); + /* drop reference from allocate - handle holds it now */ + drm_gem_object_unreference(&obj->base); + CTR2(KTR_DRM, "object_create %p %x", obj, size); - /* Cancel the retire work handler, which should be idle now. */ - taskqueue_cancel_timeout(dev_priv->tq, &dev_priv->mm.retire_task, NULL); - return (ret); + *handle_p = handle; + return 0; } -void -i915_gem_init_swizzling(struct drm_device *dev) +int +i915_gem_dumb_create(struct drm_file *file, + struct drm_device *dev, + struct drm_mode_create_dumb *args) { - drm_i915_private_t *dev_priv; - - dev_priv = dev->dev_private; - - if (INTEL_INFO(dev)->gen < 5 || - dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) - return; - - I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | - DISP_TILE_SURFACE_SWIZZLING); - - if (IS_GEN5(dev)) - return; - + /* have to work out size/pitch and return them */ + args->pitch = roundup2(args->width * ((args->bpp + 7) / 8), 64); + args->size = args->pitch * args->height; + return i915_gem_create(file, dev, + args->size, &args->handle); +} - I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL); - if (IS_GEN6(dev)) - I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB)); - else - I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB)); +int i915_gem_dumb_destroy(struct drm_file *file, + struct drm_device *dev, + uint32_t handle) +{ + return drm_gem_handle_delete(file, handle); } -void i915_gem_init_ppgtt(struct drm_device *dev) +/** + * Creates a new mm object and returns a handle to it. + */ +int +i915_gem_create_ioctl(struct drm_device *dev, void *data, + struct drm_file *file) { - drm_i915_private_t *dev_priv; - struct i915_hw_ppgtt *ppgtt; - uint32_t pd_offset, pd_entry; - vm_paddr_t pt_addr; - struct intel_ring_buffer *ring; - u_int first_pd_entry_in_global_pt, i; + struct drm_i915_gem_create *args = data; - dev_priv = dev->dev_private; - ppgtt = dev_priv->mm.aliasing_ppgtt; - if (ppgtt == NULL) - return; + return i915_gem_create(file, dev, + args->size, &args->handle); +} - first_pd_entry_in_global_pt = 512 * 1024 - I915_PPGTT_PD_ENTRIES; - for (i = 0; i < ppgtt->num_pd_entries; i++) { - pt_addr = VM_PAGE_TO_PHYS(ppgtt->pt_pages[i]); - pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr); - pd_entry |= GEN6_PDE_VALID; - intel_gtt_write(first_pd_entry_in_global_pt + i, pd_entry); - } - intel_gtt_read_pte(first_pd_entry_in_global_pt); - - pd_offset = ppgtt->pd_offset; - pd_offset /= 64; /* in cachelines, */ - pd_offset <<= 16; - - if (INTEL_INFO(dev)->gen == 6) { - uint32_t ecochk, gab_ctl, ecobits; - - ecobits = I915_READ(GAC_ECO_BITS); - I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B); - - gab_ctl = I915_READ(GAB_CTL); - I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT); - - ecochk = I915_READ(GAM_ECOCHK); - I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | - ECOCHK_PPGTT_CACHE64B); - I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); - } else if (INTEL_INFO(dev)->gen >= 7) { - I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B); - /* GFX_MODE is per-ring on gen7+ */ - } +static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj) +{ + drm_i915_private_t *dev_priv = obj->base.dev->dev_private; - for_each_ring(ring, dev_priv, i) { - if (INTEL_INFO(dev)->gen >= 7) - I915_WRITE(RING_MODE_GEN7(ring), - _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); + return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && + obj->tiling_mode != I915_TILING_NONE; +} - I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); - I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset); - } +static inline int +__copy_to_user_inatomic(void __user *to, const void *from, unsigned n) +{ + return (copyout_nofault(from, to, n) != 0 ? n : 0); } - -int -i915_gem_init_hw(struct drm_device *dev) +static inline unsigned long +__copy_from_user_inatomic_nocache(void *to, const void __user *from, + unsigned long n) { - drm_i915_private_t *dev_priv; - int ret; - - dev_priv = dev->dev_private; - i915_gem_init_swizzling(dev); + /* + * XXXKIB. Equivalent Linux function is implemented using + * MOVNTI for aligned moves. For unaligned head and tail, + * normal move is performed. As such, it is not incorrect, if + * only somewhat slower, to use normal copyin. All uses + * except shmem_pwrite_fast() have the destination mapped WC. + */ + return ((copyin_nofault(__DECONST(void *, from), to, n) != 0 ? n : 0)); +} +static inline int +fault_in_multipages_readable(const char __user *uaddr, int size) +{ + char c; + int ret = 0; + const char __user *end = uaddr + size - 1; - ret = intel_init_render_ring_buffer(dev); - if (ret != 0) - return (ret); + if (unlikely(size == 0)) + return ret; - if (HAS_BSD(dev)) { - ret = intel_init_bsd_ring_buffer(dev); + while (uaddr <= end) { + ret = -copyin(uaddr, &c, 1); if (ret != 0) - goto cleanup_render_ring; + return -EFAULT; + uaddr += PAGE_SIZE; } - if (HAS_BLT(dev)) { - ret = intel_init_blt_ring_buffer(dev); - if (ret != 0) - goto cleanup_bsd_ring; + /* Check whether the range spilled into the next page. */ + if (((unsigned long)uaddr & ~PAGE_MASK) == + ((unsigned long)end & ~PAGE_MASK)) { + ret = -copyin(end, &c, 1); } - dev_priv->next_seqno = 1; - i915_gem_context_init(dev); - i915_gem_init_ppgtt(dev); - return (0); - -cleanup_bsd_ring: - intel_cleanup_ring_buffer(&dev_priv->rings[VCS]); -cleanup_render_ring: - intel_cleanup_ring_buffer(&dev_priv->rings[RCS]); - return (ret); + return ret; } -static bool -intel_enable_ppgtt(struct drm_device *dev) +static inline int +fault_in_multipages_writeable(char __user *uaddr, int size) { - if (i915_enable_ppgtt >= 0) - return i915_enable_ppgtt; + int ret = 0; + char __user *end = uaddr + size - 1; - /* Disable ppgtt on SNB if VT-d is on. */ - if (INTEL_INFO(dev)->gen == 6 && intel_iommu_enabled) - return false; + if (unlikely(size == 0)) + return ret; - return true; -} + /* + * Writing zeroes into userspace here is OK, because we know that if + * the zero gets there, we'll be overwriting it. + */ + while (uaddr <= end) { + ret = subyte(uaddr, 0); + if (ret != 0) + return -EFAULT; + uaddr += PAGE_SIZE; + } -int i915_gem_init(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - unsigned long gtt_size, mappable_size; - int ret; + /* Check whether the range spilled into the next page. */ + if (((unsigned long)uaddr & ~PAGE_MASK) == + ((unsigned long)end & ~PAGE_MASK)) + ret = subyte(end, 0); - gtt_size = dev_priv->mm.gtt.gtt_total_entries << PAGE_SHIFT; - mappable_size = dev_priv->mm.gtt.gtt_mappable_entries << PAGE_SHIFT; + return ret; +} - DRM_LOCK(dev); - if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) { - /* PPGTT pdes are stolen from global gtt ptes, so shrink the - * aperture accordingly when using aliasing ppgtt. */ - gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE; +static inline int +__copy_to_user_swizzled(char __user *cpu_vaddr, + const char *gpu_vaddr, int gpu_offset, + int length) +{ + int ret, cpu_offset = 0; - i915_gem_init_global_gtt(dev, 0, mappable_size, gtt_size); + while (length > 0) { + int cacheline_end = roundup2(gpu_offset + 1, 64); + int this_length = min(cacheline_end - gpu_offset, length); + int swizzled_gpu_offset = gpu_offset ^ 64; - ret = i915_gem_init_aliasing_ppgtt(dev); - if (ret) { - DRM_UNLOCK(dev); - return ret; - } - } else { - /* Let GEM Manage all of the aperture. - * - * However, leave one page at the end still bound to the scratch - * page. There are a number of places where the hardware - * apparently prefetches past the end of the object, and we've - * seen multiple hangs with the GPU head pointer stuck in a - * batchbuffer bound at the last page of the aperture. One page - * should be enough to keep any prefetching inside of the - * aperture. - */ - i915_gem_init_global_gtt(dev, 0, mappable_size, - gtt_size); - } + ret = __copy_to_user(cpu_vaddr + cpu_offset, + gpu_vaddr + swizzled_gpu_offset, + this_length); + if (ret) + return ret + length; - ret = i915_gem_init_hw(dev); - DRM_UNLOCK(dev); - if (ret != 0) { - i915_gem_cleanup_aliasing_ppgtt(dev); - return (ret); + cpu_offset += this_length; + gpu_offset += this_length; + length -= this_length; } - /* Allow hardware batchbuffers unless told otherwise, but not for KMS. */ - if (!drm_core_check_feature(dev, DRIVER_MODESET)) - dev_priv->dri1.allow_batchbuffer = 1; return 0; } -int -i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, - struct drm_file *file) +static inline int +__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset, + const char __user *cpu_vaddr, + int length) { - struct drm_i915_private *dev_priv; - struct drm_i915_gem_get_aperture *args; - struct drm_i915_gem_object *obj; - size_t pinned; + int ret, cpu_offset = 0; - dev_priv = dev->dev_private; - args = data; + while (length > 0) { + int cacheline_end = roundup2(gpu_offset + 1, 64); + int this_length = min(cacheline_end - gpu_offset, length); + int swizzled_gpu_offset = gpu_offset ^ 64; - pinned = 0; - DRM_LOCK(dev); - list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) - if (obj->pin_count) - pinned += obj->gtt_space->size; - DRM_UNLOCK(dev); + ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset, + cpu_vaddr + cpu_offset, + this_length); + if (ret) + return ret + length; - args->aper_size = dev_priv->mm.gtt_total; - args->aper_available_size = args->aper_size - pinned; + cpu_offset += this_length; + gpu_offset += this_length; + length -= this_length; + } - return (0); + return 0; } -int -i915_gem_object_pin(struct drm_i915_gem_object *obj, uint32_t alignment, - bool map_and_fenceable) +/* Per-page copy function for the shmem pread fastpath. + * Flushes invalid cachelines before reading the target if + * needs_clflush is set. */ +static int +shmem_pread_fast(vm_page_t page, int shmem_page_offset, int page_length, + char __user *user_data, + bool page_do_bit17_swizzling, bool needs_clflush) { + char *vaddr; + struct sf_buf *sf; int ret; - if (obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT) - return (-EBUSY); - - if (obj->gtt_space != NULL) { - if ((alignment && obj->gtt_offset & (alignment - 1)) || - (map_and_fenceable && !obj->map_and_fenceable)) { - DRM_DEBUG("bo is already pinned with incorrect alignment:" - " offset=%x, req.alignment=%x, req.map_and_fenceable=%d," - " obj->map_and_fenceable=%d\n", - obj->gtt_offset, alignment, - map_and_fenceable, - obj->map_and_fenceable); - ret = i915_gem_object_unbind(obj); - if (ret != 0) - return (ret); - } - } + if (unlikely(page_do_bit17_swizzling)) + return -EINVAL; - if (obj->gtt_space == NULL) { - ret = i915_gem_object_bind_to_gtt(obj, alignment, - map_and_fenceable); - if (ret) - return (ret); + sched_pin(); + sf = sf_buf_alloc(page, SFB_NOWAIT | SFB_CPUPRIVATE); + if (sf == NULL) { + sched_unpin(); + return (-EFAULT); } + vaddr = (char *)sf_buf_kva(sf); + if (needs_clflush) + drm_clflush_virt_range(vaddr + shmem_page_offset, + page_length); + ret = __copy_to_user_inatomic(user_data, + vaddr + shmem_page_offset, + page_length); + sf_buf_free(sf); + sched_unpin(); - if (!obj->has_global_gtt_mapping && map_and_fenceable) - i915_gem_gtt_bind_object(obj, obj->cache_level); - - obj->pin_count++; - obj->pin_mappable |= map_and_fenceable; - - return 0; + return ret ? -EFAULT : 0; } -void -i915_gem_object_unpin(struct drm_i915_gem_object *obj) +static void +shmem_clflush_swizzled_range(char *addr, unsigned long length, + bool swizzled) { + if (unlikely(swizzled)) { + unsigned long start = (unsigned long) addr; + unsigned long end = (unsigned long) addr + length; - KASSERT(obj->pin_count != 0, ("zero pin count")); - KASSERT(obj->gtt_space != NULL, ("No gtt mapping")); + /* For swizzling simply ensure that we always flush both + * channels. Lame, but simple and it works. Swizzled + * pwrite/pread is far from a hotpath - current userspace + * doesn't use it at all. */ + start = rounddown2(start, 128); + end = roundup2(end, 128); - if (--obj->pin_count == 0) - obj->pin_mappable = false; -} + drm_clflush_virt_range((void *)start, end - start); + } else { + drm_clflush_virt_range(addr, length); + } -int -i915_gem_pin_ioctl(struct drm_device *dev, void *data, - struct drm_file *file) +} + +/* Only difference to the fast-path function is that this can handle bit17 + * and uses non-atomic copy and kmap functions. */ +static int +shmem_pread_slow(vm_page_t page, int shmem_page_offset, int page_length, + char __user *user_data, + bool page_do_bit17_swizzling, bool needs_clflush) { - struct drm_i915_gem_pin *args; - struct drm_i915_gem_object *obj; - struct drm_gem_object *gobj; + char *vaddr; + struct sf_buf *sf; int ret; - args = data; + sf = sf_buf_alloc(page, 0); + vaddr = (char *)sf_buf_kva(sf); + if (needs_clflush) + shmem_clflush_swizzled_range(vaddr + shmem_page_offset, + page_length, + page_do_bit17_swizzling); - ret = i915_mutex_lock_interruptible(dev); - if (ret != 0) - return ret; + if (page_do_bit17_swizzling) + ret = __copy_to_user_swizzled(user_data, + vaddr, shmem_page_offset, + page_length); + else + ret = __copy_to_user(user_data, + vaddr + shmem_page_offset, + page_length); + sf_buf_free(sf); - gobj = drm_gem_object_lookup(dev, file, args->handle); - if (gobj == NULL) { - ret = -ENOENT; - goto unlock; - } - obj = to_intel_bo(gobj); + return ret ? - EFAULT : 0; +} - if (obj->madv != I915_MADV_WILLNEED) { - DRM_ERROR("Attempting to pin a purgeable buffer\n"); - ret = -EINVAL; - goto out; - } +static int +i915_gem_shmem_pread(struct drm_device *dev, + struct drm_i915_gem_object *obj, + struct drm_i915_gem_pread *args, + struct drm_file *file) +{ + char __user *user_data; + ssize_t remain, sremain; + off_t offset, soffset; + int shmem_page_offset, page_length, ret = 0; + int obj_do_bit17_swizzling, page_do_bit17_swizzling; + int prefaulted = 0; + int needs_clflush = 0; - if (obj->pin_filp != NULL && obj->pin_filp != file) { - DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n", - args->handle); - ret = -EINVAL; - goto out; - } + user_data = to_user_ptr(args->data_ptr); + sremain = remain = args->size; - obj->user_pin_count++; - obj->pin_filp = file; - if (obj->user_pin_count == 1) { - ret = i915_gem_object_pin(obj, args->alignment, true); - if (ret != 0) - goto out; + obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); + + if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) { + /* If we're not in the cpu read domain, set ourself into the gtt + * read domain and manually flush cachelines (if required). This + * optimizes for the case when the gpu will dirty the data + * anyway again before the next pread happens. */ + needs_clflush = !cpu_cache_is_coherent(dev, obj->cache_level); + ret = i915_gem_object_set_to_gtt_domain(obj, false); + if (ret) + return ret; } - /* XXX - flush the CPU caches for pinned objects - * as the X server doesn't manage domains yet - */ - i915_gem_object_flush_cpu_write_domain(obj); - args->offset = obj->gtt_offset; -out: - drm_gem_object_unreference(&obj->base); -unlock: - DRM_UNLOCK(dev); - return (ret); -} + soffset = offset = args->offset; + ret = i915_gem_object_get_pages_range(obj, soffset, soffset + sremain); + if (ret) + return ret; -int -i915_gem_unpin_ioctl(struct drm_device *dev, void *data, - struct drm_file *file) -{ - struct drm_i915_gem_pin *args; - struct drm_i915_gem_object *obj; - int ret; + i915_gem_object_pin_pages(obj); - args = data; - ret = i915_mutex_lock_interruptible(dev); - if (ret != 0) - return (ret); + VM_OBJECT_WLOCK(obj->base.vm_obj); + for (vm_page_t page = vm_page_find_least(obj->base.vm_obj, + OFF_TO_IDX(offset));; page = vm_page_next(page)) { + VM_OBJECT_WUNLOCK(obj->base.vm_obj); - obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); - if (&obj->base == NULL) { - ret = -ENOENT; - goto unlock; - } + if (remain <= 0) + break; - if (obj->pin_filp != file) { - DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", - args->handle); - ret = -EINVAL; - goto out; - } - obj->user_pin_count--; - if (obj->user_pin_count == 0) { - obj->pin_filp = NULL; - i915_gem_object_unpin(obj); + /* Operation in this page + * + * shmem_page_offset = offset within page in shmem file + * page_length = bytes to copy for this page + */ + shmem_page_offset = offset_in_page(offset); + page_length = remain; + if ((shmem_page_offset + page_length) > PAGE_SIZE) + page_length = PAGE_SIZE - shmem_page_offset; + + page_do_bit17_swizzling = obj_do_bit17_swizzling && + (page_to_phys(page) & (1 << 17)) != 0; + + ret = shmem_pread_fast(page, shmem_page_offset, page_length, + user_data, page_do_bit17_swizzling, + needs_clflush); + if (ret == 0) *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201507282147.t6SLlcNG050640>