[Intel-gfx] [PATCH 13/15] drm/i915: Implement GTT variants of pread

[Chris Wilson]

Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>
---
 drivers/gpu/drm/i915/i915_gem.c |  184 ++++++++++++++++++++++++++++++++++++--
 1 files changed, 174 insertions(+), 10 deletions(-)

diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 37a8a29..8f60bc5 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -542,6 +542,146 @@ out:
 	return ret;
 }
 
+static int
+i915_gem_gtt_pread_fast(struct drm_device *dev,
+			struct drm_i915_gem_object *obj,
+			struct drm_i915_gem_pread *args,
+			struct drm_file *file_priv)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	ssize_t remain;
+	loff_t offset, page_base;
+	char __user *user_data;
+	int page_offset, page_length;
+
+	user_data = (char __user *) (uintptr_t) args->data_ptr;
+	remain = args->size;
+
+	offset = obj->gtt_offset + args->offset;
+
+	while (remain > 0) {
+		u8 __iomem *vaddr;
+		unsigned long unwritten;
+
+		/* Operation in this page
+		 *
+		 * page_base = page offset within aperture
+		 * page_offset = offset within page
+		 * page_length = bytes to copy for this page
+		 */
+		page_base = offset & PAGE_MASK;
+		page_offset = offset_in_page(offset);
+		page_length = remain;
+		if ((page_offset + remain) > PAGE_SIZE)
+			page_length = PAGE_SIZE - page_offset;
+
+		/* If we get a fault while copying data, then (presumably) our
+		 * source page isn't available.  Return the error and we'll
+		 * retry in the slow path.
+		 */
+
+		vaddr = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
+						 page_base);
+		unwritten = __copy_to_user_inatomic(user_data,
+						    vaddr + page_offset,
+						    page_length);
+		io_mapping_unmap_atomic(vaddr);
+
+		if (unwritten)
+			return -EFAULT;
+
+		remain -= page_length;
+		user_data += page_length;
+		offset += page_length;
+	}
+
+	return 0;
+}
+
+static int
+i915_gem_gtt_pread_slow(struct drm_device *dev,
+                       struct drm_i915_gem_object *obj,
+                       struct drm_i915_gem_pread *args,
+                       struct drm_file *file_priv)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	ssize_t remain;
+	loff_t gtt_page_base, offset;
+	loff_t first_data_page, last_data_page;
+	int num_pages, i;
+	struct page **user_pages;
+	int gtt_page_offset, user_page_offset, user_page_index, page_length;
+	int ret;
+	uint64_t data_ptr = args->data_ptr;
+
+	remain = args->size;
+
+	/* Pin the user pages containing the data.  We can't fault while
+	 * holding the struct mutex, and all of the pwrite implementations
+	 * want to hold it while dereferencing the user data.
+	 */
+	first_data_page = data_ptr / PAGE_SIZE;
+	last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
+	num_pages = last_data_page - first_data_page + 1;
+
+	ret = i915_gem_get_user_pages(dev, data_ptr, false,
+				      &num_pages, &user_pages);
+	if (ret)
+		goto out;
+
+	ret = i915_gem_object_set_to_gtt_domain(obj, false);
+	if (ret)
+		goto out;
+
+	offset = obj->gtt_offset + args->offset;
+
+	while (remain > 0) {
+		u8 __iomem *src_vaddr;
+		u8 *dst_vaddr;
+
+		/* Operation in this page
+		 *
+		 * gtt_page_base = page offset within aperture
+		 * gtt_page_offset = offset within page in aperture
+		 * user_page_index = page number in get_user_pages return
+		 * user_page_offset = offset with user_page_index page.
+		 * page_length = bytes to copy for this page
+		 */
+		gtt_page_base = offset & PAGE_MASK;
+		gtt_page_offset = offset_in_page(offset);
+		user_page_index = data_ptr / PAGE_SIZE - first_data_page;
+		user_page_offset = offset_in_page(data_ptr);
+
+		page_length = remain;
+		if ((gtt_page_offset + page_length) > PAGE_SIZE)
+			page_length = PAGE_SIZE - gtt_page_offset;
+		if ((user_page_offset + page_length) > PAGE_SIZE)
+			page_length = PAGE_SIZE - user_page_offset;
+
+		src_vaddr = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
+						     gtt_page_base);
+		dst_vaddr = kmap_atomic(user_pages[user_page_index]);
+
+		memcpy_fromio(dst_vaddr + user_page_offset,
+			      src_vaddr + gtt_page_offset,
+			      page_length);
+
+		kunmap_atomic(dst_vaddr);
+		io_mapping_unmap_atomic(src_vaddr);
+
+		remain -= page_length;
+		offset += page_length;
+		data_ptr += page_length;
+	}
+
+out:
+	for (i = 0; i < num_pages; i++)
+		page_cache_release(user_pages[i]);
+	drm_free_large(user_pages);
+
+	return ret;
+}
+
 /**
  * Reads data from the object referenced by handle.
  *
@@ -587,17 +727,41 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 
 	trace_i915_gem_object_pread(obj, args->offset, args->size);
 
-	ret = i915_gem_object_set_cpu_read_domain_range(obj,
-							args->offset,
-							args->size);
-	if (ret)
-		goto out;
+	if (obj->gtt_space &&
+	    obj->map_and_fenceable &&
+	    obj->cache_level == I915_CACHE_NONE &&
+	    (obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
+		ret = i915_gem_object_pin(obj, 0, true);
+		if (ret)
+			goto out;
+
+		ret = i915_gem_object_set_to_gtt_domain(obj, false);
+		if (ret)
+			goto out_unpin;
 
-	ret = -EFAULT;
-	if (!i915_gem_object_needs_bit17_swizzle(obj))
-		ret = i915_gem_shmem_pread_fast(dev, obj, args, file);
-	if (ret == -EFAULT)
-		ret = i915_gem_shmem_pread_slow(dev, obj, args, file);
+		ret = i915_gem_object_put_fence(obj);
+		if (ret)
+			goto out_unpin;
+
+		ret = i915_gem_gtt_pread_fast(dev, obj, args, file);
+		if (ret == -EFAULT)
+			ret = i915_gem_gtt_pread_slow(dev, obj, args, file);
+
+out_unpin:
+		i915_gem_object_unpin(obj);
+	} else {
+		ret = i915_gem_object_set_cpu_read_domain_range(obj,
+								args->offset,
+								args->size);
+		if (ret)
+			goto out;
+
+		ret = -EFAULT;
+		if (!i915_gem_object_needs_bit17_swizzle(obj))
+			ret = i915_gem_shmem_pread_fast(dev, obj, args, file);
+		if (ret == -EFAULT)
+			ret = i915_gem_shmem_pread_slow(dev, obj, args, file);
+	}
 
 out:
 	drm_gem_object_unreference(&obj->base);
-- 
1.7.4.1