[PATCH] drm: Optimise drm_ioctl() for small user args

[Chris Wilson]

When looking at simple ioctls coupled with conveniently small user
parameters, the overhead of the syscall and drm_ioctl() present large
low hanging fruit. Profiling trivial microbenchmarks around
i915_gem_busy_ioctl, the low hanging fruit comprises of the call to
copy_user(). Those calls are only inlined by the macro where the
constant is known at compile-time, but the ioctl argument size depends
on the ioctl number. To help the compiler, explicitly add switches for
the small sizes that expand to simple moves to/from user. Doing the
multiple inlines does add significant code bloat, so it is very
debatable as to its value. Back to the trivial, but frequently used,
example of i915_gem_busy_ioctl() on a Broadwell avoiding the call gives
us a 15-25% improvement:

			 before		  after
	single		100.173ns	 84.496ns
	parallel	204.275ns	152.957ns

Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>
---
 drivers/gpu/drm/drm_ioctl.c | 103 +++++++++++++++++++++++++++++++++-----------
 1 file changed, 77 insertions(+), 26 deletions(-)

diff --git a/drivers/gpu/drm/drm_ioctl.c b/drivers/gpu/drm/drm_ioctl.c
index 865e3ee4d743..3fb61985ba08 100644
--- a/drivers/gpu/drm/drm_ioctl.c
+++ b/drivers/gpu/drm/drm_ioctl.c
@@ -716,8 +716,14 @@ long drm_ioctl(struct file *filp,
 	drm_ioctl_t *func;
 	unsigned int nr = DRM_IOCTL_NR(cmd);
 	int retcode = -EINVAL;
-	char stack_kdata[128];
-	char *kdata = NULL;
+	union {
+		u8 k8;
+		u16 k16;
+		u32 k32;
+		u64 k64;
+		char kdata[128];
+	} stack;
+	char *kdata = stack.kdata;
 	unsigned int in_size, out_size, drv_size, ksize;
 	bool is_driver_ioctl;
 
@@ -731,12 +737,12 @@ long drm_ioctl(struct file *filp,
 	if (is_driver_ioctl) {
 		/* driver ioctl */
 		if (nr - DRM_COMMAND_BASE >= dev->driver->num_ioctls)
-			goto err_i1;
+			goto err_invalid_ioctl;
 		ioctl = &dev->driver->ioctls[nr - DRM_COMMAND_BASE];
 	} else {
 		/* core ioctl */
 		if (nr >= DRM_CORE_IOCTL_COUNT)
-			goto err_i1;
+			goto err_invalid_ioctl;
 		ioctl = &drm_ioctls[nr];
 	}
 
@@ -759,27 +765,45 @@ long drm_ioctl(struct file *filp,
 	if (unlikely(!func)) {
 		DRM_DEBUG("no function\n");
 		retcode = -EINVAL;
-		goto err_i1;
+		goto out;
 	}
 
 	retcode = drm_ioctl_permit(ioctl->flags, file_priv);
 	if (unlikely(retcode))
-		goto err_i1;
+		goto out;
 
-	if (ksize <= sizeof(stack_kdata)) {
-		kdata = stack_kdata;
-	} else {
-		kdata = kmalloc(ksize, GFP_KERNEL);
-		if (!kdata) {
-			retcode = -ENOMEM;
-			goto err_i1;
-		}
+	if (unlikely(!access_ok(VERIFY_READ, arg, ksize))) {
+		retcode = -EFAULT;
+		goto out;
 	}
 
-	if (copy_from_user(kdata, (void __user *)arg, in_size) != 0) {
-		retcode = -EFAULT;
-		goto err_i1;
+	user_access_begin();
+	switch (ksize) {
+	case 1:
+		unsafe_get_user(stack.k8, (u8 *)arg, err_invalid_user);
+		break;
+	case 2:
+		unsafe_get_user(stack.k16, (u16 *)arg, err_invalid_user);
+		break;
+	case 4:
+		unsafe_get_user(stack.k32, (u32 *)arg, err_invalid_user);
+		break;
+	case 8:
+		unsafe_get_user(stack.k64, (u64 *)arg, err_invalid_user);
+		break;
+
+	default:
+		if (ksize > sizeof(stack.kdata))
+			kdata = kmalloc(ksize, GFP_KERNEL);
+		if (!kdata)
+			retcode = -ENOMEM;
+		else if (__copy_from_user(kdata, (void __user *)arg, in_size))
+			goto err_invalid_user;
+		break;
 	}
+	user_access_end();
+	if (unlikely(retcode))
+		goto out;
 
 	if (ksize > in_size)
 		memset(kdata + in_size, 0, ksize - in_size);
@@ -794,21 +818,48 @@ long drm_ioctl(struct file *filp,
 		mutex_unlock(&drm_global_mutex);
 	}
 
-	if (copy_to_user((void __user *)arg, kdata, out_size) != 0)
-		retcode = -EFAULT;
+	user_access_begin();
+	switch (out_size) {
+	case 0:
+		break;
+	case 1:
+		unsafe_put_user(stack.k8, (u8 *)arg, err_invalid_user);
+		break;
+	case 2:
+		unsafe_put_user(stack.k16, (u16 *)arg, err_invalid_user);
+		break;
+	case 4:
+		unsafe_put_user(stack.k32, (u32 *)arg, err_invalid_user);
+		break;
+	case 8:
+		unsafe_put_user(stack.k64, (u64 *)arg, err_invalid_user);
+		break;
 
-      err_i1:
-	if (!ioctl)
-		DRM_DEBUG("invalid ioctl: pid=%d, dev=0x%lx, auth=%d, cmd=0x%02x, nr=0x%02x\n",
-			  task_pid_nr(current),
-			  (long)old_encode_dev(file_priv->minor->kdev->devt),
-			  file_priv->authenticated, cmd, nr);
+	default:
+		if (__copy_to_user((void __user *)arg, kdata, out_size))
+			goto err_invalid_user;
+	}
+	user_access_end();
 
-	if (kdata != stack_kdata)
+out:
+	if (kdata != stack.kdata)
 		kfree(kdata);
 	if (retcode)
 		DRM_DEBUG("ret = %d\n", retcode);
 	return retcode;
+
+err_invalid_ioctl:
+	DRM_DEBUG("invalid ioctl: pid=%d, dev=0x%lx, auth=%d, cmd=0x%02x, nr=0x%02x\n",
+		  task_pid_nr(current),
+		  (long)old_encode_dev(file_priv->minor->kdev->devt),
+		  file_priv->authenticated, cmd, nr);
+	goto out;
+
+err_invalid_user:
+	user_access_end();
+	retcode = -EFAULT;
+	goto out;
+
 }
 EXPORT_SYMBOL(drm_ioctl);
 
-- 
2.11.0