[Intel-gfx] [PATCH 2/9] drm/i915: Move GEM request routines to i915_gem_request.c

Mika Kuoppala mika.kuoppala at linux.intel.com
Fri Jul 15 10:43:25 UTC 2016


Chris Wilson <chris at chris-wilson.co.uk> writes:

> Migrate the request operations out of the main body of i915_gem.c and
> into their own C file for easier expansion.
>
> v2: Move __i915_add_request() across as well
>
> Signed-off-by: Chris Wilson <chris at chris-wilson.co.uk>

Acked-by: Mika Kuoppala <mika.kuoppala at intel.com>

> ---
>  drivers/gpu/drm/i915/Makefile           |   1 +
>  drivers/gpu/drm/i915/i915_drv.h         | 209 +---------
>  drivers/gpu/drm/i915/i915_gem.c         | 655 +------------------------------
>  drivers/gpu/drm/i915/i915_gem_request.c | 658 ++++++++++++++++++++++++++++++++
>  drivers/gpu/drm/i915/i915_gem_request.h | 238 ++++++++++++
>  5 files changed, 905 insertions(+), 856 deletions(-)
>  create mode 100644 drivers/gpu/drm/i915/i915_gem_request.c
>  create mode 100644 drivers/gpu/drm/i915/i915_gem_request.h
>
> diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
> index 75318ebb8d25..6092f0ea24df 100644
> --- a/drivers/gpu/drm/i915/Makefile
> +++ b/drivers/gpu/drm/i915/Makefile
> @@ -33,6 +33,7 @@ i915-y += i915_cmd_parser.o \
>  	  i915_gem_gtt.o \
>  	  i915_gem.o \
>  	  i915_gem_render_state.o \
> +	  i915_gem_request.o \
>  	  i915_gem_shrinker.o \
>  	  i915_gem_stolen.o \
>  	  i915_gem_tiling.o \
> diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
> index e73c0fc84c73..a4767c198413 100644
> --- a/drivers/gpu/drm/i915/i915_drv.h
> +++ b/drivers/gpu/drm/i915/i915_drv.h
> @@ -61,6 +61,7 @@
>  #include "i915_gem.h"
>  #include "i915_gem_gtt.h"
>  #include "i915_gem_render_state.h"
> +#include "i915_gem_request.h"
>  
>  #include "intel_gvt.h"
>  
> @@ -2365,171 +2366,6 @@ static inline struct scatterlist *__sg_next(struct scatterlist *sg)
>  	     (((__iter).curr += PAGE_SIZE) < (__iter).max) ||		\
>  	     ((__iter) = __sgt_iter(__sg_next((__iter).sgp), false), 0))
>  
> -/**
> - * Request queue structure.
> - *
> - * The request queue allows us to note sequence numbers that have been emitted
> - * and may be associated with active buffers to be retired.
> - *
> - * By keeping this list, we can avoid having to do questionable sequence
> - * number comparisons on buffer last_read|write_seqno. It also allows an
> - * emission time to be associated with the request for tracking how far ahead
> - * of the GPU the submission is.
> - *
> - * The requests are reference counted, so upon creation they should have an
> - * initial reference taken using kref_init
> - */
> -struct drm_i915_gem_request {
> -	struct kref ref;
> -
> -	/** On Which ring this request was generated */
> -	struct drm_i915_private *i915;
> -	struct intel_engine_cs *engine;
> -	struct intel_signal_node signaling;
> -
> -	 /** GEM sequence number associated with the previous request,
> -	  * when the HWS breadcrumb is equal to this the GPU is processing
> -	  * this request.
> -	  */
> -	u32 previous_seqno;
> -
> -	 /** GEM sequence number associated with this request,
> -	  * when the HWS breadcrumb is equal or greater than this the GPU
> -	  * has finished processing this request.
> -	  */
> -	u32 seqno;
> -
> -	/** Position in the ringbuffer of the start of the request */
> -	u32 head;
> -
> -	/**
> -	 * Position in the ringbuffer of the start of the postfix.
> -	 * This is required to calculate the maximum available ringbuffer
> -	 * space without overwriting the postfix.
> -	 */
> -	 u32 postfix;
> -
> -	/** Position in the ringbuffer of the end of the whole request */
> -	u32 tail;
> -
> -	/** Preallocate space in the ringbuffer for the emitting the request */
> -	u32 reserved_space;
> -
> -	/**
> -	 * Context and ring buffer related to this request
> -	 * Contexts are refcounted, so when this request is associated with a
> -	 * context, we must increment the context's refcount, to guarantee that
> -	 * it persists while any request is linked to it. Requests themselves
> -	 * are also refcounted, so the request will only be freed when the last
> -	 * reference to it is dismissed, and the code in
> -	 * i915_gem_request_free() will then decrement the refcount on the
> -	 * context.
> -	 */
> -	struct i915_gem_context *ctx;
> -	struct intel_ringbuffer *ringbuf;
> -
> -	/**
> -	 * Context related to the previous request.
> -	 * As the contexts are accessed by the hardware until the switch is
> -	 * completed to a new context, the hardware may still be writing
> -	 * to the context object after the breadcrumb is visible. We must
> -	 * not unpin/unbind/prune that object whilst still active and so
> -	 * we keep the previous context pinned until the following (this)
> -	 * request is retired.
> -	 */
> -	struct i915_gem_context *previous_context;
> -
> -	/** Batch buffer related to this request if any (used for
> -	    error state dump only) */
> -	struct drm_i915_gem_object *batch_obj;
> -
> -	/** Time at which this request was emitted, in jiffies. */
> -	unsigned long emitted_jiffies;
> -
> -	/** global list entry for this request */
> -	struct list_head list;
> -
> -	struct drm_i915_file_private *file_priv;
> -	/** file_priv list entry for this request */
> -	struct list_head client_list;
> -
> -	/** process identifier submitting this request */
> -	struct pid *pid;
> -
> -	/**
> -	 * The ELSP only accepts two elements at a time, so we queue
> -	 * context/tail pairs on a given queue (ring->execlist_queue) until the
> -	 * hardware is available. The queue serves a double purpose: we also use
> -	 * it to keep track of the up to 2 contexts currently in the hardware
> -	 * (usually one in execution and the other queued up by the GPU): We
> -	 * only remove elements from the head of the queue when the hardware
> -	 * informs us that an element has been completed.
> -	 *
> -	 * All accesses to the queue are mediated by a spinlock
> -	 * (ring->execlist_lock).
> -	 */
> -
> -	/** Execlist link in the submission queue.*/
> -	struct list_head execlist_link;
> -
> -	/** Execlists no. of times this request has been sent to the ELSP */
> -	int elsp_submitted;
> -
> -	/** Execlists context hardware id. */
> -	unsigned ctx_hw_id;
> -};
> -
> -struct drm_i915_gem_request * __must_check
> -i915_gem_request_alloc(struct intel_engine_cs *engine,
> -		       struct i915_gem_context *ctx);
> -void i915_gem_request_free(struct kref *req_ref);
> -int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
> -				   struct drm_file *file);
> -
> -static inline uint32_t
> -i915_gem_request_get_seqno(struct drm_i915_gem_request *req)
> -{
> -	return req ? req->seqno : 0;
> -}
> -
> -static inline struct intel_engine_cs *
> -i915_gem_request_get_engine(struct drm_i915_gem_request *req)
> -{
> -	return req ? req->engine : NULL;
> -}
> -
> -static inline struct drm_i915_gem_request *
> -i915_gem_request_reference(struct drm_i915_gem_request *req)
> -{
> -	if (req)
> -		kref_get(&req->ref);
> -	return req;
> -}
> -
> -static inline void
> -i915_gem_request_unreference(struct drm_i915_gem_request *req)
> -{
> -	kref_put(&req->ref, i915_gem_request_free);
> -}
> -
> -static inline void i915_gem_request_assign(struct drm_i915_gem_request **pdst,
> -					   struct drm_i915_gem_request *src)
> -{
> -	if (src)
> -		i915_gem_request_reference(src);
> -
> -	if (*pdst)
> -		i915_gem_request_unreference(*pdst);
> -
> -	*pdst = src;
> -}
> -
> -/*
> - * XXX: i915_gem_request_completed should be here but currently needs the
> - * definition of i915_seqno_passed() which is below. It will be moved in
> - * a later patch when the call to i915_seqno_passed() is obsoleted...
> - */
> -
>  /*
>   * A command that requires special handling by the command parser.
>   */
> @@ -3297,37 +3133,6 @@ void i915_gem_track_fb(struct drm_i915_gem_object *old,
>  		       struct drm_i915_gem_object *new,
>  		       unsigned frontbuffer_bits);
>  
> -/**
> - * Returns true if seq1 is later than seq2.
> - */
> -static inline bool
> -i915_seqno_passed(uint32_t seq1, uint32_t seq2)
> -{
> -	return (int32_t)(seq1 - seq2) >= 0;
> -}
> -
> -static inline bool i915_gem_request_started(const struct drm_i915_gem_request *req)
> -{
> -	return i915_seqno_passed(intel_engine_get_seqno(req->engine),
> -				 req->previous_seqno);
> -}
> -
> -static inline bool i915_gem_request_completed(const struct drm_i915_gem_request *req)
> -{
> -	return i915_seqno_passed(intel_engine_get_seqno(req->engine),
> -				 req->seqno);
> -}
> -
> -bool __i915_spin_request(const struct drm_i915_gem_request *request,
> -			 int state, unsigned long timeout_us);
> -static inline bool i915_spin_request(const struct drm_i915_gem_request *request,
> -				     int state, unsigned long timeout_us)
> -{
> -	return (i915_gem_request_started(request) &&
> -		__i915_spin_request(request, state, timeout_us));
> -}
> -
> -int __must_check i915_gem_get_seqno(struct drm_i915_private *dev_priv, u32 *seqno);
>  int __must_check i915_gem_set_seqno(struct drm_device *dev, u32 seqno);
>  
>  struct drm_i915_gem_request *
> @@ -3385,18 +3190,6 @@ void i915_gem_cleanup_engines(struct drm_device *dev);
>  int __must_check i915_gem_wait_for_idle(struct drm_i915_private *dev_priv);
>  int __must_check i915_gem_suspend(struct drm_device *dev);
>  void i915_gem_resume(struct drm_device *dev);
> -void __i915_add_request(struct drm_i915_gem_request *req,
> -			struct drm_i915_gem_object *batch_obj,
> -			bool flush_caches);
> -#define i915_add_request(req) \
> -	__i915_add_request(req, NULL, true)
> -#define i915_add_request_no_flush(req) \
> -	__i915_add_request(req, NULL, false)
> -int __i915_wait_request(struct drm_i915_gem_request *req,
> -			bool interruptible,
> -			s64 *timeout,
> -			struct intel_rps_client *rps);
> -int __must_check i915_wait_request(struct drm_i915_gem_request *req);
>  int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf);
>  int __must_check
>  i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
> diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
> index 8b42a5101f11..10e5db3f294e 100644
> --- a/drivers/gpu/drm/i915/i915_gem.c
> +++ b/drivers/gpu/drm/i915/i915_gem.c
> @@ -1325,365 +1325,6 @@ put_rpm:
>  	return ret;
>  }
>  
> -static int
> -i915_gem_check_wedge(unsigned reset_counter, bool interruptible)
> -{
> -	if (__i915_terminally_wedged(reset_counter))
> -		return -EIO;
> -
> -	if (__i915_reset_in_progress(reset_counter)) {
> -		/* Non-interruptible callers can't handle -EAGAIN, hence return
> -		 * -EIO unconditionally for these. */
> -		if (!interruptible)
> -			return -EIO;
> -
> -		return -EAGAIN;
> -	}
> -
> -	return 0;
> -}
> -
> -static unsigned long local_clock_us(unsigned *cpu)
> -{
> -	unsigned long t;
> -
> -	/* Cheaply and approximately convert from nanoseconds to microseconds.
> -	 * The result and subsequent calculations are also defined in the same
> -	 * approximate microseconds units. The principal source of timing
> -	 * error here is from the simple truncation.
> -	 *
> -	 * Note that local_clock() is only defined wrt to the current CPU;
> -	 * the comparisons are no longer valid if we switch CPUs. Instead of
> -	 * blocking preemption for the entire busywait, we can detect the CPU
> -	 * switch and use that as indicator of system load and a reason to
> -	 * stop busywaiting, see busywait_stop().
> -	 */
> -	*cpu = get_cpu();
> -	t = local_clock() >> 10;
> -	put_cpu();
> -
> -	return t;
> -}
> -
> -static bool busywait_stop(unsigned long timeout, unsigned cpu)
> -{
> -	unsigned this_cpu;
> -
> -	if (time_after(local_clock_us(&this_cpu), timeout))
> -		return true;
> -
> -	return this_cpu != cpu;
> -}
> -
> -bool __i915_spin_request(const struct drm_i915_gem_request *req,
> -			 int state, unsigned long timeout_us)
> -{
> -	unsigned cpu;
> -
> -	/* When waiting for high frequency requests, e.g. during synchronous
> -	 * rendering split between the CPU and GPU, the finite amount of time
> -	 * required to set up the irq and wait upon it limits the response
> -	 * rate. By busywaiting on the request completion for a short while we
> -	 * can service the high frequency waits as quick as possible. However,
> -	 * if it is a slow request, we want to sleep as quickly as possible.
> -	 * The tradeoff between waiting and sleeping is roughly the time it
> -	 * takes to sleep on a request, on the order of a microsecond.
> -	 */
> -
> -	timeout_us += local_clock_us(&cpu);
> -	do {
> -		if (i915_gem_request_completed(req))
> -			return true;
> -
> -		if (signal_pending_state(state, current))
> -			break;
> -
> -		if (busywait_stop(timeout_us, cpu))
> -			break;
> -
> -		cpu_relax_lowlatency();
> -	} while (!need_resched());
> -
> -	return false;
> -}
> -
> -/**
> - * __i915_wait_request - wait until execution of request has finished
> - * @req: duh!
> - * @interruptible: do an interruptible wait (normally yes)
> - * @timeout: in - how long to wait (NULL forever); out - how much time remaining
> - * @rps: RPS client
> - *
> - * Note: It is of utmost importance that the passed in seqno and reset_counter
> - * values have been read by the caller in an smp safe manner. Where read-side
> - * locks are involved, it is sufficient to read the reset_counter before
> - * unlocking the lock that protects the seqno. For lockless tricks, the
> - * reset_counter _must_ be read before, and an appropriate smp_rmb must be
> - * inserted.
> - *
> - * Returns 0 if the request was found within the alloted time. Else returns the
> - * errno with remaining time filled in timeout argument.
> - */
> -int __i915_wait_request(struct drm_i915_gem_request *req,
> -			bool interruptible,
> -			s64 *timeout,
> -			struct intel_rps_client *rps)
> -{
> -	int state = interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
> -	DEFINE_WAIT(reset);
> -	struct intel_wait wait;
> -	unsigned long timeout_remain;
> -	s64 before = 0; /* Only to silence a compiler warning. */
> -	int ret = 0;
> -
> -	might_sleep();
> -
> -	if (list_empty(&req->list))
> -		return 0;
> -
> -	if (i915_gem_request_completed(req))
> -		return 0;
> -
> -	timeout_remain = MAX_SCHEDULE_TIMEOUT;
> -	if (timeout) {
> -		if (WARN_ON(*timeout < 0))
> -			return -EINVAL;
> -
> -		if (*timeout == 0)
> -			return -ETIME;
> -
> -		timeout_remain = nsecs_to_jiffies_timeout(*timeout);
> -
> -		/*
> -		 * Record current time in case interrupted by signal, or wedged.
> -		 */
> -		before = ktime_get_raw_ns();
> -	}
> -
> -	trace_i915_gem_request_wait_begin(req);
> -
> -	/* This client is about to stall waiting for the GPU. In many cases
> -	 * this is undesirable and limits the throughput of the system, as
> -	 * many clients cannot continue processing user input/output whilst
> -	 * blocked. RPS autotuning may take tens of milliseconds to respond
> -	 * to the GPU load and thus incurs additional latency for the client.
> -	 * We can circumvent that by promoting the GPU frequency to maximum
> -	 * before we wait. This makes the GPU throttle up much more quickly
> -	 * (good for benchmarks and user experience, e.g. window animations),
> -	 * but at a cost of spending more power processing the workload
> -	 * (bad for battery). Not all clients even want their results
> -	 * immediately and for them we should just let the GPU select its own
> -	 * frequency to maximise efficiency. To prevent a single client from
> -	 * forcing the clocks too high for the whole system, we only allow
> -	 * each client to waitboost once in a busy period.
> -	 */
> -	if (INTEL_INFO(req->i915)->gen >= 6)
> -		gen6_rps_boost(req->i915, rps, req->emitted_jiffies);
> -
> -	/* Optimistic spin for the next ~jiffie before touching IRQs */
> -	if (i915_spin_request(req, state, 5))
> -		goto complete;
> -
> -	set_current_state(state);
> -	add_wait_queue(&req->i915->gpu_error.wait_queue, &reset);
> -
> -	intel_wait_init(&wait, req->seqno);
> -	if (intel_engine_add_wait(req->engine, &wait))
> -		/* In order to check that we haven't missed the interrupt
> -		 * as we enabled it, we need to kick ourselves to do a
> -		 * coherent check on the seqno before we sleep.
> -		 */
> -		goto wakeup;
> -
> -	for (;;) {
> -		if (signal_pending_state(state, current)) {
> -			ret = -ERESTARTSYS;
> -			break;
> -		}
> -
> -		timeout_remain = io_schedule_timeout(timeout_remain);
> -		if (timeout_remain == 0) {
> -			ret = -ETIME;
> -			break;
> -		}
> -
> -		if (intel_wait_complete(&wait))
> -			break;
> -
> -		set_current_state(state);
> -
> -wakeup:
> -		/* Carefully check if the request is complete, giving time
> -		 * for the seqno to be visible following the interrupt.
> -		 * We also have to check in case we are kicked by the GPU
> -		 * reset in order to drop the struct_mutex.
> -		 */
> -		if (__i915_request_irq_complete(req))
> -			break;
> -
> -		/* Only spin if we know the GPU is processing this request */
> -		if (i915_spin_request(req, state, 2))
> -			break;
> -	}
> -	remove_wait_queue(&req->i915->gpu_error.wait_queue, &reset);
> -
> -	intel_engine_remove_wait(req->engine, &wait);
> -	__set_current_state(TASK_RUNNING);
> -complete:
> -	trace_i915_gem_request_wait_end(req);
> -
> -	if (timeout) {
> -		s64 tres = *timeout - (ktime_get_raw_ns() - before);
> -
> -		*timeout = tres < 0 ? 0 : tres;
> -
> -		/*
> -		 * Apparently ktime isn't accurate enough and occasionally has a
> -		 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
> -		 * things up to make the test happy. We allow up to 1 jiffy.
> -		 *
> -		 * This is a regrssion from the timespec->ktime conversion.
> -		 */
> -		if (ret == -ETIME && *timeout < jiffies_to_usecs(1)*1000)
> -			*timeout = 0;
> -	}
> -
> -	if (rps && req->seqno == req->engine->last_submitted_seqno) {
> -		/* The GPU is now idle and this client has stalled.
> -		 * Since no other client has submitted a request in the
> -		 * meantime, assume that this client is the only one
> -		 * supplying work to the GPU but is unable to keep that
> -		 * work supplied because it is waiting. Since the GPU is
> -		 * then never kept fully busy, RPS autoclocking will
> -		 * keep the clocks relatively low, causing further delays.
> -		 * Compensate by giving the synchronous client credit for
> -		 * a waitboost next time.
> -		 */
> -		spin_lock(&req->i915->rps.client_lock);
> -		list_del_init(&rps->link);
> -		spin_unlock(&req->i915->rps.client_lock);
> -	}
> -
> -	return ret;
> -}
> -
> -int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
> -				   struct drm_file *file)
> -{
> -	struct drm_i915_file_private *file_priv;
> -
> -	WARN_ON(!req || !file || req->file_priv);
> -
> -	if (!req || !file)
> -		return -EINVAL;
> -
> -	if (req->file_priv)
> -		return -EINVAL;
> -
> -	file_priv = file->driver_priv;
> -
> -	spin_lock(&file_priv->mm.lock);
> -	req->file_priv = file_priv;
> -	list_add_tail(&req->client_list, &file_priv->mm.request_list);
> -	spin_unlock(&file_priv->mm.lock);
> -
> -	req->pid = get_pid(task_pid(current));
> -
> -	return 0;
> -}
> -
> -static inline void
> -i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
> -{
> -	struct drm_i915_file_private *file_priv = request->file_priv;
> -
> -	if (!file_priv)
> -		return;
> -
> -	spin_lock(&file_priv->mm.lock);
> -	list_del(&request->client_list);
> -	request->file_priv = NULL;
> -	spin_unlock(&file_priv->mm.lock);
> -
> -	put_pid(request->pid);
> -	request->pid = NULL;
> -}
> -
> -static void i915_gem_request_retire(struct drm_i915_gem_request *request)
> -{
> -	trace_i915_gem_request_retire(request);
> -
> -	/* We know the GPU must have read the request to have
> -	 * sent us the seqno + interrupt, so use the position
> -	 * of tail of the request to update the last known position
> -	 * of the GPU head.
> -	 *
> -	 * Note this requires that we are always called in request
> -	 * completion order.
> -	 */
> -	request->ringbuf->last_retired_head = request->postfix;
> -
> -	list_del_init(&request->list);
> -	i915_gem_request_remove_from_client(request);
> -
> -	if (request->previous_context) {
> -		if (i915.enable_execlists)
> -			intel_lr_context_unpin(request->previous_context,
> -					       request->engine);
> -	}
> -
> -	i915_gem_context_unreference(request->ctx);
> -	i915_gem_request_unreference(request);
> -}
> -
> -static void
> -__i915_gem_request_retire__upto(struct drm_i915_gem_request *req)
> -{
> -	struct intel_engine_cs *engine = req->engine;
> -	struct drm_i915_gem_request *tmp;
> -
> -	lockdep_assert_held(&engine->i915->drm.struct_mutex);
> -
> -	if (list_empty(&req->list))
> -		return;
> -
> -	do {
> -		tmp = list_first_entry(&engine->request_list,
> -				       typeof(*tmp), list);
> -
> -		i915_gem_request_retire(tmp);
> -	} while (tmp != req);
> -
> -	WARN_ON(i915_verify_lists(engine->dev));
> -}
> -
> -/**
> - * Waits for a request to be signaled, and cleans up the
> - * request and object lists appropriately for that event.
> - * @req: request to wait on
> - */
> -int
> -i915_wait_request(struct drm_i915_gem_request *req)
> -{
> -	struct drm_i915_private *dev_priv = req->i915;
> -	bool interruptible;
> -	int ret;
> -
> -	interruptible = dev_priv->mm.interruptible;
> -
> -	BUG_ON(!mutex_is_locked(&dev_priv->drm.struct_mutex));
> -
> -	ret = __i915_wait_request(req, interruptible, NULL, NULL);
> -	if (ret)
> -		return ret;
> -
> -	/* If the GPU hung, we want to keep the requests to find the guilty. */
> -	if (!i915_reset_in_progress(&dev_priv->gpu_error))
> -		__i915_gem_request_retire__upto(req);
> -
> -	return 0;
> -}
> -
>  /**
>   * Ensures that all rendering to the object has completed and the object is
>   * safe to unbind from the GTT or access from the CPU.
> @@ -1740,7 +1381,7 @@ i915_gem_object_retire_request(struct drm_i915_gem_object *obj,
>  		i915_gem_object_retire__write(obj);
>  
>  	if (!i915_reset_in_progress(&req->i915->gpu_error))
> -		__i915_gem_request_retire__upto(req);
> +		i915_gem_request_retire_upto(req);
>  }
>  
>  /* A nonblocking variant of the above wait. This is a highly dangerous routine
> @@ -2761,193 +2402,6 @@ i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring)
>  	drm_gem_object_unreference(&obj->base);
>  }
>  
> -static int
> -i915_gem_init_seqno(struct drm_i915_private *dev_priv, u32 seqno)
> -{
> -	struct intel_engine_cs *engine;
> -	int ret;
> -
> -	/* Carefully retire all requests without writing to the rings */
> -	for_each_engine(engine, dev_priv) {
> -		ret = intel_engine_idle(engine);
> -		if (ret)
> -			return ret;
> -	}
> -	i915_gem_retire_requests(dev_priv);
> -
> -	/* If the seqno wraps around, we need to clear the breadcrumb rbtree */
> -	if (!i915_seqno_passed(seqno, dev_priv->next_seqno)) {
> -		while (intel_kick_waiters(dev_priv) ||
> -		       intel_kick_signalers(dev_priv))
> -			yield();
> -	}
> -
> -	/* Finally reset hw state */
> -	for_each_engine(engine, dev_priv)
> -		intel_ring_init_seqno(engine, seqno);
> -
> -	return 0;
> -}
> -
> -int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
> -{
> -	struct drm_i915_private *dev_priv = to_i915(dev);
> -	int ret;
> -
> -	if (seqno == 0)
> -		return -EINVAL;
> -
> -	/* HWS page needs to be set less than what we
> -	 * will inject to ring
> -	 */
> -	ret = i915_gem_init_seqno(dev_priv, seqno - 1);
> -	if (ret)
> -		return ret;
> -
> -	/* Carefully set the last_seqno value so that wrap
> -	 * detection still works
> -	 */
> -	dev_priv->next_seqno = seqno;
> -	dev_priv->last_seqno = seqno - 1;
> -	if (dev_priv->last_seqno == 0)
> -		dev_priv->last_seqno--;
> -
> -	return 0;
> -}
> -
> -int
> -i915_gem_get_seqno(struct drm_i915_private *dev_priv, u32 *seqno)
> -{
> -	/* reserve 0 for non-seqno */
> -	if (dev_priv->next_seqno == 0) {
> -		int ret = i915_gem_init_seqno(dev_priv, 0);
> -		if (ret)
> -			return ret;
> -
> -		dev_priv->next_seqno = 1;
> -	}
> -
> -	*seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
> -	return 0;
> -}
> -
> -static void i915_gem_mark_busy(const struct intel_engine_cs *engine)
> -{
> -	struct drm_i915_private *dev_priv = engine->i915;
> -
> -	dev_priv->gt.active_engines |= intel_engine_flag(engine);
> -	if (dev_priv->gt.awake)
> -		return;
> -
> -	intel_runtime_pm_get_noresume(dev_priv);
> -	dev_priv->gt.awake = true;
> -
> -	intel_enable_gt_powersave(dev_priv);
> -	i915_update_gfx_val(dev_priv);
> -	if (INTEL_GEN(dev_priv) >= 6)
> -		gen6_rps_busy(dev_priv);
> -
> -	queue_delayed_work(dev_priv->wq,
> -			   &dev_priv->gt.retire_work,
> -			   round_jiffies_up_relative(HZ));
> -}
> -
> -/*
> - * NB: This function is not allowed to fail. Doing so would mean the the
> - * request is not being tracked for completion but the work itself is
> - * going to happen on the hardware. This would be a Bad Thing(tm).
> - */
> -void __i915_add_request(struct drm_i915_gem_request *request,
> -			struct drm_i915_gem_object *obj,
> -			bool flush_caches)
> -{
> -	struct intel_engine_cs *engine;
> -	struct intel_ringbuffer *ringbuf;
> -	u32 request_start;
> -	u32 reserved_tail;
> -	int ret;
> -
> -	if (WARN_ON(request == NULL))
> -		return;
> -
> -	engine = request->engine;
> -	ringbuf = request->ringbuf;
> -
> -	/*
> -	 * To ensure that this call will not fail, space for its emissions
> -	 * should already have been reserved in the ring buffer. Let the ring
> -	 * know that it is time to use that space up.
> -	 */
> -	request_start = intel_ring_get_tail(ringbuf);
> -	reserved_tail = request->reserved_space;
> -	request->reserved_space = 0;
> -
> -	/*
> -	 * Emit any outstanding flushes - execbuf can fail to emit the flush
> -	 * after having emitted the batchbuffer command. Hence we need to fix
> -	 * things up similar to emitting the lazy request. The difference here
> -	 * is that the flush _must_ happen before the next request, no matter
> -	 * what.
> -	 */
> -	if (flush_caches) {
> -		if (i915.enable_execlists)
> -			ret = logical_ring_flush_all_caches(request);
> -		else
> -			ret = intel_ring_flush_all_caches(request);
> -		/* Not allowed to fail! */
> -		WARN(ret, "*_ring_flush_all_caches failed: %d!\n", ret);
> -	}
> -
> -	trace_i915_gem_request_add(request);
> -
> -	request->head = request_start;
> -
> -	/* Whilst this request exists, batch_obj will be on the
> -	 * active_list, and so will hold the active reference. Only when this
> -	 * request is retired will the the batch_obj be moved onto the
> -	 * inactive_list and lose its active reference. Hence we do not need
> -	 * to explicitly hold another reference here.
> -	 */
> -	request->batch_obj = obj;
> -
> -	/* Seal the request and mark it as pending execution. Note that
> -	 * we may inspect this state, without holding any locks, during
> -	 * hangcheck. Hence we apply the barrier to ensure that we do not
> -	 * see a more recent value in the hws than we are tracking.
> -	 */
> -	request->emitted_jiffies = jiffies;
> -	request->previous_seqno = engine->last_submitted_seqno;
> -	smp_store_mb(engine->last_submitted_seqno, request->seqno);
> -	list_add_tail(&request->list, &engine->request_list);
> -
> -	/* Record the position of the start of the request so that
> -	 * should we detect the updated seqno part-way through the
> -	 * GPU processing the request, we never over-estimate the
> -	 * position of the head.
> -	 */
> -	request->postfix = intel_ring_get_tail(ringbuf);
> -
> -	if (i915.enable_execlists)
> -		ret = engine->emit_request(request);
> -	else {
> -		ret = engine->add_request(request);
> -
> -		request->tail = intel_ring_get_tail(ringbuf);
> -	}
> -	/* Not allowed to fail! */
> -	WARN(ret, "emit|add_request failed: %d!\n", ret);
> -	/* Sanity check that the reserved size was large enough. */
> -	ret = intel_ring_get_tail(ringbuf) - request_start;
> -	if (ret < 0)
> -		ret += ringbuf->size;
> -	WARN_ONCE(ret > reserved_tail,
> -		  "Not enough space reserved (%d bytes) "
> -		  "for adding the request (%d bytes)\n",
> -		  reserved_tail, ret);
> -
> -	i915_gem_mark_busy(engine);
> -}
> -
>  static bool i915_context_is_banned(const struct i915_gem_context *ctx)
>  {
>  	unsigned long elapsed;
> @@ -2979,101 +2433,6 @@ static void i915_set_reset_status(struct i915_gem_context *ctx,
>  	}
>  }
>  
> -void i915_gem_request_free(struct kref *req_ref)
> -{
> -	struct drm_i915_gem_request *req = container_of(req_ref,
> -						 typeof(*req), ref);
> -	kmem_cache_free(req->i915->requests, req);
> -}
> -
> -static inline int
> -__i915_gem_request_alloc(struct intel_engine_cs *engine,
> -			 struct i915_gem_context *ctx,
> -			 struct drm_i915_gem_request **req_out)
> -{
> -	struct drm_i915_private *dev_priv = engine->i915;
> -	unsigned reset_counter = i915_reset_counter(&dev_priv->gpu_error);
> -	struct drm_i915_gem_request *req;
> -	int ret;
> -
> -	if (!req_out)
> -		return -EINVAL;
> -
> -	*req_out = NULL;
> -
> -	/* ABI: Before userspace accesses the GPU (e.g. execbuffer), report
> -	 * EIO if the GPU is already wedged, or EAGAIN to drop the struct_mutex
> -	 * and restart.
> -	 */
> -	ret = i915_gem_check_wedge(reset_counter, dev_priv->mm.interruptible);
> -	if (ret)
> -		return ret;
> -
> -	req = kmem_cache_zalloc(dev_priv->requests, GFP_KERNEL);
> -	if (req == NULL)
> -		return -ENOMEM;
> -
> -	ret = i915_gem_get_seqno(engine->i915, &req->seqno);
> -	if (ret)
> -		goto err;
> -
> -	kref_init(&req->ref);
> -	req->i915 = dev_priv;
> -	req->engine = engine;
> -	req->ctx  = ctx;
> -	i915_gem_context_reference(req->ctx);
> -
> -	/*
> -	 * Reserve space in the ring buffer for all the commands required to
> -	 * eventually emit this request. This is to guarantee that the
> -	 * i915_add_request() call can't fail. Note that the reserve may need
> -	 * to be redone if the request is not actually submitted straight
> -	 * away, e.g. because a GPU scheduler has deferred it.
> -	 */
> -	req->reserved_space = MIN_SPACE_FOR_ADD_REQUEST;
> -
> -	if (i915.enable_execlists)
> -		ret = intel_logical_ring_alloc_request_extras(req);
> -	else
> -		ret = intel_ring_alloc_request_extras(req);
> -	if (ret)
> -		goto err_ctx;
> -
> -	*req_out = req;
> -	return 0;
> -
> -err_ctx:
> -	i915_gem_context_unreference(ctx);
> -err:
> -	kmem_cache_free(dev_priv->requests, req);
> -	return ret;
> -}
> -
> -/**
> - * i915_gem_request_alloc - allocate a request structure
> - *
> - * @engine: engine that we wish to issue the request on.
> - * @ctx: context that the request will be associated with.
> - *       This can be NULL if the request is not directly related to
> - *       any specific user context, in which case this function will
> - *       choose an appropriate context to use.
> - *
> - * Returns a pointer to the allocated request if successful,
> - * or an error code if not.
> - */
> -struct drm_i915_gem_request *
> -i915_gem_request_alloc(struct intel_engine_cs *engine,
> -		       struct i915_gem_context *ctx)
> -{
> -	struct drm_i915_gem_request *req;
> -	int err;
> -
> -	if (ctx == NULL)
> -		ctx = engine->i915->kernel_context;
> -	err = __i915_gem_request_alloc(engine, ctx, &req);
> -	return err ? ERR_PTR(err) : req;
> -}
> -
>  struct drm_i915_gem_request *
>  i915_gem_find_active_request(struct intel_engine_cs *engine)
>  {
> @@ -3147,14 +2506,14 @@ static void i915_gem_reset_engine_cleanup(struct intel_engine_cs *engine)
>  	 * implicit references on things like e.g. ppgtt address spaces through
>  	 * the request.
>  	 */
> -	while (!list_empty(&engine->request_list)) {
> +	if (!list_empty(&engine->request_list)) {
>  		struct drm_i915_gem_request *request;
>  
> -		request = list_first_entry(&engine->request_list,
> -					   struct drm_i915_gem_request,
> -					   list);
> +		request = list_last_entry(&engine->request_list,
> +					  struct drm_i915_gem_request,
> +					  list);
>  
> -		i915_gem_request_retire(request);
> +		i915_gem_request_retire_upto(request);
>  	}
>  
>  	/* Having flushed all requests from all queues, we know that all
> @@ -3222,7 +2581,7 @@ i915_gem_retire_requests_ring(struct intel_engine_cs *engine)
>  		if (!i915_gem_request_completed(request))
>  			break;
>  
> -		i915_gem_request_retire(request);
> +		i915_gem_request_retire_upto(request);
>  	}
>  
>  	/* Move any buffers on the active list that are no longer referenced
> diff --git a/drivers/gpu/drm/i915/i915_gem_request.c b/drivers/gpu/drm/i915/i915_gem_request.c
> new file mode 100644
> index 000000000000..9e9aa6b725f7
> --- /dev/null
> +++ b/drivers/gpu/drm/i915/i915_gem_request.c
> @@ -0,0 +1,658 @@
> +/*
> + * Copyright © 2008-2015 Intel Corporation
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a
> + * copy of this software and associated documentation files (the "Software"),
> + * to deal in the Software without restriction, including without limitation
> + * the rights to use, copy, modify, merge, publish, distribute, sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice (including the next
> + * paragraph) shall be included in all copies or substantial portions of the
> + * Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
> + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
> + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
> + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
> + * IN THE SOFTWARE.
> + *
> + */
> +
> +#include "i915_drv.h"
> +
> +int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
> +				   struct drm_file *file)
> +{
> +	struct drm_i915_private *dev_private;
> +	struct drm_i915_file_private *file_priv;
> +
> +	WARN_ON(!req || !file || req->file_priv);
> +
> +	if (!req || !file)
> +		return -EINVAL;
> +
> +	if (req->file_priv)
> +		return -EINVAL;
> +
> +	dev_private = req->i915;
> +	file_priv = file->driver_priv;
> +
> +	spin_lock(&file_priv->mm.lock);
> +	req->file_priv = file_priv;
> +	list_add_tail(&req->client_list, &file_priv->mm.request_list);
> +	spin_unlock(&file_priv->mm.lock);
> +
> +	req->pid = get_pid(task_pid(current));
> +
> +	return 0;
> +}
> +
> +static inline void
> +i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
> +{
> +	struct drm_i915_file_private *file_priv = request->file_priv;
> +
> +	if (!file_priv)
> +		return;
> +
> +	spin_lock(&file_priv->mm.lock);
> +	list_del(&request->client_list);
> +	request->file_priv = NULL;
> +	spin_unlock(&file_priv->mm.lock);
> +
> +	put_pid(request->pid);
> +	request->pid = NULL;
> +}
> +
> +static void i915_gem_request_retire(struct drm_i915_gem_request *request)
> +{
> +	trace_i915_gem_request_retire(request);
> +	list_del_init(&request->list);
> +
> +	/* We know the GPU must have read the request to have
> +	 * sent us the seqno + interrupt, so use the position
> +	 * of tail of the request to update the last known position
> +	 * of the GPU head.
> +	 *
> +	 * Note this requires that we are always called in request
> +	 * completion order.
> +	 */
> +	request->ringbuf->last_retired_head = request->postfix;
> +
> +	i915_gem_request_remove_from_client(request);
> +
> +	if (request->previous_context) {
> +		if (i915.enable_execlists)
> +			intel_lr_context_unpin(request->previous_context,
> +					       request->engine);
> +	}
> +
> +	i915_gem_context_unreference(request->ctx);
> +	i915_gem_request_unreference(request);
> +}
> +
> +void i915_gem_request_retire_upto(struct drm_i915_gem_request *req)
> +{
> +	struct intel_engine_cs *engine = req->engine;
> +	struct drm_i915_gem_request *tmp;
> +
> +	lockdep_assert_held(&req->i915->drm.struct_mutex);
> +
> +	if (list_empty(&req->list))
> +		return;
> +
> +	do {
> +		tmp = list_first_entry(&engine->request_list,
> +				       typeof(*tmp), list);
> +
> +		i915_gem_request_retire(tmp);
> +	} while (tmp != req);
> +
> +	WARN_ON(i915_verify_lists(engine->dev));
> +}
> +
> +static int i915_gem_check_wedge(unsigned int reset_counter, bool interruptible)
> +{
> +	if (__i915_terminally_wedged(reset_counter))
> +		return -EIO;
> +
> +	if (__i915_reset_in_progress(reset_counter)) {
> +		/* Non-interruptible callers can't handle -EAGAIN, hence return
> +		 * -EIO unconditionally for these.
> +		 */
> +		if (!interruptible)
> +			return -EIO;
> +
> +		return -EAGAIN;
> +	}
> +
> +	return 0;
> +}
> +
> +static int i915_gem_init_seqno(struct drm_i915_private *dev_priv, u32 seqno)
> +{
> +	struct intel_engine_cs *engine;
> +	int ret;
> +
> +	/* Carefully retire all requests without writing to the rings */
> +	for_each_engine(engine, dev_priv) {
> +		ret = intel_engine_idle(engine);
> +		if (ret)
> +			return ret;
> +	}
> +	i915_gem_retire_requests(dev_priv);
> +
> +	/* If the seqno wraps around, we need to clear the breadcrumb rbtree */
> +	if (!i915_seqno_passed(seqno, dev_priv->next_seqno)) {
> +		while (intel_kick_waiters(dev_priv) ||
> +		       intel_kick_signalers(dev_priv))
> +			yield();
> +	}
> +
> +	/* Finally reset hw state */
> +	for_each_engine(engine, dev_priv)
> +		intel_ring_init_seqno(engine, seqno);
> +
> +	return 0;
> +}
> +
> +int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
> +{
> +	struct drm_i915_private *dev_priv = to_i915(dev);
> +	int ret;
> +
> +	if (seqno == 0)
> +		return -EINVAL;
> +
> +	/* HWS page needs to be set less than what we
> +	 * will inject to ring
> +	 */
> +	ret = i915_gem_init_seqno(dev_priv, seqno - 1);
> +	if (ret)
> +		return ret;
> +
> +	/* Carefully set the last_seqno value so that wrap
> +	 * detection still works
> +	 */
> +	dev_priv->next_seqno = seqno;
> +	dev_priv->last_seqno = seqno - 1;
> +	if (dev_priv->last_seqno == 0)
> +		dev_priv->last_seqno--;
> +
> +	return 0;
> +}
> +
> +static int i915_gem_get_seqno(struct drm_i915_private *dev_priv, u32 *seqno)
> +{
> +	/* reserve 0 for non-seqno */
> +	if (unlikely(dev_priv->next_seqno == 0)) {
> +		int ret;
> +
> +		ret = i915_gem_init_seqno(dev_priv, 0);
> +		if (ret)
> +			return ret;
> +
> +		dev_priv->next_seqno = 1;
> +	}
> +
> +	*seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
> +	return 0;
> +}
> +
> +static inline int
> +__i915_gem_request_alloc(struct intel_engine_cs *engine,
> +			 struct i915_gem_context *ctx,
> +			 struct drm_i915_gem_request **req_out)
> +{
> +	struct drm_i915_private *dev_priv = engine->i915;
> +	unsigned int reset_counter = i915_reset_counter(&dev_priv->gpu_error);
> +	struct drm_i915_gem_request *req;
> +	int ret;
> +
> +	if (!req_out)
> +		return -EINVAL;
> +
> +	*req_out = NULL;
> +
> +	/* ABI: Before userspace accesses the GPU (e.g. execbuffer), report
> +	 * EIO if the GPU is already wedged, or EAGAIN to drop the struct_mutex
> +	 * and restart.
> +	 */
> +	ret = i915_gem_check_wedge(reset_counter, dev_priv->mm.interruptible);
> +	if (ret)
> +		return ret;
> +
> +	req = kmem_cache_zalloc(dev_priv->requests, GFP_KERNEL);
> +	if (!req)
> +		return -ENOMEM;
> +
> +	ret = i915_gem_get_seqno(dev_priv, &req->seqno);
> +	if (ret)
> +		goto err;
> +
> +	kref_init(&req->ref);
> +	req->i915 = dev_priv;
> +	req->engine = engine;
> +	req->ctx = ctx;
> +	i915_gem_context_reference(ctx);
> +
> +	/*
> +	 * Reserve space in the ring buffer for all the commands required to
> +	 * eventually emit this request. This is to guarantee that the
> +	 * i915_add_request() call can't fail. Note that the reserve may need
> +	 * to be redone if the request is not actually submitted straight
> +	 * away, e.g. because a GPU scheduler has deferred it.
> +	 */
> +	req->reserved_space = MIN_SPACE_FOR_ADD_REQUEST;
> +
> +	if (i915.enable_execlists)
> +		ret = intel_logical_ring_alloc_request_extras(req);
> +	else
> +		ret = intel_ring_alloc_request_extras(req);
> +	if (ret)
> +		goto err_ctx;
> +
> +	*req_out = req;
> +	return 0;
> +
> +err_ctx:
> +	i915_gem_context_unreference(ctx);
> +err:
> +	kmem_cache_free(dev_priv->requests, req);
> +	return ret;
> +}
> +
> +/**
> + * i915_gem_request_alloc - allocate a request structure
> + *
> + * @engine: engine that we wish to issue the request on.
> + * @ctx: context that the request will be associated with.
> + *       This can be NULL if the request is not directly related to
> + *       any specific user context, in which case this function will
> + *       choose an appropriate context to use.
> + *
> + * Returns a pointer to the allocated request if successful,
> + * or an error code if not.
> + */
> +struct drm_i915_gem_request *
> +i915_gem_request_alloc(struct intel_engine_cs *engine,
> +		       struct i915_gem_context *ctx)
> +{
> +	struct drm_i915_gem_request *req;
> +	int err;
> +
> +	if (!ctx)
> +		ctx = engine->i915->kernel_context;
> +	err = __i915_gem_request_alloc(engine, ctx, &req);
> +	return err ? ERR_PTR(err) : req;
> +}
> +
> +static void i915_gem_mark_busy(const struct intel_engine_cs *engine)
> +{
> +	struct drm_i915_private *dev_priv = engine->i915;
> +
> +	dev_priv->gt.active_engines |= intel_engine_flag(engine);
> +	if (dev_priv->gt.awake)
> +		return;
> +
> +	intel_runtime_pm_get_noresume(dev_priv);
> +	dev_priv->gt.awake = true;
> +
> +	intel_enable_gt_powersave(dev_priv);
> +	i915_update_gfx_val(dev_priv);
> +	if (INTEL_GEN(dev_priv) >= 6)
> +		gen6_rps_busy(dev_priv);
> +
> +	queue_delayed_work(dev_priv->wq,
> +			   &dev_priv->gt.retire_work,
> +			   round_jiffies_up_relative(HZ));
> +}
> +
> +/*
> + * NB: This function is not allowed to fail. Doing so would mean the the
> + * request is not being tracked for completion but the work itself is
> + * going to happen on the hardware. This would be a Bad Thing(tm).
> + */
> +void __i915_add_request(struct drm_i915_gem_request *request,
> +			struct drm_i915_gem_object *obj,
> +			bool flush_caches)
> +{
> +	struct intel_engine_cs *engine;
> +	struct intel_ringbuffer *ringbuf;
> +	u32 request_start;
> +	u32 reserved_tail;
> +	int ret;
> +
> +	if (WARN_ON(!request))
> +		return;
> +
> +	engine = request->engine;
> +	ringbuf = request->ringbuf;
> +
> +	/*
> +	 * To ensure that this call will not fail, space for its emissions
> +	 * should already have been reserved in the ring buffer. Let the ring
> +	 * know that it is time to use that space up.
> +	 */
> +	request_start = intel_ring_get_tail(ringbuf);
> +	reserved_tail = request->reserved_space;
> +	request->reserved_space = 0;
> +
> +	/*
> +	 * Emit any outstanding flushes - execbuf can fail to emit the flush
> +	 * after having emitted the batchbuffer command. Hence we need to fix
> +	 * things up similar to emitting the lazy request. The difference here
> +	 * is that the flush _must_ happen before the next request, no matter
> +	 * what.
> +	 */
> +	if (flush_caches) {
> +		if (i915.enable_execlists)
> +			ret = logical_ring_flush_all_caches(request);
> +		else
> +			ret = intel_ring_flush_all_caches(request);
> +		/* Not allowed to fail! */
> +		WARN(ret, "*_ring_flush_all_caches failed: %d!\n", ret);
> +	}
> +
> +	trace_i915_gem_request_add(request);
> +
> +	request->head = request_start;
> +
> +	/* Whilst this request exists, batch_obj will be on the
> +	 * active_list, and so will hold the active reference. Only when this
> +	 * request is retired will the the batch_obj be moved onto the
> +	 * inactive_list and lose its active reference. Hence we do not need
> +	 * to explicitly hold another reference here.
> +	 */
> +	request->batch_obj = obj;
> +
> +	/* Seal the request and mark it as pending execution. Note that
> +	 * we may inspect this state, without holding any locks, during
> +	 * hangcheck. Hence we apply the barrier to ensure that we do not
> +	 * see a more recent value in the hws than we are tracking.
> +	 */
> +	request->emitted_jiffies = jiffies;
> +	request->previous_seqno = engine->last_submitted_seqno;
> +	smp_store_mb(engine->last_submitted_seqno, request->seqno);
> +	list_add_tail(&request->list, &engine->request_list);
> +
> +	/* Record the position of the start of the request so that
> +	 * should we detect the updated seqno part-way through the
> +	 * GPU processing the request, we never over-estimate the
> +	 * position of the head.
> +	 */
> +	request->postfix = intel_ring_get_tail(ringbuf);
> +
> +	if (i915.enable_execlists) {
> +		ret = engine->emit_request(request);
> +	} else {
> +		ret = engine->add_request(request);
> +
> +		request->tail = intel_ring_get_tail(ringbuf);
> +	}
> +	/* Not allowed to fail! */
> +	WARN(ret, "emit|add_request failed: %d!\n", ret);
> +	/* Sanity check that the reserved size was large enough. */
> +	ret = intel_ring_get_tail(ringbuf) - request_start;
> +	if (ret < 0)
> +		ret += ringbuf->size;
> +	WARN_ONCE(ret > reserved_tail,
> +		  "Not enough space reserved (%d bytes) "
> +		  "for adding the request (%d bytes)\n",
> +		  reserved_tail, ret);
> +
> +	i915_gem_mark_busy(engine);
> +}
> +
> +static unsigned long local_clock_us(unsigned int *cpu)
> +{
> +	unsigned long t;
> +
> +	/* Cheaply and approximately convert from nanoseconds to microseconds.
> +	 * The result and subsequent calculations are also defined in the same
> +	 * approximate microseconds units. The principal source of timing
> +	 * error here is from the simple truncation.
> +	 *
> +	 * Note that local_clock() is only defined wrt to the current CPU;
> +	 * the comparisons are no longer valid if we switch CPUs. Instead of
> +	 * blocking preemption for the entire busywait, we can detect the CPU
> +	 * switch and use that as indicator of system load and a reason to
> +	 * stop busywaiting, see busywait_stop().
> +	 */
> +	*cpu = get_cpu();
> +	t = local_clock() >> 10;
> +	put_cpu();
> +
> +	return t;
> +}
> +
> +static bool busywait_stop(unsigned long timeout, unsigned int cpu)
> +{
> +	unsigned int this_cpu;
> +
> +	if (time_after(local_clock_us(&this_cpu), timeout))
> +		return true;
> +
> +	return this_cpu != cpu;
> +}
> +
> +bool __i915_spin_request(const struct drm_i915_gem_request *req,
> +			 int state, unsigned long timeout_us)
> +{
> +	unsigned int cpu;
> +
> +	/* When waiting for high frequency requests, e.g. during synchronous
> +	 * rendering split between the CPU and GPU, the finite amount of time
> +	 * required to set up the irq and wait upon it limits the response
> +	 * rate. By busywaiting on the request completion for a short while we
> +	 * can service the high frequency waits as quick as possible. However,
> +	 * if it is a slow request, we want to sleep as quickly as possible.
> +	 * The tradeoff between waiting and sleeping is roughly the time it
> +	 * takes to sleep on a request, on the order of a microsecond.
> +	 */
> +
> +	timeout_us += local_clock_us(&cpu);
> +	do {
> +		if (i915_gem_request_completed(req))
> +			return true;
> +
> +		if (signal_pending_state(state, current))
> +			break;
> +
> +		if (busywait_stop(timeout_us, cpu))
> +			break;
> +
> +		cpu_relax_lowlatency();
> +	} while (!need_resched());
> +
> +	return false;
> +}
> +
> +/**
> + * __i915_wait_request - wait until execution of request has finished
> + * @req: duh!
> + * @interruptible: do an interruptible wait (normally yes)
> + * @timeout: in - how long to wait (NULL forever); out - how much time remaining
> + * @rps: client to charge for RPS boosting
> + *
> + * Note: It is of utmost importance that the passed in seqno and reset_counter
> + * values have been read by the caller in an smp safe manner. Where read-side
> + * locks are involved, it is sufficient to read the reset_counter before
> + * unlocking the lock that protects the seqno. For lockless tricks, the
> + * reset_counter _must_ be read before, and an appropriate smp_rmb must be
> + * inserted.
> + *
> + * Returns 0 if the request was found within the alloted time. Else returns the
> + * errno with remaining time filled in timeout argument.
> + */
> +int __i915_wait_request(struct drm_i915_gem_request *req,
> +			bool interruptible,
> +			s64 *timeout,
> +			struct intel_rps_client *rps)
> +{
> +	int state = interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
> +	DEFINE_WAIT(reset);
> +	struct intel_wait wait;
> +	unsigned long timeout_remain;
> +	int ret = 0;
> +
> +	might_sleep();
> +
> +	if (list_empty(&req->list))
> +		return 0;
> +
> +	if (i915_gem_request_completed(req))
> +		return 0;
> +
> +	timeout_remain = MAX_SCHEDULE_TIMEOUT;
> +	if (timeout) {
> +		if (WARN_ON(*timeout < 0))
> +			return -EINVAL;
> +
> +		if (*timeout == 0)
> +			return -ETIME;
> +
> +		/* Record current time in case interrupted, or wedged */
> +		timeout_remain = nsecs_to_jiffies_timeout(*timeout);
> +		*timeout += ktime_get_raw_ns();
> +	}
> +
> +	trace_i915_gem_request_wait_begin(req);
> +
> +	/* This client is about to stall waiting for the GPU. In many cases
> +	 * this is undesirable and limits the throughput of the system, as
> +	 * many clients cannot continue processing user input/output whilst
> +	 * blocked. RPS autotuning may take tens of milliseconds to respond
> +	 * to the GPU load and thus incurs additional latency for the client.
> +	 * We can circumvent that by promoting the GPU frequency to maximum
> +	 * before we wait. This makes the GPU throttle up much more quickly
> +	 * (good for benchmarks and user experience, e.g. window animations),
> +	 * but at a cost of spending more power processing the workload
> +	 * (bad for battery). Not all clients even want their results
> +	 * immediately and for them we should just let the GPU select its own
> +	 * frequency to maximise efficiency. To prevent a single client from
> +	 * forcing the clocks too high for the whole system, we only allow
> +	 * each client to waitboost once in a busy period.
> +	 */
> +	if (INTEL_GEN(req->i915) >= 6)
> +		gen6_rps_boost(req->i915, rps, req->emitted_jiffies);
> +
> +	/* Optimistic spin for the next ~jiffie before touching IRQs */
> +	if (i915_spin_request(req, state, 5))
> +		goto complete;
> +
> +	set_current_state(state);
> +	add_wait_queue(&req->i915->gpu_error.wait_queue, &reset);
> +
> +	intel_wait_init(&wait, req->seqno);
> +	if (intel_engine_add_wait(req->engine, &wait))
> +		/* In order to check that we haven't missed the interrupt
> +		 * as we enabled it, we need to kick ourselves to do a
> +		 * coherent check on the seqno before we sleep.
> +		 */
> +		goto wakeup;
> +
> +	for (;;) {
> +		if (signal_pending_state(state, current)) {
> +			ret = -ERESTARTSYS;
> +			break;
> +		}
> +
> +		timeout_remain = io_schedule_timeout(timeout_remain);
> +		if (timeout_remain == 0) {
> +			ret = -ETIME;
> +			break;
> +		}
> +
> +		if (intel_wait_complete(&wait))
> +			break;
> +
> +		set_current_state(state);
> +
> +wakeup:
> +		/* Carefully check if the request is complete, giving time
> +		 * for the seqno to be visible following the interrupt.
> +		 * We also have to check in case we are kicked by the GPU
> +		 * reset in order to drop the struct_mutex.
> +		 */
> +		if (__i915_request_irq_complete(req))
> +			break;
> +
> +		/* Only spin if we know the GPU is processing this request */
> +		if (i915_spin_request(req, state, 2))
> +			break;
> +	}
> +	remove_wait_queue(&req->i915->gpu_error.wait_queue, &reset);
> +
> +	intel_engine_remove_wait(req->engine, &wait);
> +	__set_current_state(TASK_RUNNING);
> +complete:
> +	trace_i915_gem_request_wait_end(req);
> +
> +	if (timeout) {
> +		*timeout -= ktime_get_raw_ns();
> +		if (*timeout < 0)
> +			*timeout = 0;
> +
> +		/*
> +		 * Apparently ktime isn't accurate enough and occasionally has a
> +		 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
> +		 * things up to make the test happy. We allow up to 1 jiffy.
> +		 *
> +		 * This is a regrssion from the timespec->ktime conversion.
> +		 */
> +		if (ret == -ETIME && *timeout < jiffies_to_usecs(1)*1000)
> +			*timeout = 0;
> +	}
> +
> +	if (rps && req->seqno == req->engine->last_submitted_seqno) {
> +		/* The GPU is now idle and this client has stalled.
> +		 * Since no other client has submitted a request in the
> +		 * meantime, assume that this client is the only one
> +		 * supplying work to the GPU but is unable to keep that
> +		 * work supplied because it is waiting. Since the GPU is
> +		 * then never kept fully busy, RPS autoclocking will
> +		 * keep the clocks relatively low, causing further delays.
> +		 * Compensate by giving the synchronous client credit for
> +		 * a waitboost next time.
> +		 */
> +		spin_lock(&req->i915->rps.client_lock);
> +		list_del_init(&rps->link);
> +		spin_unlock(&req->i915->rps.client_lock);
> +	}
> +
> +	return ret;
> +}
> +
> +/**
> + * Waits for a request to be signaled, and cleans up the
> + * request and object lists appropriately for that event.
> + */
> +int i915_wait_request(struct drm_i915_gem_request *req)
> +{
> +	int ret;
> +
> +	GEM_BUG_ON(!req);
> +	lockdep_assert_held(&req->i915->drm.struct_mutex);
> +
> +	ret = __i915_wait_request(req, req->i915->mm.interruptible, NULL, NULL);
> +	if (ret)
> +		return ret;
> +
> +	/* If the GPU hung, we want to keep the requests to find the guilty. */
> +	if (!i915_reset_in_progress(&req->i915->gpu_error))
> +		i915_gem_request_retire_upto(req);
> +
> +	return 0;
> +}
> +
> +void i915_gem_request_free(struct kref *req_ref)
> +{
> +	struct drm_i915_gem_request *req =
> +		container_of(req_ref, typeof(*req), ref);
> +	kmem_cache_free(req->i915->requests, req);
> +}
> diff --git a/drivers/gpu/drm/i915/i915_gem_request.h b/drivers/gpu/drm/i915/i915_gem_request.h
> new file mode 100644
> index 000000000000..ea700befcc28
> --- /dev/null
> +++ b/drivers/gpu/drm/i915/i915_gem_request.h
> @@ -0,0 +1,238 @@
> +/*
> + * Copyright © 2008-2015 Intel Corporation
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a
> + * copy of this software and associated documentation files (the "Software"),
> + * to deal in the Software without restriction, including without limitation
> + * the rights to use, copy, modify, merge, publish, distribute, sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice (including the next
> + * paragraph) shall be included in all copies or substantial portions of the
> + * Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
> + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
> + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
> + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
> + * IN THE SOFTWARE.
> + *
> + */
> +
> +#ifndef I915_GEM_REQUEST_H
> +#define I915_GEM_REQUEST_H
> +
> +/**
> + * Request queue structure.
> + *
> + * The request queue allows us to note sequence numbers that have been emitted
> + * and may be associated with active buffers to be retired.
> + *
> + * By keeping this list, we can avoid having to do questionable sequence
> + * number comparisons on buffer last_read|write_seqno. It also allows an
> + * emission time to be associated with the request for tracking how far ahead
> + * of the GPU the submission is.
> + *
> + * The requests are reference counted, so upon creation they should have an
> + * initial reference taken using kref_init
> + */
> +struct drm_i915_gem_request {
> +	struct kref ref;
> +
> +	/** On Which ring this request was generated */
> +	struct drm_i915_private *i915;
> +
> +	/**
> +	 * Context and ring buffer related to this request
> +	 * Contexts are refcounted, so when this request is associated with a
> +	 * context, we must increment the context's refcount, to guarantee that
> +	 * it persists while any request is linked to it. Requests themselves
> +	 * are also refcounted, so the request will only be freed when the last
> +	 * reference to it is dismissed, and the code in
> +	 * i915_gem_request_free() will then decrement the refcount on the
> +	 * context.
> +	 */
> +	struct i915_gem_context *ctx;
> +	struct intel_engine_cs *engine;
> +	struct intel_ringbuffer *ringbuf;
> +	struct intel_signal_node signaling;
> +
> +	/** GEM sequence number associated with the previous request,
> +	 * when the HWS breadcrumb is equal to this the GPU is processing
> +	 * this request.
> +	 */
> +	u32 previous_seqno;
> +
> +	/** GEM sequence number associated with this request,
> +	 * when the HWS breadcrumb is equal or greater than this the GPU
> +	 * has finished processing this request.
> +	 */
> +	u32 seqno;
> +
> +	/** Position in the ringbuffer of the start of the request */
> +	u32 head;
> +
> +	/**
> +	 * Position in the ringbuffer of the start of the postfix.
> +	 * This is required to calculate the maximum available ringbuffer
> +	 * space without overwriting the postfix.
> +	 */
> +	u32 postfix;
> +
> +	/** Position in the ringbuffer of the end of the whole request */
> +	u32 tail;
> +
> +	/** Preallocate space in the ringbuffer for the emitting the request */
> +	u32 reserved_space;
> +
> +	/**
> +	 * Context related to the previous request.
> +	 * As the contexts are accessed by the hardware until the switch is
> +	 * completed to a new context, the hardware may still be writing
> +	 * to the context object after the breadcrumb is visible. We must
> +	 * not unpin/unbind/prune that object whilst still active and so
> +	 * we keep the previous context pinned until the following (this)
> +	 * request is retired.
> +	 */
> +	struct i915_gem_context *previous_context;
> +
> +	/** Batch buffer related to this request if any (used for
> +	 * error state dump only).
> +	 */
> +	struct drm_i915_gem_object *batch_obj;
> +
> +	/** Time at which this request was emitted, in jiffies. */
> +	unsigned long emitted_jiffies;
> +
> +	/** global list entry for this request */
> +	struct list_head list;
> +
> +	struct drm_i915_file_private *file_priv;
> +	/** file_priv list entry for this request */
> +	struct list_head client_list;
> +
> +	/** process identifier submitting this request */
> +	struct pid *pid;
> +
> +	/**
> +	 * The ELSP only accepts two elements at a time, so we queue
> +	 * context/tail pairs on a given queue (ring->execlist_queue) until the
> +	 * hardware is available. The queue serves a double purpose: we also use
> +	 * it to keep track of the up to 2 contexts currently in the hardware
> +	 * (usually one in execution and the other queued up by the GPU): We
> +	 * only remove elements from the head of the queue when the hardware
> +	 * informs us that an element has been completed.
> +	 *
> +	 * All accesses to the queue are mediated by a spinlock
> +	 * (ring->execlist_lock).
> +	 */
> +
> +	/** Execlist link in the submission queue.*/
> +	struct list_head execlist_link;
> +
> +	/** Execlists no. of times this request has been sent to the ELSP */
> +	int elsp_submitted;
> +
> +	/** Execlists context hardware id. */
> +	unsigned int ctx_hw_id;
> +};
> +
> +struct drm_i915_gem_request * __must_check
> +i915_gem_request_alloc(struct intel_engine_cs *engine,
> +		       struct i915_gem_context *ctx);
> +void i915_gem_request_free(struct kref *req_ref);
> +int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
> +				   struct drm_file *file);
> +void i915_gem_request_retire_upto(struct drm_i915_gem_request *req);
> +
> +static inline u32
> +i915_gem_request_get_seqno(struct drm_i915_gem_request *req)
> +{
> +	return req ? req->seqno : 0;
> +}
> +
> +static inline struct intel_engine_cs *
> +i915_gem_request_get_engine(struct drm_i915_gem_request *req)
> +{
> +	return req ? req->engine : NULL;
> +}
> +
> +static inline struct drm_i915_gem_request *
> +i915_gem_request_reference(struct drm_i915_gem_request *req)
> +{
> +	if (req)
> +		kref_get(&req->ref);
> +	return req;
> +}
> +
> +static inline void
> +i915_gem_request_unreference(struct drm_i915_gem_request *req)
> +{
> +	kref_put(&req->ref, i915_gem_request_free);
> +}
> +
> +static inline void i915_gem_request_assign(struct drm_i915_gem_request **pdst,
> +					   struct drm_i915_gem_request *src)
> +{
> +	if (src)
> +		i915_gem_request_reference(src);
> +
> +	if (*pdst)
> +		i915_gem_request_unreference(*pdst);
> +
> +	*pdst = src;
> +}
> +
> +void __i915_add_request(struct drm_i915_gem_request *req,
> +			struct drm_i915_gem_object *batch_obj,
> +			bool flush_caches);
> +#define i915_add_request(req) \
> +	__i915_add_request(req, NULL, true)
> +#define i915_add_request_no_flush(req) \
> +	__i915_add_request(req, NULL, false)
> +
> +struct intel_rps_client;
> +
> +int __i915_wait_request(struct drm_i915_gem_request *req,
> +			bool interruptible,
> +			s64 *timeout,
> +			struct intel_rps_client *rps);
> +int __must_check i915_wait_request(struct drm_i915_gem_request *req);
> +
> +static inline u32 intel_engine_get_seqno(struct intel_engine_cs *engine);
> +
> +/**
> + * Returns true if seq1 is later than seq2.
> + */
> +static inline bool i915_seqno_passed(u32 seq1, u32 seq2)
> +{
> +	return (s32)(seq1 - seq2) >= 0;
> +}
> +
> +static inline bool
> +i915_gem_request_started(const struct drm_i915_gem_request *req)
> +{
> +	return i915_seqno_passed(intel_engine_get_seqno(req->engine),
> +				 req->previous_seqno);
> +}
> +
> +static inline bool
> +i915_gem_request_completed(const struct drm_i915_gem_request *req)
> +{
> +	return i915_seqno_passed(intel_engine_get_seqno(req->engine),
> +				 req->seqno);
> +}
> +
> +bool __i915_spin_request(const struct drm_i915_gem_request *request,
> +			 int state, unsigned long timeout_us);
> +static inline bool i915_spin_request(const struct drm_i915_gem_request *request,
> +				     int state, unsigned long timeout_us)
> +{
> +	return (i915_gem_request_started(request) &&
> +		__i915_spin_request(request, state, timeout_us));
> +}
> +
> +#endif /* I915_GEM_REQUEST_H */
> -- 
> 2.8.1
>
> _______________________________________________
> Intel-gfx mailing list
> Intel-gfx at lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/intel-gfx


More information about the Intel-gfx mailing list