iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

drm/i915/gt: Use virtual_engine during execlists_dequeue

Browse Source 
Rather than going back and forth between the rb_node entry and the
virtual_engine type, store the ve local and reuse it. As the
container_of conversion from rb_node to virtual_engine requires a
variable offset, performing that conversion just once shaves off a bit
of code.

v2: Keep a single virtual engine lookup, for typical use.

Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Reviewed-by: Matthew Auld <matthew.auld@intel.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20201224135544.1713-2-chris@chris-wilson.co.uk
This commit is contained in:
Chris Wilson 2020-12-24 13:55:37 +00:00
parent 16f2941ad3
commit 64b7a3fa7e
1 changed files with 113 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

255
drivers/gpu/drm/i915/gt/intel_execlists_submission.c
View File

@ -293,9 +293,15 @@ static int queue_prio(const struct intel_engine_execlists *execlists)
return ((p->priority + 1) << I915_USER_PRIORITY_SHIFT) - ffs(p->used);
}
static int virtual_prio(const struct intel_engine_execlists *el)
{
struct rb_node *rb = rb_first_cached(&el->virtual);
return rb ? rb_entry(rb, struct ve_node, rb)->prio : INT_MIN;
}
static inline bool need_preempt(const struct intel_engine_cs *engine,
const struct i915_request *rq,
struct rb_node *rb)
const struct i915_request *rq)
{
int last_prio;
@ -332,25 +338,6 @@ static inline bool need_preempt(const struct intel_engine_cs *engine,
rq_prio(list_next_entry(rq, sched.link)) > last_prio)
return true;
if (rb) {
struct virtual_engine *ve =
rb_entry(rb, typeof(*ve), nodes[engine->id].rb);
bool preempt = false;
if (engine == ve->siblings[0]) { /* only preempt one sibling */
struct i915_request *next;
rcu_read_lock();
next = READ_ONCE(ve->request);
if (next)
preempt = rq_prio(next) > last_prio;
rcu_read_unlock();
}
if (preempt)
return preempt;
}
/*
* If the inflight context did not trigger the preemption, then maybe
* it was the set of queued requests? Pick the highest priority in
@ -361,7 +348,8 @@ static inline bool need_preempt(const struct intel_engine_cs *engine,
* ELSP[0] or ELSP[1] as, thanks again to PI, if it was the same
* context, it's priority would not exceed ELSP[0] aka last_prio.
*/
return queue_prio(&engine->execlists) > last_prio;
return max(virtual_prio(&engine->execlists),
queue_prio(&engine->execlists)) > last_prio;
}
__maybe_unused static inline bool
@ -997,6 +985,35 @@ static bool virtual_matches(const struct virtual_engine *ve,
return true;
}
static struct virtual_engine *
first_virtual_engine(struct intel_engine_cs *engine)
{
struct intel_engine_execlists *el = &engine->execlists;
struct rb_node *rb = rb_first_cached(&el->virtual);
while (rb) {
struct virtual_engine *ve =
rb_entry(rb, typeof(*ve), nodes[engine->id].rb);
struct i915_request *rq = READ_ONCE(ve->request);
/* lazily cleanup after another engine handled rq */
if (!rq) {
rb_erase_cached(rb, &el->virtual);
RB_CLEAR_NODE(rb);
rb = rb_first_cached(&el->virtual);
continue;
}
if (!virtual_matches(ve, rq, engine)) {
rb = rb_next(rb);
continue;
}
return ve;
}
return NULL;
}
static void virtual_xfer_context(struct virtual_engine *ve,
struct intel_engine_cs *engine)
{
@ -1084,32 +1101,15 @@ static void defer_active(struct intel_engine_cs *engine)
static bool
need_timeslice(const struct intel_engine_cs *engine,
const struct i915_request *rq,
const struct rb_node *rb)
const struct i915_request *rq)
{
int hint;
if (!intel_engine_has_timeslices(engine))
return false;
hint = engine->execlists.queue_priority_hint;
if (rb) {
const struct virtual_engine *ve =
rb_entry(rb, typeof(*ve), nodes[engine->id].rb);
const struct intel_engine_cs *inflight =
intel_context_inflight(&ve->context);
if (!inflight || inflight == engine) {
struct i915_request *next;
rcu_read_lock();
next = READ_ONCE(ve->request);
if (next)
hint = max(hint, rq_prio(next));
rcu_read_unlock();
}
}
hint = max(engine->execlists.queue_priority_hint,
virtual_prio(&engine->execlists));
if (!list_is_last(&rq->sched.link, &engine->active.requests))
hint = max(hint, rq_prio(list_next_entry(rq, sched.link)));
@ -1256,6 +1256,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
struct i915_request **port = execlists->pending;
struct i915_request ** const last_port = port + execlists->port_mask;
struct i915_request *last = *execlists->active;
struct virtual_engine *ve;
struct rb_node *rb;
bool submit = false;
@ -1283,26 +1284,6 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
spin_lock(&engine->active.lock);
for (rb = rb_first_cached(&execlists->virtual); rb; ) {
struct virtual_engine *ve =
rb_entry(rb, typeof(*ve), nodes[engine->id].rb);
struct i915_request *rq = READ_ONCE(ve->request);
if (!rq) { /* lazily cleanup after another engine handled rq */
rb_erase_cached(rb, &execlists->virtual);
RB_CLEAR_NODE(rb);
rb = rb_first_cached(&execlists->virtual);
continue;
}
if (!virtual_matches(ve, rq, engine)) {
rb = rb_next(rb);
continue;
}
break;
}
/*
* If the queue is higher priority than the last
* request in the currently active context, submit afresh.
@ -1325,7 +1306,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
if (last) {
if (i915_request_completed(last)) {
goto check_secondary;
} else if (need_preempt(engine, last, rb)) {
} else if (need_preempt(engine, last)) {
ENGINE_TRACE(engine,
"preempting last=%llx:%lld, prio=%d, hint=%d\n",
last->fence.context,
@ -1351,7 +1332,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
__unwind_incomplete_requests(engine);
last = NULL;
} else if (need_timeslice(engine, last, rb) &&
} else if (need_timeslice(engine, last) &&
timeslice_expired(execlists, last)) {
ENGINE_TRACE(engine,
"expired last=%llx:%lld, prio=%d, hint=%d, yield?=%s\n",
@ -1402,96 +1383,86 @@ check_secondary:
}
}
while (rb) { /* XXX virtual is always taking precedence */
struct virtual_engine *ve =
rb_entry(rb, typeof(*ve), nodes[engine->id].rb);
/* XXX virtual is always taking precedence */
while ((ve = first_virtual_engine(engine))) {
struct i915_request *rq;
spin_lock(&ve->base.active.lock);
rq = ve->request;
if (unlikely(!rq)) { /* lost the race to a sibling */
spin_unlock(&ve->base.active.lock);
rb_erase_cached(rb, &execlists->virtual);
RB_CLEAR_NODE(rb);
rb = rb_first_cached(&execlists->virtual);
continue;
}
if (unlikely(!rq)) /* lost the race to a sibling */
goto unlock;
GEM_BUG_ON(rq != ve->request);
GEM_BUG_ON(rq->engine != &ve->base);
GEM_BUG_ON(rq->context != &ve->context);
if (rq_prio(rq) >= queue_prio(execlists)) {
if (!virtual_matches(ve, rq, engine)) {
spin_unlock(&ve->base.active.lock);
rb = rb_next(rb);
continue;
}
if (last && !can_merge_rq(last, rq)) {
spin_unlock(&ve->base.active.lock);
spin_unlock(&engine->active.lock);
start_timeslice(engine, rq_prio(rq));
return; /* leave this for another sibling */
}
ENGINE_TRACE(engine,
"virtual rq=%llx:%lld%s, new engine? %s\n",
rq->fence.context,
rq->fence.seqno,
i915_request_completed(rq) ? "!" :
i915_request_started(rq) ? "*" :
"",
yesno(engine != ve->siblings[0]));
WRITE_ONCE(ve->request, NULL);
WRITE_ONCE(ve->base.execlists.queue_priority_hint,
INT_MIN);
rb_erase_cached(rb, &execlists->virtual);
RB_CLEAR_NODE(rb);
GEM_BUG_ON(!(rq->execution_mask & engine->mask));
WRITE_ONCE(rq->engine, engine);
if (__i915_request_submit(rq)) {
/*
* Only after we confirm that we will submit
* this request (i.e. it has not already
* completed), do we want to update the context.
*
* This serves two purposes. It avoids
* unnecessary work if we are resubmitting an
* already completed request after timeslicing.
* But more importantly, it prevents us altering
* ve->siblings[] on an idle context, where
* we may be using ve->siblings[] in
* virtual_context_enter / virtual_context_exit.
*/
virtual_xfer_context(ve, engine);
GEM_BUG_ON(ve->siblings[0] != engine);
submit = true;
last = rq;
}
i915_request_put(rq);
/*
* Hmm, we have a bunch of virtual engine requests,
* but the first one was already completed (thanks
* preempt-to-busy!). Keep looking at the veng queue
* until we have no more relevant requests (i.e.
* the normal submit queue has higher priority).
*/
if (!submit) {
spin_unlock(&ve->base.active.lock);
rb = rb_first_cached(&execlists->virtual);
continue;
}
if (unlikely(rq_prio(rq) < queue_prio(execlists))) {
spin_unlock(&ve->base.active.lock);
break;
}
GEM_BUG_ON(!virtual_matches(ve, rq, engine));
if (last && !can_merge_rq(last, rq)) {
spin_unlock(&ve->base.active.lock);
spin_unlock(&engine->active.lock);
start_timeslice(engine, rq_prio(rq));
return; /* leave this for another sibling */
}
ENGINE_TRACE(engine,
"virtual rq=%llx:%lld%s, new engine? %s\n",
rq->fence.context,
rq->fence.seqno,
i915_request_completed(rq) ? "!" :
i915_request_started(rq) ? "*" :
"",
yesno(engine != ve->siblings[0]));
WRITE_ONCE(ve->request, NULL);
WRITE_ONCE(ve->base.execlists.queue_priority_hint, INT_MIN);
rb = &ve->nodes[engine->id].rb;
rb_erase_cached(rb, &execlists->virtual);
RB_CLEAR_NODE(rb);
GEM_BUG_ON(!(rq->execution_mask & engine->mask));
WRITE_ONCE(rq->engine, engine);
if (__i915_request_submit(rq)) {
/*
* Only after we confirm that we will submit
* this request (i.e. it has not already
* completed), do we want to update the context.
*
* This serves two purposes. It avoids
* unnecessary work if we are resubmitting an
* already completed request after timeslicing.
* But more importantly, it prevents us altering
* ve->siblings[] on an idle context, where
* we may be using ve->siblings[] in
* virtual_context_enter / virtual_context_exit.
*/
virtual_xfer_context(ve, engine);
GEM_BUG_ON(ve->siblings[0] != engine);
submit = true;
last = rq;
}
i915_request_put(rq);
unlock:
spin_unlock(&ve->base.active.lock);
break;
/*
* Hmm, we have a bunch of virtual engine requests,
* but the first one was already completed (thanks
* preempt-to-busy!). Keep looking at the veng queue
* until we have no more relevant requests (i.e.
* the normal submit queue has higher priority).
*/
if (submit)
break;
}
while ((rb = rb_first_cached(&execlists->queue))) {