From 67059b9fb8997f3d4515d72052c331503b00274b Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:32 +1000 Subject: [PATCH] drm/nouveau/fifo: add chan start()/stop() - nvkm_chan_error() built on top, stops channel and sends 'killed' event - removes an odd double-bashing of channel enable regs on kepler and up - pokes doorbell on turing and up, after enabling channel Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../drm/nouveau/include/nvkm/engine/fifo.h | 7 +- .../gpu/drm/nouveau/nvkm/engine/fifo/base.c | 13 +--- .../gpu/drm/nouveau/nvkm/engine/fifo/chan.c | 73 +++++++++++++------ .../gpu/drm/nouveau/nvkm/engine/fifo/chan.h | 7 +- .../drm/nouveau/nvkm/engine/fifo/channv04.h | 2 - .../drm/nouveau/nvkm/engine/fifo/channv50.c | 7 -- .../drm/nouveau/nvkm/engine/fifo/dmanv04.c | 2 - .../drm/nouveau/nvkm/engine/fifo/dmanv40.c | 2 - .../gpu/drm/nouveau/nvkm/engine/fifo/g84.c | 2 + .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 22 +++++- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 24 ++++-- .../gpu/drm/nouveau/nvkm/engine/fifo/gk110.c | 2 + .../gpu/drm/nouveau/nvkm/engine/fifo/gm107.c | 2 + .../nouveau/nvkm/engine/fifo/gpfifogf100.c | 6 -- .../nouveau/nvkm/engine/fifo/gpfifogk104.c | 7 -- .../nouveau/nvkm/engine/fifo/gpfifogv100.c | 9 +-- .../nouveau/nvkm/engine/fifo/gpfifotu102.c | 8 -- .../gpu/drm/nouveau/nvkm/engine/fifo/gv100.c | 9 +++ .../gpu/drm/nouveau/nvkm/engine/fifo/nv04.c | 27 ++++--- .../gpu/drm/nouveau/nvkm/engine/fifo/nv10.c | 2 + .../gpu/drm/nouveau/nvkm/engine/fifo/nv17.c | 2 + .../gpu/drm/nouveau/nvkm/engine/fifo/nv40.c | 2 + .../gpu/drm/nouveau/nvkm/engine/fifo/nv50.c | 18 +++++ .../gpu/drm/nouveau/nvkm/engine/fifo/priv.h | 7 +- .../gpu/drm/nouveau/nvkm/engine/fifo/runl.h | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/tu102.c | 25 +++++-- .../gpu/drm/nouveau/nvkm/engine/fifo/uchan.c | 11 ++- 27 files changed, 191 insertions(+), 108 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h index a4cc7f07bb89..798fbd3b530c 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h @@ -20,6 +20,10 @@ struct nvkm_chan { union { int id; int chid; }; /*FIXME: remove later */ + spinlock_t lock; + atomic_t blocked; + atomic_t errored; + struct list_head cctxs; struct nvkm_fifo *fifo; @@ -62,9 +66,6 @@ struct nvkm_fifo { struct list_head chan; spinlock_t lock; struct mutex mutex; - -#define NVKM_FIFO_EVENT_KILLED BIT(0) - struct nvkm_event kevent; /* channel killed */ }; void nvkm_fifo_fault(struct nvkm_fifo *, struct nvkm_fault_data *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index e08d8d14a19a..549e4b1fcbe1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -62,16 +62,6 @@ nvkm_fifo_fault(struct nvkm_fifo *fifo, struct nvkm_fault_data *info) return fifo->func->mmu_fault->recover(fifo, info); } -void -nvkm_fifo_kevent(struct nvkm_fifo *fifo, int chid) -{ - nvkm_event_ntfy(&fifo->kevent, chid, NVKM_FIFO_EVENT_KILLED); -} - -static const struct nvkm_event_func -nvkm_fifo_kevent_func = { -}; - static int nvkm_fifo_class_new(struct nvkm_device *device, const struct nvkm_oclass *oclass, void *argv, u32 argc, struct nvkm_object **pobject) @@ -293,7 +283,6 @@ nvkm_fifo_dtor(struct nvkm_engine *engine) if (fifo->func->dtor) data = fifo->func->dtor(fifo); - nvkm_event_fini(&fifo->kevent); nvkm_event_fini(&fifo->nonstall.event); mutex_destroy(&fifo->mutex); return data; @@ -343,5 +332,5 @@ nvkm_fifo_ctor(const struct nvkm_fifo_func *func, struct nvkm_device *device, return ret; } - return nvkm_event_init(&nvkm_fifo_kevent_func, &fifo->engine.subdev, 1, nr, &fifo->kevent); + return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c index 748f3f199b9f..4f11448ed720 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c @@ -24,6 +24,7 @@ #include "chan.h" #include "chid.h" #include "cgrp.h" +#include "chid.h" #include "runl.h" #include "priv.h" @@ -219,6 +220,8 @@ nvkm_chan_cctx_bind(struct nvkm_chan *chan, struct nvkm_oproxy *oproxy, struct n */ if (cgrp->hw) nvkm_runl_block(runl); + else + nvkm_chan_block(chan); /* Update context pointer. */ if (cctx) @@ -229,6 +232,8 @@ nvkm_chan_cctx_bind(struct nvkm_chan *chan, struct nvkm_oproxy *oproxy, struct n /* Resume normal operation. */ if (cgrp->hw) nvkm_runl_allow(runl); + else + nvkm_chan_allow(chan); } void @@ -295,23 +300,6 @@ done: return ret; } -static int -nvkm_fifo_chan_uevent(struct nvkm_object *object, void *argv, u32 argc, struct nvkm_uevent *uevent) -{ - struct nvkm_fifo_chan *chan = nvkm_fifo_chan(object); - union nvif_chan_event_args *args = argv; - - switch (args->v0.type) { - case NVIF_CHAN_EVENT_V0_KILLED: - return nvkm_uevent_add(uevent, &chan->fifo->kevent, chan->chid, - NVKM_FIFO_EVENT_KILLED, NULL); - default: - break; - } - - return -ENOSYS; -} - static int nvkm_fifo_chan_map(struct nvkm_object *object, void *argv, u32 argc, enum nvkm_object_map *type, u64 *addr, u64 *size) @@ -327,7 +315,8 @@ static int nvkm_fifo_chan_fini(struct nvkm_object *object, bool suspend) { struct nvkm_fifo_chan *chan = nvkm_fifo_chan(object); - chan->func->fini(chan); + if (chan->func->fini) + chan->func->fini(chan); return 0; } @@ -335,10 +324,51 @@ static int nvkm_fifo_chan_init(struct nvkm_object *object) { struct nvkm_fifo_chan *chan = nvkm_fifo_chan(object); - chan->func->init(chan); + if (chan->func->init) + chan->func->init(chan); return 0; } +static void +nvkm_chan_block_locked(struct nvkm_chan *chan) +{ + CHAN_TRACE(chan, "block %d", atomic_read(&chan->blocked)); + if (atomic_inc_return(&chan->blocked) == 1) + chan->func->stop(chan); +} + +void +nvkm_chan_error(struct nvkm_chan *chan, bool preempt) +{ + unsigned long flags; + + spin_lock_irqsave(&chan->lock, flags); + if (atomic_inc_return(&chan->errored) == 1) { + CHAN_ERROR(chan, "errored - disabling channel"); + nvkm_chan_block_locked(chan); + nvkm_event_ntfy(&chan->cgrp->runl->chid->event, chan->id, NVKM_CHAN_EVENT_ERRORED); + } + spin_unlock_irqrestore(&chan->lock, flags); +} + +void +nvkm_chan_block(struct nvkm_chan *chan) +{ + spin_lock_irq(&chan->lock); + nvkm_chan_block_locked(chan); + spin_unlock_irq(&chan->lock); +} + +void +nvkm_chan_allow(struct nvkm_chan *chan) +{ + spin_lock_irq(&chan->lock); + CHAN_TRACE(chan, "allow %d", atomic_read(&chan->blocked)); + if (atomic_dec_and_test(&chan->blocked)) + chan->func->start(chan); + spin_unlock_irq(&chan->lock); +} + void nvkm_chan_del(struct nvkm_chan **pchan) { @@ -437,7 +467,6 @@ nvkm_fifo_chan_func = { .init = nvkm_fifo_chan_init, .fini = nvkm_fifo_chan_fini, .map = nvkm_fifo_chan_map, - .uevent = nvkm_fifo_chan_uevent, }; int @@ -481,10 +510,12 @@ nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *fn, func->engine_fini = fn->engine_fini; func->object_ctor = fn->object_ctor; func->object_dtor = fn->object_dtor; - func->submit_token = fn->submit_token; chan->func = func; chan->id = -1; + spin_lock_init(&chan->lock); + atomic_set(&chan->blocked, 1); + atomic_set(&chan->errored, 0); nvkm_object_ctor(&nvkm_fifo_chan_func, oclass, &chan->object); chan->fifo = fifo; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h index fe398ed544cf..d573d9fdb4d8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h @@ -18,6 +18,9 @@ struct nvkm_cctx { struct nvkm_chan_func { void (*bind)(struct nvkm_chan *); void (*unbind)(struct nvkm_chan *); + void (*start)(struct nvkm_chan *); + void (*stop)(struct nvkm_chan *); + u32 (*doorbell_handle)(struct nvkm_chan *); void *(*dtor)(struct nvkm_fifo_chan *); void (*init)(struct nvkm_fifo_chan *); @@ -30,7 +33,6 @@ struct nvkm_chan_func { bool suspend); int (*object_ctor)(struct nvkm_fifo_chan *, struct nvkm_object *); void (*object_dtor)(struct nvkm_fifo_chan *, int); - u32 (*submit_token)(struct nvkm_fifo_chan *); }; int nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *, struct nvkm_fifo *, @@ -38,6 +40,9 @@ int nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *, struct nvkm_fifo *, u32 engm, int bar, u32 base, u32 user, const struct nvkm_oclass *, struct nvkm_fifo_chan *); void nvkm_chan_del(struct nvkm_chan **); +void nvkm_chan_allow(struct nvkm_chan *); +void nvkm_chan_block(struct nvkm_chan *); +void nvkm_chan_error(struct nvkm_chan *, bool preempt); int nvkm_chan_cctx_get(struct nvkm_chan *, struct nvkm_engn *, struct nvkm_cctx **, struct nvkm_client * /*TODO: remove need for this */); void nvkm_chan_cctx_put(struct nvkm_chan *, struct nvkm_cctx **); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv04.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv04.h index 727bc8976b40..837f9f7f69ce 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv04.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv04.h @@ -18,8 +18,6 @@ struct nv04_fifo_chan { extern const struct nvkm_fifo_chan_func nv04_fifo_dma_func; void *nv04_fifo_dma_dtor(struct nvkm_fifo_chan *); -void nv04_fifo_dma_init(struct nvkm_fifo_chan *); -void nv04_fifo_dma_fini(struct nvkm_fifo_chan *); void nv04_fifo_dma_object_dtor(struct nvkm_fifo_chan *, int); extern const struct nvkm_fifo_chan_oclass nv04_fifo_dma_oclass; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c index 0fc97c4c341a..7669d17406c6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c @@ -188,11 +188,7 @@ nv50_fifo_chan_fini(struct nvkm_fifo_chan *base) { struct nv50_fifo_chan *chan = nv50_fifo_chan(base); struct nv50_fifo *fifo = chan->fifo; - struct nvkm_device *device = fifo->base.engine.subdev.device; - u32 chid = chan->base.chid; - /* remove channel from runlist, fifo will unload context */ - nvkm_mask(device, 0x002600 + (chid * 4), 0x80000000, 0x00000000); nv50_fifo_runlist_update(fifo); } @@ -201,10 +197,7 @@ nv50_fifo_chan_init(struct nvkm_fifo_chan *base) { struct nv50_fifo_chan *chan = nv50_fifo_chan(base); struct nv50_fifo *fifo = chan->fifo; - struct nvkm_device *device = fifo->base.engine.subdev.device; - u32 chid = chan->base.chid; - nvkm_mask(device, 0x002600 + (chid * 4), 0x80000000, 0x80000000); nv50_fifo_runlist_update(fifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c index ed2ae9e69987..d515cf08fc3b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c @@ -88,8 +88,6 @@ nv04_fifo_dma_dtor(struct nvkm_fifo_chan *base) const struct nvkm_fifo_chan_func nv04_fifo_dma_func = { .dtor = nv04_fifo_dma_dtor, - .init = nv04_fifo_dma_init, - .fini = nv04_fifo_dma_fini, .object_ctor = nv04_fifo_dma_object_ctor, .object_dtor = nv04_fifo_dma_object_dtor, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv40.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv40.c index c42c2e01d801..2f0dceaf3b01 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv40.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv40.c @@ -176,8 +176,6 @@ nv40_fifo_dma_object_ctor(struct nvkm_fifo_chan *base, static const struct nvkm_fifo_chan_func nv40_fifo_dma_func = { .dtor = nv04_fifo_dma_dtor, - .init = nv04_fifo_dma_init, - .fini = nv04_fifo_dma_fini, .engine_ctor = nv40_fifo_dma_engine_ctor, .engine_dtor = nv40_fifo_dma_engine_dtor, .engine_init = nv40_fifo_dma_engine_init, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c index 2777f03ffc29..91b43f17120b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c @@ -44,6 +44,8 @@ const struct nvkm_chan_func g84_chan = { .bind = g84_chan_bind, .unbind = nv50_chan_unbind, + .start = nv50_chan_start, + .stop = nv50_chan_stop, }; const struct nvkm_engn_func diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index 50109f4f3860..e47ca113e9e8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -39,6 +39,22 @@ #include +static void +gf100_chan_stop(struct nvkm_chan *chan) +{ + struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device; + + nvkm_mask(device, 0x003004 + (chan->id * 8), 0x00000001, 0x00000000); +} + +static void +gf100_chan_start(struct nvkm_chan *chan) +{ + struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device; + + nvkm_wr32(device, 0x003004 + (chan->id * 8), 0x001f0001); +} + static void gf100_fifo_intr_engine(struct nvkm_fifo *); static void @@ -65,6 +81,8 @@ static const struct nvkm_chan_func gf100_chan = { .bind = gf100_chan_bind, .unbind = gf100_chan_unbind, + .start = gf100_chan_start, + .stop = gf100_chan_stop, }; static const struct nvkm_engn_func @@ -321,7 +339,6 @@ gf100_fifo_recover(struct gf100_fifo *fifo, struct nvkm_engine *engine, struct gf100_fifo_chan *chan) { struct nvkm_subdev *subdev = &fifo->base.engine.subdev; - struct nvkm_device *device = subdev->device; u32 chid = chan->base.chid; int engi = gf100_fifo_engine_id(&fifo->base, engine); @@ -329,14 +346,13 @@ gf100_fifo_recover(struct gf100_fifo *fifo, struct nvkm_engine *engine, engine->subdev.name, chid); assert_spin_locked(&fifo->base.lock); - nvkm_mask(device, 0x003004 + (chid * 0x08), 0x00000001, 0x00000000); + nvkm_chan_error(&chan->base, false); list_del_init(&chan->head); chan->killed = true; if (engi >= 0 && engi != GF100_FIFO_ENGN_SW) fifo->recover.mask |= BIT(engi); schedule_work(&fifo->recover.work); - nvkm_fifo_kevent(&fifo->base, chid); } static const struct nvkm_enum diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index fafe9453ab0c..5ba922cf3c2c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -38,6 +38,22 @@ #include +void +gk104_chan_stop(struct nvkm_chan *chan) +{ + struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device; + + nvkm_mask(device, 0x800004 + (chan->id * 8), 0x00000800, 0x00000800); +} + +void +gk104_chan_start(struct nvkm_chan *chan) +{ + struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device; + + nvkm_mask(device, 0x800004 + (chan->id * 8), 0x00000400, 0x00000400); +} + void gk104_chan_unbind(struct nvkm_chan *chan) { @@ -68,6 +84,8 @@ static const struct nvkm_chan_func gk104_chan = { .bind = gk104_chan_bind, .unbind = gk104_chan_unbind, + .start = gk104_chan_start, + .stop = gk104_chan_stop, }; void @@ -469,13 +487,9 @@ gk104_fifo_recover_chan(struct nvkm_fifo *base, int chid) chan = gk104_fifo_recover_chid(fifo, runl, chid); if (chan) { chan->killed = true; - nvkm_fifo_kevent(&fifo->base, chid); + nvkm_chan_error(&chan->base, false); } - /* Disable channel. */ - nvkm_wr32(device, 0x800004 + (chid * 0x08), stat | 0x00000800); - nvkm_warn(subdev, "channel %d: killed\n", chid); - /* Block channel assignments from changing during recovery. */ gk104_fifo_recover_runl(fifo, runl); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c index 1724937a2f5f..3582b9f55611 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c @@ -37,6 +37,8 @@ const struct nvkm_chan_func gk110_chan = { .bind = gk104_chan_bind, .unbind = gk104_chan_unbind, + .start = gk104_chan_start, + .stop = gk104_chan_stop, }; const struct nvkm_cgrp_func diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c index 742404bf8415..2924ce10c337 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c @@ -36,6 +36,8 @@ const struct nvkm_chan_func gm107_chan = { .bind = gk104_chan_bind_inst, .unbind = gk104_chan_unbind, + .start = gk104_chan_start, + .stop = gk104_chan_stop, }; static void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c index 93739b34bfcd..29783d271f1e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c @@ -149,12 +149,9 @@ gf100_fifo_gpfifo_fini(struct nvkm_fifo_chan *base) { struct gf100_fifo_chan *chan = gf100_fifo_chan(base); struct gf100_fifo *fifo = chan->fifo; - struct nvkm_device *device = fifo->base.engine.subdev.device; - u32 coff = chan->base.chid * 8; if (!list_empty(&chan->head) && !chan->killed) { gf100_fifo_runlist_remove(fifo, chan); - nvkm_mask(device, 0x003004 + coff, 0x00000001, 0x00000000); gf100_fifo_runlist_commit(fifo); } } @@ -164,12 +161,9 @@ gf100_fifo_gpfifo_init(struct nvkm_fifo_chan *base) { struct gf100_fifo_chan *chan = gf100_fifo_chan(base); struct gf100_fifo *fifo = chan->fifo; - struct nvkm_device *device = fifo->base.engine.subdev.device; - u32 coff = chan->base.chid * 8; if (list_empty(&chan->head) && !chan->killed) { gf100_fifo_runlist_insert(fifo, chan); - nvkm_wr32(device, 0x003004 + coff, 0x001f0001); gf100_fifo_runlist_commit(fifo); } } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c index c2b5eaa9cdea..1847b1b7af8b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c @@ -199,12 +199,9 @@ gk104_fifo_gpfifo_fini(struct nvkm_fifo_chan *base) { struct gk104_fifo_chan *chan = gk104_fifo_chan(base); struct gk104_fifo *fifo = chan->fifo; - struct nvkm_device *device = fifo->base.engine.subdev.device; - u32 coff = chan->base.chid * 8; if (!list_empty(&chan->head)) { gk104_fifo_runlist_remove(fifo, chan); - nvkm_mask(device, 0x800004 + coff, 0x00000800, 0x00000800); gk104_fifo_gpfifo_kick(chan); gk104_fifo_runlist_update(fifo, chan->runl); } @@ -215,14 +212,10 @@ gk104_fifo_gpfifo_init(struct nvkm_fifo_chan *base) { struct gk104_fifo_chan *chan = gk104_fifo_chan(base); struct gk104_fifo *fifo = chan->fifo; - struct nvkm_device *device = fifo->base.engine.subdev.device; - u32 coff = chan->base.chid * 8; if (list_empty(&chan->head) && !chan->killed) { gk104_fifo_runlist_insert(fifo, chan); - nvkm_mask(device, 0x800004 + coff, 0x00000400, 0x00000400); gk104_fifo_runlist_update(fifo, chan->runl); - nvkm_mask(device, 0x800004 + coff, 0x00000400, 0x00000400); } } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c index 6e74fdc0dc24..a901ce269e06 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c @@ -28,12 +28,6 @@ #include #include -static u32 -gv100_fifo_gpfifo_submit_token(struct nvkm_fifo_chan *chan) -{ - return chan->chid; -} - static int gv100_fifo_gpfifo_engine_valid(struct gk104_fifo_chan *chan, bool ce, bool valid) { @@ -125,7 +119,6 @@ gv100_fifo_gpfifo = { .engine_dtor = gk104_fifo_gpfifo_engine_dtor, .engine_init = gv100_fifo_gpfifo_engine_init, .engine_fini = gv100_fifo_gpfifo_engine_fini, - .submit_token = gv100_fifo_gpfifo_submit_token, }; int @@ -159,7 +152,7 @@ gv100_fifo_gpfifo_new_(const struct nvkm_fifo_chan_func *func, *chid = chan->base.chid; *inst = chan->base.inst->addr; - *token = chan->base.func->submit_token(&chan->base); + *token = chan->base.func->doorbell_handle(&chan->base); /* Hack to support GPUs where even individual channels should be * part of a channel group. diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c index 4d35bee0ab54..1c48a109a2b8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c @@ -28,13 +28,6 @@ #include #include -static u32 -tu102_fifo_gpfifo_submit_token(struct nvkm_fifo_chan *base) -{ - struct gk104_fifo_chan *chan = gk104_fifo_chan(base); - return (chan->runl << 16) | chan->base.chid; -} - static const struct nvkm_fifo_chan_func tu102_fifo_gpfifo = { .dtor = gk104_fifo_gpfifo_dtor, @@ -44,7 +37,6 @@ tu102_fifo_gpfifo = { .engine_dtor = gk104_fifo_gpfifo_engine_dtor, .engine_init = gv100_fifo_gpfifo_engine_init, .engine_fini = gv100_fifo_gpfifo_engine_fini, - .submit_token = tu102_fifo_gpfifo_submit_token, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c index eb59527544b2..0e9e070201b1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c @@ -31,10 +31,19 @@ #include +static u32 +gv100_chan_doorbell_handle(struct nvkm_chan *chan) +{ + return chan->id; +} + static const struct nvkm_chan_func gv100_chan = { .bind = gk104_chan_bind_inst, .unbind = gk104_chan_unbind, + .start = gk104_chan_start, + .stop = gk104_chan_stop, + .doorbell_handle = gv100_chan_doorbell_handle, }; const struct nvkm_engn_func diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c index 446e65dcb036..a0a28a81f7f4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c @@ -52,15 +52,14 @@ nv04_fifo_ramfc[] = { }; void -nv04_fifo_dma_fini(struct nvkm_fifo_chan *base) +nv04_chan_stop(struct nvkm_chan *chan) { - struct nv04_fifo_chan *chan = nv04_fifo_chan(base); - struct nv04_fifo *fifo = chan->fifo; + struct nv04_fifo *fifo = nv04_fifo(chan->cgrp->runl->fifo); struct nvkm_device *device = fifo->base.engine.subdev.device; struct nvkm_memory *fctx = device->imem->ramfc; const struct nv04_fifo_ramfc *c; unsigned long flags; - u32 data = chan->ramfc; + u32 data = nv04_fifo_chan(chan)->ramfc; u32 chid; /* prevent fifo context switches */ @@ -69,7 +68,7 @@ nv04_fifo_dma_fini(struct nvkm_fifo_chan *base) /* if this channel is active, replace it with a null context */ chid = nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH1) & fifo->base.chid->mask; - if (chid == chan->base.chid) { + if (chid == chan->id) { nvkm_mask(device, NV04_PFIFO_CACHE1_DMA_PUSH, 0x00000001, 0); nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 0); nvkm_mask(device, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0); @@ -98,26 +97,26 @@ nv04_fifo_dma_fini(struct nvkm_fifo_chan *base) } /* restore normal operation, after disabling dma mode */ - nvkm_mask(device, NV04_PFIFO_MODE, 1 << chan->base.chid, 0); + nvkm_mask(device, NV04_PFIFO_MODE, BIT(chan->id), 0); nvkm_wr32(device, NV03_PFIFO_CACHES, 1); spin_unlock_irqrestore(&fifo->base.lock, flags); } void -nv04_fifo_dma_init(struct nvkm_fifo_chan *base) +nv04_chan_start(struct nvkm_chan *chan) { - struct nv04_fifo_chan *chan = nv04_fifo_chan(base); - struct nv04_fifo *fifo = chan->fifo; - struct nvkm_device *device = fifo->base.engine.subdev.device; - u32 mask = 1 << chan->base.chid; + struct nvkm_fifo *fifo = chan->cgrp->runl->fifo; unsigned long flags; - spin_lock_irqsave(&fifo->base.lock, flags); - nvkm_mask(device, NV04_PFIFO_MODE, mask, mask); - spin_unlock_irqrestore(&fifo->base.lock, flags); + + spin_lock_irqsave(&fifo->lock, flags); + nvkm_mask(fifo->engine.subdev.device, NV04_PFIFO_MODE, BIT(chan->id), BIT(chan->id)); + spin_unlock_irqrestore(&fifo->lock, flags); } static const struct nvkm_chan_func nv04_chan = { + .start = nv04_chan_start, + .stop = nv04_chan_stop, }; const struct nvkm_cgrp_func diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c index 0f7c88ae5a1c..8d302cb8ba8c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c @@ -45,6 +45,8 @@ nv10_fifo_ramfc[] = { static const struct nvkm_chan_func nv10_chan = { + .start = nv04_chan_start, + .stop = nv04_chan_stop, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c index 59b752986ac1..7b1b90d285c1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c @@ -54,6 +54,8 @@ nv17_fifo_ramfc[] = { static const struct nvkm_chan_func nv17_chan = { + .start = nv04_chan_start, + .stop = nv04_chan_stop, }; static void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c index d0fc6c528825..19f7c0407e70 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c @@ -64,6 +64,8 @@ nv40_fifo_ramfc[] = { static const struct nvkm_chan_func nv40_chan = { + .start = nv04_chan_start, + .stop = nv04_chan_stop, }; static const struct nvkm_engn_func diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c index d7adb057e4ca..38d36749862c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c @@ -34,6 +34,22 @@ #include +void +nv50_chan_stop(struct nvkm_chan *chan) +{ + struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device; + + nvkm_mask(device, 0x002600 + (chan->id * 4), 0x80000000, 0x00000000); +} + +void +nv50_chan_start(struct nvkm_chan *chan) +{ + struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device; + + nvkm_mask(device, 0x002600 + (chan->id * 4), 0x80000000, 0x80000000); +} + void nv50_chan_unbind(struct nvkm_chan *chan) { @@ -54,6 +70,8 @@ static const struct nvkm_chan_func nv50_chan = { .bind = nv50_chan_bind, .unbind = nv50_chan_unbind, + .start = nv50_chan_start, + .stop = nv50_chan_stop, }; static const struct nvkm_engn_func diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index f023aa0a559e..d9de5ab26452 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -11,7 +11,6 @@ struct nvkm_runq; struct gk104_fifo; struct gk104_fifo_chan; -void nvkm_fifo_kevent(struct nvkm_fifo *, int chid); void nvkm_fifo_recover_chan(struct nvkm_fifo *, int chid); struct nvkm_fifo_chan_oclass; @@ -95,6 +94,8 @@ void nv04_fifo_start(struct nvkm_fifo *, unsigned long *); extern const struct nvkm_runl_func nv04_runl; extern const struct nvkm_engn_func nv04_engn; extern const struct nvkm_cgrp_func nv04_cgrp; +void nv04_chan_start(struct nvkm_chan *); +void nv04_chan_stop(struct nvkm_chan *); int nv10_fifo_chid_nr(struct nvkm_fifo *); @@ -104,6 +105,8 @@ extern const struct nvkm_runl_func nv50_runl; int nv50_runl_wait(struct nvkm_runl *); extern const struct nvkm_engn_func nv50_engn_sw; void nv50_chan_unbind(struct nvkm_chan *); +void nv50_chan_start(struct nvkm_chan *); +void nv50_chan_stop(struct nvkm_chan *); extern const struct nvkm_event_func g84_fifo_nonstall; extern const struct nvkm_engn_func g84_engn; @@ -146,6 +149,8 @@ extern const struct nvkm_engn_func gk104_engn_ce; void gk104_chan_bind(struct nvkm_chan *); void gk104_chan_bind_inst(struct nvkm_chan *); void gk104_chan_unbind(struct nvkm_chan *); +void gk104_chan_start(struct nvkm_chan *); +void gk104_chan_stop(struct nvkm_chan *); int gk110_fifo_chid_ctor(struct nvkm_fifo *, int); extern const struct nvkm_runl_func gk110_runl; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h index aebd8a750d38..35dce7c83124 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h @@ -34,6 +34,7 @@ struct nvkm_runl { u32 addr; struct nvkm_chid *cgid; +#define NVKM_CHAN_EVENT_ERRORED BIT(0) struct nvkm_chid *chid; struct list_head engns; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c index b23fc330aa6c..4ae8544a22cf 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c @@ -28,13 +28,32 @@ #include #include +#include #include +static u32 +tu102_chan_doorbell_handle(struct nvkm_chan *chan) +{ + return (chan->cgrp->runl->id << 16) | chan->id; +} + +static void +tu102_chan_start(struct nvkm_chan *chan) +{ + struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device; + + gk104_chan_start(chan); + nvkm_wr32(device, device->vfn->addr.user + 0x0090, chan->func->doorbell_handle(chan)); +} + static const struct nvkm_chan_func tu102_chan = { .bind = gk104_chan_bind_inst, .unbind = gk104_chan_unbind, + .start = tu102_chan_start, + .stop = gk104_chan_stop, + .doorbell_handle = tu102_chan_doorbell_handle, }; static bool @@ -202,13 +221,9 @@ tu102_fifo_recover_chan(struct nvkm_fifo *base, int chid) chan = tu102_fifo_recover_chid(fifo, runl, chid); if (chan) { chan->killed = true; - nvkm_fifo_kevent(&fifo->base, chid); + nvkm_chan_error(&chan->base, false); } - /* Disable channel. */ - nvkm_wr32(device, 0x800004 + (chid * 0x08), stat | 0x00000800); - nvkm_warn(subdev, "channel %d: killed\n", chid); - /* Block channel assignments from changing during recovery. */ tu102_fifo_recover_runl(fifo, runl); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c index c1d2035e00cf..096e09f23177 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c @@ -22,6 +22,7 @@ #define nvkm_uchan(p) container_of((p), struct nvkm_uchan, object) #include "cgrp.h" #include "chan.h" +#include "chid.h" #include "runl.h" #include @@ -52,7 +53,8 @@ nvkm_uchan_uevent(struct nvkm_object *object, void *argv, u32 argc, struct nvkm_ return nvkm_uevent_add(uevent, &runl->fifo->nonstall.event, 0, NVKM_FIFO_NONSTALL_EVENT, NULL); case NVIF_CHAN_EVENT_V0_KILLED: - return chan->object.func->uevent(&chan->object, argv, argc, uevent); + return nvkm_uevent_add(uevent, &runl->chid->event, chan->id, + NVKM_CHAN_EVENT_ERRORED, NULL); default: break; } @@ -213,6 +215,8 @@ nvkm_uchan_fini(struct nvkm_object *object, bool suspend) struct nvkm_chan *chan = nvkm_uchan(object)->chan; int ret; + nvkm_chan_block(chan); + ret = chan->object.func->fini(&chan->object, suspend); if (ret && suspend) return ret; @@ -228,9 +232,14 @@ nvkm_uchan_init(struct nvkm_object *object) { struct nvkm_chan *chan = nvkm_uchan(object)->chan; + if (atomic_read(&chan->errored)) + return 0; + if (chan->func->bind) chan->func->bind(chan); + nvkm_chan_allow(chan); + return chan->object.func->init(&chan->object); }