<html> <head> <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1"> <style type="text/css" style="display:none;"> P {margin-top:0;margin-bottom:0;} </style> </head> <body dir="ltr"> [AMD Official Use Only - Internal Distribution Only] <div> <div style="font-family: Calibri, Arial, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);"> I spotted two cosmetic issues when browsing it. see them inline [yz].</div> <div style="font-family: Calibri, Arial, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);"> </div> <div style="font-family: Calibri, Arial, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);"> Yong</div> <div id="appendonsend"></div> <div style="font-family:Calibri,Arial,Helvetica,sans-serif; font-size:12pt; color:rgb(0,0,0)"> </div> <hr tabindex="-1" style="display:inline-block; width:98%"> <div id="divRplyFwdMsg" dir="ltr">From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> on behalf of Gang Ba <gaba@amd.com> Sent: Tuesday, April 28, 2020 9:58 AM To: amd-gfx@lists.freedesktop.org <amd-gfx@lists.freedesktop.org> Subject: [PATCH] drm/amd: add Streaming Performance Monitor feature <div> </div> </div> <div class="BodyFragment"> <div class="PlainText">Signed-off-by: Gang Ba <gaba@amperd.com> Change-Id: If83ee0a14ef834699de6733eeab0f140159bbd6e --- drivers/gpu/drm/amd/amdgpu/Makefile | 3 +- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 10 + drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_rlc_spm.c | 165 ++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h | 27 + drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 175 +++++++ drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c | 176 +++++++ drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 168 ++++++ drivers/gpu/drm/amd/amdkfd/Makefile | 3 +- drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c | 10 +- drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 17 + drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c | 11 +- drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 20 + drivers/gpu/drm/amd/amdkfd/kfd_process.c | 1 + drivers/gpu/drm/amd/amdkfd/kfd_spm.c | 577 +++++++++++++++++++++ drivers/gpu/drm/amd/amdkfd/soc15_int.h | 1 + include/uapi/linux/kfd_ioctl.h | 55 +- 16 files changed, 1413 insertions(+), 6 deletions(-) create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_rlc_spm.c create mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_spm.c diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile index 210d57a..1498b18 100644 --- a/drivers/gpu/drm/amd/amdgpu/Makefile +++ b/drivers/gpu/drm/amd/amdgpu/Makefile @@ -177,7 +177,8 @@ amdgpu-y += \ amdgpu_amdkfd_gfx_v8.o \ amdgpu_amdkfd_gfx_v9.o \ amdgpu_amdkfd_arcturus.o \ - amdgpu_amdkfd_gfx_v10.o + amdgpu_amdkfd_gfx_v10.o \ + amdgpu_amdkfd_rlc_spm.o \ ifneq ($(CONFIG_DRM_AMDGPU_CIK),) amdgpu-y += amdgpu_amdkfd_gfx_v7.o diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h index d065c50..fdc438a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h @@ -246,6 +246,16 @@ void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo); int amdgpu_amdkfd_get_tile_config(struct kgd_dev *kgd, struct tile_config *config); +void amdgpu_amdkfd_rlc_spm_cntl(struct kgd_dev *kgd, bool cntl); +int amdgpu_amdkfd_rlc_spm(struct kgd_dev *kgd, void *args); +void amdgpu_amdkfd_rlc_spm_acquire(struct kgd_dev *kgd, u64 gpu_addr, u32 size); +int amdgpu_amdkfd_rlc_spm_release(struct kgd_dev *kgd); +u32 amdgpu_amdkfd_rlc_spm_get_rdptr(struct kgd_dev *kgd); +void amdgpu_amdkfd_rlc_spm_set_rdptr(struct kgd_dev *kgd, u32 rptr); +u32 amdgpu_amdkfd_rlc_spm_get_wrptr(struct kgd_dev *kgd); +void amdgpu_amdkfd_rlc_spm_set_wrptr(struct kgd_dev *kgd, u32 wptr); +int amdgpu_amdkfd_rlc_spm_set_reg(struct kgd_dev *kgd, u64 uReg, u32 value); + /* KGD2KFD callbacks */ int kgd2kfd_init(void); void kgd2kfd_exit(void); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_rlc_spm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_rlc_spm.c new file mode 100644 index 0000000..b492c1e --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_rlc_spm.c @@ -0,0 +1,165 @@ +/* + * Copyright 2014-2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include <linux/dma-buf.h> +#include <linux/list.h> +#include <linux/pagemap.h> +#include <linux/sched/mm.h> +#include <linux/sched/task.h> + +#include "amdgpu_object.h" +#include "amdgpu_vm.h" +#include "amdgpu_amdkfd.h" +#include <uapi/linux/kfd_ioctl.h> + + + +void amdgpu_amdkfd_rlc_spm_cntl(struct kgd_dev *kgd, bool cntl) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)kgd; + struct amdgpu_spm *spm = &adev->gfx.spm; + struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; + + spin_lock(&adev->gfx.kiq.ring_lock); + amdgpu_ring_alloc(kiq_ring, spm->spmf->get_spm_data_size); + cntl == true ? adev->gfx.spm.spmf->start(adev):adev->gfx.spm.spmf->stop(adev); + amdgpu_ring_commit(kiq_ring); + spin_unlock(&adev->gfx.kiq.ring_lock); +} + +u32 amdgpu_amdkfd_rlc_spm_get_rdptr(struct kgd_dev *kgd) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)kgd; + struct amdgpu_spm *spm = &adev->gfx.spm; + struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; + u32 rptr = 0; + + spin_lock(&adev->gfx.kiq.ring_lock); + amdgpu_ring_alloc(kiq_ring, spm->spmf->get_spm_data_size); + rptr = adev->gfx.spm.spmf->get_rdptr(adev); + amdgpu_ring_commit(kiq_ring); + spin_unlock(&adev->gfx.kiq.ring_lock); + + return rptr; +} + +void amdgpu_amdkfd_rlc_spm_set_rdptr(struct kgd_dev *kgd, u32 rptr) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)kgd; + struct amdgpu_spm *spm = &adev->gfx.spm; + struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; + + spin_lock(&adev->gfx.kiq.ring_lock); + amdgpu_ring_alloc(kiq_ring, spm->spmf->get_spm_data_size); + adev->gfx.spm.spmf->set_rdptr(adev, rptr); + amdgpu_ring_commit(kiq_ring); + spin_unlock(&adev->gfx.kiq.ring_lock); + +} + +u32 amdgpu_amdkfd_rlc_spm_get_wrptr(struct kgd_dev *kgd) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)kgd; + struct amdgpu_spm *spm = &adev->gfx.spm; + struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; + u32 wptr; + + spin_lock(&adev->gfx.kiq.ring_lock); + amdgpu_ring_alloc(kiq_ring, spm->spmf->get_spm_data_size); + wptr = adev->gfx.spm.spmf->get_wrptr(adev); + amdgpu_ring_commit(kiq_ring); + spin_unlock(&adev->gfx.kiq.ring_lock); + return wptr; +} + +void amdgpu_amdkfd_rlc_spm_set_wrptr(struct kgd_dev *kgd, u32 wptr) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)kgd; + struct amdgpu_spm *spm = &adev->gfx.spm; + struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; + + spin_lock(&adev->gfx.kiq.ring_lock); + amdgpu_ring_alloc(kiq_ring, spm->spmf->get_spm_data_size); + adev->gfx.spm.spmf->set_wrptr(adev, wptr); + amdgpu_ring_commit(kiq_ring); + spin_unlock(&adev->gfx.kiq.ring_lock); +} + +void amdgpu_amdkfd_rlc_spm_acquire(struct kgd_dev *kgd, u64 gpu_addr, u32 size) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)kgd; + struct amdgpu_spm *spm = &adev->gfx.spm; + struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; + + /* init spm vmid with 0xf */ + if (adev->gfx.rlc.funcs->update_spm_vmid) + adev->gfx.rlc.funcs->update_spm_vmid(adev, 0x0); + + /* set spm ring registers */ + spin_lock(&adev->gfx.kiq.ring_lock); + amdgpu_ring_alloc(kiq_ring, spm->spmf->get_spm_data_size); + adev->gfx.spm.spmf->set_spm_porfmon_ring_buf(adev, gpu_addr, size); + amdgpu_ring_commit(kiq_ring); + spin_unlock(&adev->gfx.kiq.ring_lock); +} + + +int amdgpu_amdkfd_rlc_spm_release(struct kgd_dev *kgd) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)kgd; + struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; + struct amdgpu_spm *spm = &adev->gfx.spm; + + /* stop spm stream and interupt */ + spin_lock(&adev->gfx.kiq.ring_lock); + amdgpu_ring_alloc(kiq_ring, spm->spmf->get_spm_data_size); + adev->gfx.spm.spmf->stop(adev); + amdgpu_ring_commit(kiq_ring); + spin_unlock(&adev->gfx.kiq.ring_lock); + + amdgpu_irq_put(adev, &adev->gfx.spm_irq, 0); + + /* revert spm vmid with 0xf */ + if (adev->gfx.rlc.funcs->update_spm_vmid) + adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf); + + return 0; +} + +int amdgpu_amdkfd_rlc_spm_set_reg(struct kgd_dev *kgd, u64 uReg, u32 value) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)kgd; + struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; + struct amdgpu_spm *spm = &adev->gfx.spm; + + pr_debug("[%s]\n", __func__); + + /* stop spm stream and interupt */ + spin_lock(&adev->gfx.kiq.ring_lock); + amdgpu_ring_alloc(kiq_ring, spm->spmf->get_spm_data_size); + adev->gfx.spm.spmf->set_reg(adev, uReg, value); + amdgpu_ring_commit(kiq_ring); + spin_unlock(&adev->gfx.kiq.ring_lock); + + return 0; +} + diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h index ee698f0..ba4da52 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h @@ -105,6 +105,31 @@ struct amdgpu_kiq { const struct kiq_pm4_funcs *pmf; }; +struct spm_funcs { + void (*start)(struct amdgpu_device *adev); + void (*stop)(struct amdgpu_device *adev); + void (*set_reg)(struct amdgpu_device *adev, uint64_t reg, uint32_t val); + u32 (*get_rdptr)(struct amdgpu_device *adev); + void (*set_rdptr)(struct amdgpu_device *adev, u32 rptr); + u32 (*get_wrptr)(struct amdgpu_device *adev); + void (*set_wrptr)(struct amdgpu_device *adev, u32 wptr); + void (*set_spm_porfmon_ring_buf)(struct amdgpu_device *adev, u64 gpu_rptr, u32 size); + + /* Packet sizes */ + int set_spm_config_size; + int get_spm_data_size; +}; + +struct amdgpu_spm { + const struct spm_funcs *spmf; + u64 spm_gpu_addr; + u32 spm_gpu_size; + u32 spm_ring_rptr; + u32 spm_ring_rptrsize_to_read; + struct amdgpu_bo *spm_obj; + void *spm_cpu_addr; +}; + /* * GPU scratch registers structures, functions & helpers */ @@ -256,6 +281,7 @@ struct amdgpu_gfx { struct amdgpu_me me; struct amdgpu_mec mec; struct amdgpu_kiq kiq; + struct amdgpu_spm spm; struct amdgpu_scratch scratch; const struct firmware *me_fw; /* ME firmware */ uint32_t me_fw_version; @@ -292,6 +318,7 @@ struct amdgpu_gfx { struct amdgpu_irq_src priv_reg_irq; struct amdgpu_irq_src priv_inst_irq; struct amdgpu_irq_src cp_ecc_error_irq; + struct amdgpu_irq_src spm_irq; struct amdgpu_irq_src sq_irq; struct sq_work sq_work; diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c index 63ac430..9c507d2 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c @@ -4121,6 +4121,13 @@ static int gfx_v10_0_sw_init(void *handle) if (r) return r; + /* KIQ SPM */ + r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_RLC, + GFX_10_1__SRCID__RLC_STRM_PERF_MONITOR_INTERRUPT, + &adev->gfx.spm_irq); + if (r) + return r; + /* EOP Event */ r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_10_1__SRCID__CP_EOP_INTERRUPT, @@ -6603,6 +6610,7 @@ static int gfx_v10_0_hw_fini(void *handle) amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); + amdgpu_irq_put(adev, &adev->gfx.spm_irq, 0); #ifndef BRING_UP_DEBUG if (amdgpu_async_gfx_ring) { r = gfx_v10_0_kiq_disable_kgq(adev); @@ -6768,6 +6776,126 @@ static void gfx_v10_0_ring_emit_gds_switch(struct amdgpu_ring *ring, (1 << (oa_size + oa_base)) - (1 << oa_base)); } +static void gfx_v10_0_spm_start(struct amdgpu_device *adev) +{ + struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; + uint32_t data = 0; + u8 se; + + for (se = 0; se < adev->gfx.config.max_shader_engines + 1; ++se) + { + uint32_t mux_addr_reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_GLOBAL_MUXSEL_ADDR); + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, + INSTANCE_BROADCAST_WRITES, 1); + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, + SA_BROADCAST_WRITES, 1); + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, + SE_BROADCAST_WRITES, 1); + + if (se < adev->gfx.config.max_shader_engines) // SE else GB + { + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se); + mux_addr_reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_SE_MUXSEL_ADDR); + } + gfx_v10_0_write_data_to_reg(kiq_ring, 0, false, SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX), data); + gfx_v10_0_write_data_to_reg(kiq_ring, 0, false, mux_addr_reg, 0); + } + + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, 0); + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1); + gfx_v10_0_write_data_to_reg(kiq_ring, 0, false, SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX), data); + + data = 0; + data |= CP_PERFMON_STATE_DISABLE_AND_RESET <<4 ; + gfx_v10_0_write_data_to_reg(kiq_ring, 0, false, SOC15_REG_OFFSET(GC, 0, mmCP_PERFMON_CNTL), data); + + data = 0; + data |= STRM_PERFMON_STATE_START_COUNTING <<4 ; + gfx_v10_0_write_data_to_reg(kiq_ring, 0, false, SOC15_REG_OFFSET(GC, 0, mmCP_PERFMON_CNTL), data); + + gfx_v10_0_write_data_to_reg(kiq_ring, 0, false, SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_INT_CNTL) ,1); +} + +static void gfx_v10_0_spm_stop(struct amdgpu_device *adev) +{ + struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; + uint32_t data = 0; + + data = CP_PERFMON_STATE_STOP_COUNTING; + gfx_v10_0_write_data_to_reg(kiq_ring, 0, false, SOC15_REG_OFFSET(GC, 0, mmCP_PERFMON_CNTL), data); + + data |= (1<<10); + gfx_v10_0_write_data_to_reg(kiq_ring, 0, false, SOC15_REG_OFFSET(GC, 0, mmCP_PERFMON_CNTL), data); + + gfx_v10_0_write_data_to_reg(kiq_ring, 0, false, SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_INT_CNTL), 0); +} + +static u32 gfx_v10_0_spm_get_rdptr(struct amdgpu_device *adev) +{ + return RREG32_SOC15(GC, 0, mmRLC_SPM_RING_RDPTR);; +} + +static void gfx_v10_0_spm_set_rdptr(struct amdgpu_device *adev, u32 rptr) +{ + struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; + + gfx_v10_0_write_data_to_reg(kiq_ring, 0, false, SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_RING_RDPTR), rptr); +} + +static u32 gfx_v10_0_spm_get_wrptr(struct amdgpu_device *adev) +{ + return RREG32_SOC15(GC, 0, mmRLC_SPM_RING_WRPTR); +} + +static void gfx_v10_0_spm_set_wrptr(struct amdgpu_device *adev, u32 wptr) +{ + struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; + + gfx_v10_0_write_data_to_reg(kiq_ring, 0, false, SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_RING_WRPTR), wptr); +} + +static void gfx_v10_0_set_spm_porfmon_ring_buf(struct amdgpu_device *adev, u64 gpu_addr, u32 size) +{ + struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; + + gfx_v10_0_write_data_to_reg(kiq_ring, 0, false, SOC15_REG_OFFSET(GC, 0, + mmRLC_SPM_PERFMON_RING_BASE_LO), lower_32_bits(gpu_addr)); + + gfx_v10_0_write_data_to_reg(kiq_ring, 0, false, + SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_PERFMON_RING_BASE_HI), upper_32_bits (gpu_addr)); + + gfx_v10_0_write_data_to_reg(kiq_ring, 0, false, + SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_PERFMON_RING_SIZE), size); +} + +static void gfx_v10_0_spm_set_reg(struct amdgpu_device *adev, u64 reg, u32 value) +{ + struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; + uint32_t grbm_cntl; + grbm_cntl = adev->reg_offset[GC_HWIP][0][1] + reg; + + gfx_v10_0_write_data_to_reg(kiq_ring, 0, false, grbm_cntl, value); +} + +static const struct spm_funcs gfx_v10_0_spm_funcs = { + .start = gfx_v10_0_spm_start, + .stop = gfx_v10_0_spm_stop, + .get_rdptr= gfx_v10_0_spm_get_rdptr, + .set_rdptr= gfx_v10_0_spm_set_rdptr, + .get_wrptr= gfx_v10_0_spm_get_wrptr, + .set_wrptr= gfx_v10_0_spm_set_wrptr, + .set_spm_porfmon_ring_buf = gfx_v10_0_set_spm_porfmon_ring_buf, + .set_reg = gfx_v10_0_spm_set_reg, + .set_spm_config_size = 3, + .get_spm_data_size = 128, +}; + +static void gfx_v10_0_set_spm_funcs(struct amdgpu_device *adev) +{ + adev->gfx.spm.spmf = &gfx_v10_0_spm_funcs; +} + + static int gfx_v10_0_early_init(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; @@ -6776,6 +6904,7 @@ static int gfx_v10_0_early_init(void *handle) adev->gfx.num_compute_rings = AMDGPU_MAX_COMPUTE_RINGS; + gfx_v10_0_set_spm_funcs(adev); gfx_v10_0_set_kiq_pm4_funcs(adev); gfx_v10_0_set_ring_funcs(adev); gfx_v10_0_set_irq_funcs(adev); @@ -6794,6 +6923,10 @@ static int gfx_v10_0_late_init(void *handle) if (r) return r; + r = amdgpu_irq_get(adev, &adev->gfx.spm_irq, 0); + if (r) + return r; + r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); if (r) return r; @@ -6860,6 +6993,7 @@ static void gfx_v10_0_update_medium_grain_clock_gating(struct amdgpu_device *ade if (def != data) WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); + /* MGLS is a global flag to control all MGLS in GFX */ if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { /* 2 - RLC memory Light sleep */ @@ -8018,6 +8152,39 @@ static void gfx_v10_0_emit_mem_sync(struct amdgpu_ring *ring) amdgpu_ring_write(ring, gcr_cntl); /* GCR_CNTL */ } +static int gfx_v10_0_spm_set_interrupt_state(struct amdgpu_device *adev, + struct amdgpu_irq_src *src, + unsigned int type, + enum amdgpu_interrupt_state state) +{ + switch (state) { + case AMDGPU_IRQ_STATE_DISABLE: + WREG32_SOC15(GC, 0, mmRLC_SPM_INT_CNTL, 0); + break; + case AMDGPU_IRQ_STATE_ENABLE: + WREG32_SOC15(GC, 0, mmRLC_SPM_INT_CNTL, 1); + break; + default: + break; + } + return 0; +} + +static int gfx_v10_0_spm_irq(struct amdgpu_device *adev, + struct amdgpu_irq_src *source, + struct amdgpu_iv_entry *entry) +{ + u8 me_id, pipe_id, queue_id; + + me_id = (entry->ring_id & 0x0c) >> 2; + pipe_id = (entry->ring_id & 0x03) >> 0; + queue_id = (entry->ring_id & 0x70) >> 4; + pr_debug ("IH: RLC_RPM_INT, me:%d, pipe:%d, queue:%d\n", + me_id, pipe_id, queue_id); + return 0; /* This prevents sending it to KFD */ +} + + static const struct amd_ip_funcs gfx_v10_0_ip_funcs = { .name = "gfx_v10_0", .early_init = gfx_v10_0_early_init, @@ -8189,6 +8356,11 @@ static const struct amdgpu_irq_src_funcs gfx_v10_0_kiq_irq_funcs = { .process = gfx_v10_0_kiq_irq, }; +static const struct amdgpu_irq_src_funcs gfx_v10_0_spm_irq_funcs = { + .set = gfx_v10_0_spm_set_interrupt_state, + .process = gfx_v10_0_spm_irq, +}; + static void gfx_v10_0_set_irq_funcs(struct amdgpu_device *adev) { adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; @@ -8197,6 +8369,9 @@ static void gfx_v10_0_set_irq_funcs(struct amdgpu_device *adev) adev->gfx.kiq.irq.num_types = AMDGPU_CP_KIQ_IRQ_LAST; adev->gfx.kiq.irq.funcs = &gfx_v10_0_kiq_irq_funcs; + adev->gfx.spm_irq.num_types = 1; + adev->gfx.spm_irq.funcs = &gfx_v10_0_spm_irq_funcs; + adev->gfx.priv_reg_irq.num_types = 1; adev->gfx.priv_reg_irq.funcs = &gfx_v10_0_priv_reg_irq_funcs; diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c index 14790f8..1125b91 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c @@ -1955,6 +1955,12 @@ static int gfx_v8_0_sw_init(void *handle) adev->gfx.mec.num_pipe_per_mec = 4; adev->gfx.mec.num_queue_per_pipe = 8; + /* KIQ SPM */ + r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_RLC_STRM_PERF_MONITOR, + &adev->gfx.spm_irq); + if (r) + return r; + /* EOP Event */ r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_END_OF_PIPE, &adev->gfx.eop_irq); if (r) @@ -4927,6 +4933,7 @@ static int gfx_v8_0_hw_fini(void *handle) amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0); amdgpu_irq_put(adev, &adev->gfx.sq_irq, 0); + amdgpu_irq_put(adev, &adev->gfx.spm_irq, 0); /* disable KCQ to avoid CPC touch memory not valid anymore */ gfx_v8_0_kcq_disable(adev); @@ -5291,6 +5298,126 @@ static const struct amdgpu_gfx_funcs gfx_v8_0_gfx_funcs = { .select_me_pipe_q = &gfx_v8_0_select_me_pipe_q }; +static void gfx_v8_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel, + bool wc, uint32_t reg, uint32_t val) +{ + amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); + amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) | + WRITE_DATA_DST_SEL(0) | + (wc ? WR_CONFIRM : 0)); + amdgpu_ring_write(ring, reg); + amdgpu_ring_write(ring, 0); + amdgpu_ring_write(ring, val); +} + +static void gfx_v8_0_spm_start(struct amdgpu_device *adev) +{ + struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; + uint32_t data = 0; + u8 se; + + for (se = 0; se < adev->gfx.config.max_shader_engines + 1; ++se) + { + uint32_t mux_addr_reg = mmRLC_SPM_GLOBAL_MUXSEL_ADDR; + + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, + INSTANCE_BROADCAST_WRITES, 1); + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, + SH_BROADCAST_WRITES, 1); + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, + SE_BROADCAST_WRITES, 1); + + if (se < adev->gfx.config.max_shader_engines) // SE else GB + { + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se); + + mux_addr_reg = mmRLC_SPM_SE_MUXSEL_ADDR; + } + gfx_v8_0_write_data_to_reg(kiq_ring, 0, false, mmGRBM_GFX_INDEX, data); + // init addr + gfx_v8_0_write_data_to_reg(kiq_ring, 0, false, mux_addr_reg, data); + } + + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, 0); + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1); + gfx_v8_0_write_data_to_reg(kiq_ring, 0, false, mmGRBM_GFX_INDEX, data); + + data = 0; + data |= CP_PERFMON_STATE_DISABLE_AND_RESET <<4 ; + gfx_v8_0_write_data_to_reg(kiq_ring, 0, false, mmCP_PERFMON_CNTL, data); + + data = 0; + data |= STRM_PERFMON_STATE_START_COUNTING <<4 ; + gfx_v8_0_write_data_to_reg(kiq_ring, 0, false, mmCP_PERFMON_CNTL, data);</div> <div class="PlainText">[yz] unnecessary spaces here + + gfx_v8_0_write_data_to_reg(kiq_ring, 0, false, mmRLC_SPM_INT_CNTL ,1); +} + +static void gfx_v8_0_spm_stop(struct amdgpu_device *adev) +{ + struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; + uint32_t data = 0; + + data = CP_PERFMON_STATE_STOP_COUNTING; + gfx_v8_0_write_data_to_reg(kiq_ring, 0, false, mmCP_PERFMON_CNTL, data); + data |= (1<<10); + gfx_v8_0_write_data_to_reg(kiq_ring, 0, false, mmCP_PERFMON_CNTL, data); + + gfx_v8_0_write_data_to_reg(kiq_ring, 0, false, mmRLC_SPM_INT_CNTL, 0); +} + + +static u32 gfx_v8_0_spm_get_rdptr(struct amdgpu_device *adev) +{ + return RREG32 (mmRLC_SPM_RING_RDPTR); +} + +static void gfx_v8_0_spm_set_rdptr(struct amdgpu_device *adev, u32 rptr) +{ + struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; + + gfx_v8_0_write_data_to_reg(kiq_ring, 0, false, mmRLC_SPM_RING_RDPTR, rptr); +} + +static u32 gfx_v8_0_spm_get_wrptr(struct amdgpu_device *adev) +{ + return -1; +} + +static void gfx_v8_0_set_spm_porfmon_ring_buf(struct amdgpu_device *adev, u64 gpu_addr, u32 size) +{ + struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; + + gfx_v8_0_write_data_to_reg(kiq_ring, 0, false, mmRLC_SPM_PERFMON_RING_BASE_LO, lower_32_bits(gpu_addr)); + + gfx_v8_0_write_data_to_reg(kiq_ring, 0, false, mmRLC_SPM_PERFMON_RING_BASE_HI, upper_32_bits (gpu_addr)); + + gfx_v8_0_write_data_to_reg(kiq_ring, 0, false, mmRLC_SPM_PERFMON_RING_SIZE, size); +} + +static void gfx_v8_0_spm_set_reg(struct amdgpu_device *adev, u64 reg, u32 value) +{ + struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; + gfx_v8_0_write_data_to_reg(kiq_ring, 0, false, reg, value); +} + +static const struct spm_funcs gfx_v8_0_spm_funcs = { + .start = gfx_v8_0_spm_start, + .stop = gfx_v8_0_spm_stop, + .get_rdptr= gfx_v8_0_spm_get_rdptr, + .set_rdptr= gfx_v8_0_spm_set_rdptr, + .get_wrptr= gfx_v8_0_spm_get_wrptr, + .set_spm_porfmon_ring_buf = gfx_v8_0_set_spm_porfmon_ring_buf, + .set_reg = gfx_v8_0_spm_set_reg, + .set_spm_config_size = 3, + .get_spm_data_size = 128, +}; + +static void gfx_v8_0_set_spm_funcs(struct amdgpu_device *adev) +{ + adev->gfx.spm.spmf = &gfx_v8_0_spm_funcs; +} + static int gfx_v8_0_early_init(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; @@ -5298,6 +5425,8 @@ static int gfx_v8_0_early_init(void *handle) adev->gfx.num_gfx_rings = GFX8_NUM_GFX_RINGS; adev->gfx.num_compute_rings = AMDGPU_MAX_COMPUTE_RINGS; adev->gfx.funcs = &gfx_v8_0_gfx_funcs; + + gfx_v8_0_set_spm_funcs(adev); gfx_v8_0_set_ring_funcs(adev); gfx_v8_0_set_irq_funcs(adev); gfx_v8_0_set_gds_init(adev); @@ -5338,6 +5467,10 @@ static int gfx_v8_0_late_init(void *handle) return r; } + r = amdgpu_irq_get(adev, &adev->gfx.spm_irq, 0); + if (r) + return r; + return 0; } @@ -6845,6 +6978,41 @@ static void gfx_v8_0_emit_mem_sync_compute(struct amdgpu_ring *ring) amdgpu_ring_write(ring, 0x0000000A); /* poll interval */ } +static int gfx_v8_0_spm_set_interrupt_state(struct amdgpu_device *adev, + struct amdgpu_irq_src *src, + unsigned int type, + enum amdgpu_interrupt_state state) +{ + switch (state) { + case AMDGPU_IRQ_STATE_DISABLE: + WREG32(mmRLC_SPM_INT_CNTL, 0); + break; + case AMDGPU_IRQ_STATE_ENABLE: + WREG32(mmRLC_SPM_INT_CNTL, 1); + break; + default: + break; + } + return 0; +} + +static int gfx_v8_0_spm_irq(struct amdgpu_device *adev, + struct amdgpu_irq_src *source, + struct amdgpu_iv_entry *entry) +{ + u8 me_id, pipe_id, queue_id; + struct amdgpu_ring *ring = &(adev->gfx.kiq.ring); + + me_id = (entry->ring_id & 0x0c) >> 2; + pipe_id = (entry->ring_id & 0x03) >> 0; + queue_id = (entry->ring_id & 0x70) >> 4; + pr_debug("IH: RLC_RPM_INT, me:%d, pipe:%d, queue:%d\n", + me_id, pipe_id, queue_id); + + amdgpu_fence_process(ring); + return 0; +} + static const struct amd_ip_funcs gfx_v8_0_ip_funcs = { .name = "gfx_v8_0", .early_init = gfx_v8_0_early_init, @@ -7005,11 +7173,19 @@ static const struct amdgpu_irq_src_funcs gfx_v8_0_sq_irq_funcs = { .process = gfx_v8_0_sq_irq, }; +static const struct amdgpu_irq_src_funcs gfx_v8_0_spm_irq_funcs = { + .set = gfx_v8_0_spm_set_interrupt_state, + .process = gfx_v8_0_spm_irq, +}; + static void gfx_v8_0_set_irq_funcs(struct amdgpu_device *adev) { adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; adev->gfx.eop_irq.funcs = &gfx_v8_0_eop_irq_funcs; + adev->gfx.spm_irq.num_types = 1; + adev->gfx.spm_irq.funcs = &gfx_v8_0_spm_irq_funcs; + adev->gfx.priv_reg_irq.num_types = 1; adev->gfx.priv_reg_irq.funcs = &gfx_v8_0_priv_reg_irq_funcs; diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c index 2767c6d..bfde274 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c @@ -2249,6 +2249,13 @@ static int gfx_v9_0_sw_init(void *handle) adev->gfx.mec.num_pipe_per_mec = 4; adev->gfx.mec.num_queue_per_pipe = 8; + /* KIQ SPM */ + r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_RLC, + GFX_9_0__SRCID__RLC_STRM_PERF_MONITOR_INTERRUPT, + &adev->gfx.spm_irq); + if (r) + return r; + /* EOP Event */ r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq); if (r) @@ -3907,6 +3914,7 @@ static int gfx_v9_0_hw_fini(void *handle) amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0); amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); + amdgpu_irq_put(adev, &adev->gfx.spm_irq, 0); amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); /* DF freeze and kcq disable will fail */ @@ -4617,6 +4625,121 @@ static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev) return r; } +static void gfx_v9_0_spm_start(struct amdgpu_device *adev) +{ + struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; + uint32_t data = 0; + u8 se; + + for (se = 0; se < adev->gfx.config.max_shader_engines + 1; ++se) + { + uint32_t mux_addr_reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_GLOBAL_MUXSEL_ADDR); + + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, + INSTANCE_BROADCAST_WRITES, 1); + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, + SH_BROADCAST_WRITES, 1); + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, + SE_BROADCAST_WRITES, 1); + + if (se < adev->gfx.config.max_shader_engines) // SE else GB + { + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se); + + mux_addr_reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_SE_MUXSEL_ADDR); + } + gfx_v9_0_write_data_to_reg(kiq_ring, 0, false, SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX), data); + // init addr + gfx_v9_0_write_data_to_reg(kiq_ring, 0, false, mux_addr_reg, data); + } + + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, 0); + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1); + gfx_v9_0_write_data_to_reg(kiq_ring, 0, false, SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX), data); + + data = 0; + data |= CP_PERFMON_STATE_DISABLE_AND_RESET <<4 ; + gfx_v9_0_write_data_to_reg(kiq_ring, 0, false, SOC15_REG_OFFSET(GC, 0, mmCP_PERFMON_CNTL), data); + + data = 0; + data |= STRM_PERFMON_STATE_START_COUNTING <<4 ; + gfx_v9_0_write_data_to_reg(kiq_ring, 0, false, SOC15_REG_OFFSET(GC, 0, mmCP_PERFMON_CNTL), data); + + gfx_v9_0_write_data_to_reg(kiq_ring, 0, false, SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_INT_CNTL) ,1); +} + + +static void gfx_v9_0_spm_stop(struct amdgpu_device *adev) +{ + struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; + uint32_t data = 0; + + + data = CP_PERFMON_STATE_STOP_COUNTING; + gfx_v9_0_write_data_to_reg(kiq_ring, 0, false, SOC15_REG_OFFSET(GC, 0, mmCP_PERFMON_CNTL), data); + data |= (1<<10); + gfx_v9_0_write_data_to_reg(kiq_ring, 0, false, SOC15_REG_OFFSET(GC, 0, mmCP_PERFMON_CNTL), data); + + gfx_v9_0_write_data_to_reg(kiq_ring, 0, false, SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_INT_CNTL), 0); +} + +static u32 gfx_v9_0_spm_get_rdptr(struct amdgpu_device *adev) +{ + return RREG32_SOC15(GC, 0, mmRLC_SPM_RING_RDPTR);; +} + +static void gfx_v9_0_spm_set_rdptr(struct amdgpu_device *adev, u32 rptr) +{ + struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; + + gfx_v9_0_write_data_to_reg(kiq_ring, 0, false, SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_RING_RDPTR), rptr); +} + +static u32 gfx_v9_0_spm_get_wrptr(struct amdgpu_device *adev) +{ + return -1; +} + +static void gfx_v9_0_set_spm_porfmon_ring_buf(struct amdgpu_device *adev, u64 gpu_addr, u32 size)</div> <div class="PlainText">[yz] <span style="font-family: "Segoe UI", "Segoe UI Web (West European)", "Segoe UI", -apple-system, BlinkMacSystemFont, Roboto, "Helvetica Neue", sans-serif; background-color: rgb(255, 255, 255); display: inline !important">porfmon -> perfmon +{ + struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; + + gfx_v9_0_write_data_to_reg(kiq_ring, 0, false, SOC15_REG_OFFSET(GC, 0, + mmRLC_SPM_PERFMON_RING_BASE_LO), lower_32_bits(gpu_addr)); + + gfx_v9_0_write_data_to_reg(kiq_ring, 0, false, + SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_PERFMON_RING_BASE_HI), upper_32_bits (gpu_addr)); + + gfx_v9_0_write_data_to_reg(kiq_ring, 0, false, + SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_PERFMON_RING_SIZE), size); +} + +static void gfx_v9_0_spm_set_reg(struct amdgpu_device *adev, u64 reg, u32 value) +{ + struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; + uint32_t grbm_cntl; + grbm_cntl = adev->reg_offset[GC_HWIP][0][1] + reg; + + gfx_v9_0_write_data_to_reg(kiq_ring, 0, false, grbm_cntl, value); +} + +static const struct spm_funcs gfx_v9_0_spm_funcs = { + .start = gfx_v9_0_spm_start, + .stop = gfx_v9_0_spm_stop, + .get_rdptr= gfx_v9_0_spm_get_rdptr, + .set_rdptr= gfx_v9_0_spm_set_rdptr, + .get_wrptr= gfx_v9_0_spm_get_wrptr, + .set_spm_porfmon_ring_buf = gfx_v9_0_set_spm_porfmon_ring_buf, + .set_reg = gfx_v9_0_spm_set_reg, + .set_spm_config_size = 3, + .get_spm_data_size = 128, +}; + +static void gfx_v9_0_set_spm_funcs(struct amdgpu_device *adev) +{ + adev->gfx.spm.spmf = &gfx_v9_0_spm_funcs; +} + static int gfx_v9_0_early_init(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; @@ -4626,6 +4749,7 @@ static int gfx_v9_0_early_init(void *handle) else adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS; adev->gfx.num_compute_rings = AMDGPU_MAX_COMPUTE_RINGS; + gfx_v9_0_set_spm_funcs(adev); gfx_v9_0_set_kiq_pm4_funcs(adev); gfx_v9_0_set_ring_funcs(adev); gfx_v9_0_set_irq_funcs(adev); @@ -4677,6 +4801,10 @@ static int gfx_v9_0_late_init(void *handle) if (r) return r; + r = amdgpu_irq_get(adev, &adev->gfx.spm_irq, 0); + if (r) + return r; + r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); if (r) return r; @@ -6657,6 +6785,39 @@ static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring) amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */ } +static int gfx_v9_0_spm_set_interrupt_state(struct amdgpu_device *adev, + struct amdgpu_irq_src *src, + unsigned int type, + enum amdgpu_interrupt_state state) +{ + switch (state) { + case AMDGPU_IRQ_STATE_DISABLE: + WREG32(mmRLC_SPM_INT_CNTL, 0); + break; + case AMDGPU_IRQ_STATE_ENABLE: + WREG32(mmRLC_SPM_INT_CNTL, 1); + break; + default: + break; + } + return 0; +} + +static int gfx_v9_0_spm_irq(struct amdgpu_device *adev, + struct amdgpu_irq_src *source, + struct amdgpu_iv_entry *entry) +{ + u8 me_id, pipe_id, queue_id; + + me_id = (entry->ring_id & 0x0c) >> 2; + pipe_id = (entry->ring_id & 0x03) >> 0; + queue_id = (entry->ring_id & 0x70) >> 4; + pr_debug("IH: RLC_RPM_INT, me:%d, pipe:%d, queue:%d\n", + me_id, pipe_id, queue_id); + + return 0; /* This also prevents sending it to KFD */ +} + static const struct amd_ip_funcs gfx_v9_0_ip_funcs = { .name = "gfx_v9_0", .early_init = gfx_v9_0_early_init, @@ -6825,12 +6986,19 @@ static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = { .process = amdgpu_gfx_cp_ecc_error_irq, }; +static const struct amdgpu_irq_src_funcs gfx_v9_0_spm_irq_funcs = { + .set = gfx_v9_0_spm_set_interrupt_state, + .process = gfx_v9_0_spm_irq, +}; static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev) { adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs; + adev->gfx.spm_irq.num_types = 1; + adev->gfx.spm_irq.funcs = &gfx_v9_0_spm_irq_funcs; + adev->gfx.priv_reg_irq.num_types = 1; adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs; diff --git a/drivers/gpu/drm/amd/amdkfd/Makefile b/drivers/gpu/drm/amd/amdkfd/Makefile index 6147462..43edba0 100644 --- a/drivers/gpu/drm/amd/amdkfd/Makefile +++ b/drivers/gpu/drm/amd/amdkfd/Makefile @@ -53,7 +53,8 @@ AMDKFD_FILES := $(AMDKFD_PATH)/kfd_module.o \ $(AMDKFD_PATH)/kfd_int_process_v9.o \ $(AMDKFD_PATH)/kfd_dbgdev.o \ $(AMDKFD_PATH)/kfd_dbgmgr.o \ - $(AMDKFD_PATH)/kfd_crat.o + $(AMDKFD_PATH)/kfd_crat.o \ + $(AMDKFD_PATH)/kfd_spm.o ifneq ($(CONFIG_AMD_IOMMU_V2),) AMDKFD_FILES += $(AMDKFD_PATH)/kfd_iommu.o diff --git a/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c b/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c index 9f59ba9..cd394cd 100644 --- a/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c +++ b/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c @@ -24,6 +24,7 @@ #include "kfd_events.h" #include "cik_int.h" #include "amdgpu_amdkfd.h" +#include "ivsrcid/ivsrcid_vislands30.h" static bool cik_event_interrupt_isr(struct kfd_dev *dev, const uint32_t *ih_ring_entry, @@ -37,6 +38,11 @@ static bool cik_event_interrupt_isr(struct kfd_dev *dev, uint16_t pasid; bool ret; + vmid = f2g->read_vmid_from_vmfault_reg(dev->kgd); + + if ((ihre->source_id == VISLANDS30_IV_SRCID_RLC_STRM_PERF_MONITOR) && (vmid == 0)) + return (kfd_spm_interrupt_isr (dev, ihre->source_id, ihre->source_id)); + /* This workaround is due to HW/FW limitation on Hawaii that * VMID and PASID are not written into ih_ring_entry */ @@ -49,7 +55,6 @@ static bool cik_event_interrupt_isr(struct kfd_dev *dev, *patched_flag = true; *tmp_ihre = *ihre; - vmid = f2g->read_vmid_from_vmfault_reg(dev->kgd); ret = f2g->get_atc_vmid_pasid_mapping_info(dev->kgd, vmid, &pasid); tmp_ihre->ring_id &= 0x000000ff; @@ -95,6 +100,9 @@ static void cik_event_interrupt_wq(struct kfd_dev *dev, if (pasid == 0) return; + if ((ihre->source_id == VISLANDS30_IV_SRCID_RLC_STRM_PERF_MONITOR) && (vmid == 0)) + kfd_spm_interrupt_wq(dev, ihre->source_id); + if (ihre->source_id == CIK_INTSRC_CP_END_OF_PIPE) kfd_signal_event_interrupt(pasid, context_id, 28); else if (ihre->source_id == CIK_INTSRC_SDMA_TRAP) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index f8fa03a..bfc83beb 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -1732,6 +1732,20 @@ static int kfd_ioctl_import_dmabuf(struct file *filep, return r; } +static int kfd_ioctl_rlc_spm(struct file *filep, + struct kfd_process *p, void *data) +{ + struct kfd_ioctl_spm_args *args = data; + int err; + + err = kfd_rlc_spm(p, + (void __user *)args, + args->buf_size, + args->op); + + return err; +} + #define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \ [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, \ .cmd_drv = 0, .name = #ioctl} @@ -1827,6 +1841,9 @@ static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = { AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_QUEUE_GWS, kfd_ioctl_alloc_queue_gws, 0), + + AMDKFD_IOCTL_DEF(AMDKFD_IOC_RLC_SPM, + kfd_ioctl_rlc_spm, 0), }; #define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c index e05d75e..481f0ae 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c @@ -24,6 +24,7 @@ #include "kfd_events.h" #include "soc15_int.h" #include "kfd_device_queue_manager.h" +#include "ivsrcid/gfx/irqsrcs_gfx_9_0.h" static bool event_interrupt_isr_v9(struct kfd_dev *dev, const uint32_t *ih_ring_entry, @@ -35,12 +36,15 @@ static bool event_interrupt_isr_v9(struct kfd_dev *dev, /* Only handle interrupts from KFD VMIDs */ vmid = SOC15_VMID_FROM_IH_ENTRY(ih_ring_entry); + source_id = SOC15_SOURCE_ID_FROM_IH_ENTRY(ih_ring_entry); + client_id = SOC15_CLIENT_ID_FROM_IH_ENTRY(ih_ring_entry); + if ((source_id == GFX_9_0__SRCID__RLC_STRM_PERF_MONITOR_INTERRUPT) && (vmid == 0)) + return (kfd_spm_interrupt_isr (dev, source_id, client_id)); + if (vmid < dev->vm_info.first_vmid_kfd || vmid > dev->vm_info.last_vmid_kfd) return 0; - source_id = SOC15_SOURCE_ID_FROM_IH_ENTRY(ih_ring_entry); - client_id = SOC15_CLIENT_ID_FROM_IH_ENTRY(ih_ring_entry); pasid = SOC15_PASID_FROM_IH_ENTRY(ih_ring_entry); /* This is a known issue for gfx9. Under non HWS, pasid is not set @@ -95,6 +99,9 @@ static void event_interrupt_wq_v9(struct kfd_dev *dev, vmid = SOC15_VMID_FROM_IH_ENTRY(ih_ring_entry); context_id = SOC15_CONTEXT_ID0_FROM_IH_ENTRY(ih_ring_entry); + if ((source_id == GFX_9_0__SRCID__RLC_STRM_PERF_MONITOR_INTERRUPT) && (vmid == 0)) + kfd_spm_interrupt_wq(dev, source_id); + if (source_id == SOC15_INTSRC_CP_END_OF_PIPE) kfd_signal_event_interrupt(pasid, context_id, 32); else if (source_id == SOC15_INTSRC_SDMA_TRAP) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index 43b888b..707d672 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -309,6 +309,9 @@ struct kfd_dev { /* Global GWS resource shared b/t processes*/ void *gws; + + /*spm process id */ + unsigned int spm_pasid; }; enum kfd_mempool { @@ -740,6 +743,13 @@ struct kfd_process { struct kobject *kobj; struct kobject *kobj_queues; struct attribute attr_pasid; + /* spm data */ + struct kfd_spm_cntr *spm_cntr; + bool is_spm_acquired; + /* Work items for tranfer data to user */ + struct delayed_work copy_to_user_work; + /* spm-related data */ + struct mutex spm_mutex; }; #define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */ @@ -1045,10 +1055,20 @@ int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p); bool kfd_is_locked(void); +void kfd_spm_init_process(struct kfd_process *p); +int kfd_rlc_spm(struct kfd_process *p, void __user *data, + uint32_t buf_size, __u32 op); + /* Compute profile */ void kfd_inc_compute_active(struct kfd_dev *dev); void kfd_dec_compute_active(struct kfd_dev *dev); +/* spm interrupt */ +bool kfd_spm_interrupt_isr(struct kfd_dev *dev, + uint16_t source_id, uint32_t client_id); +void kfd_spm_interrupt_wq(struct kfd_dev *dev, uint16_t source_id); + + /* Cgroup Support */ /* Check with device cgroup if @kfd device is accessible */ static inline int kfd_devcgroup_check_permission(struct kfd_dev *kfd) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c index fe0cd49..338868d 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c @@ -746,6 +746,7 @@ static struct kfd_process *create_process(const struct task_struct *thread) INIT_DELAYED_WORK(&process->restore_work, restore_process_worker); process->last_restore_timestamp = get_jiffies_64(); kfd_event_init_process(process); + kfd_spm_init_process(process); process->is_32bit_user_mode = in_compat_syscall(); process->pasid = kfd_pasid_alloc(); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_spm.c b/drivers/gpu/drm/amd/amdkfd/kfd_spm.c new file mode 100644 index 0000000..773e2ee --- /dev/null +++ b/drivers/gpu/drm/amd/amdkfd/kfd_spm.c @@ -0,0 +1,577 @@ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include <linux/device.h> +#include "kfd_priv.h" +#include "amdgpu_amdkfd.h" +#include "soc15_int.h" +#include "ivsrcid/gfx/irqsrcs_gfx_9_0.h" +#include "ivsrcid/ivsrcid_vislands30.h" +#include <linux/delay.h> + +#define SAMPLING_MAX 4 + +struct user_buf { + uint64_t __user *user_addr; + u32 ubufsize; + +}; + +struct kfd_spm_cntr { + struct kgd_dev *kgd; + u64 spm_gpu_addr; + u32 spm_ring_buf_size; + u32 spm_ring_buf_mark; + u32 spm_ring_rptr; + u32 spm_ring_wptr; + u32 spm_ring_size_copied; + void *spm_obj; + u32 *spm_cpu_addr; + struct user_buf ubuf [SAMPLING_MAX]; + u32 buf_count; + bool is_uesr_buf_filled; + bool is_nowait; + struct task_struct *thread; + bool thread_ready; + char *name; + wait_queue_head_t spm_wq; + int wanted_cluster; + bool is_timeout; +}; + +static int kfd_spm_data_cocy (struct kfd_spm_cntr * spm_cntr, u32 size_to_copy) +{ + u32 user_buf_space_left; + int ret = 0; + u32 bufSize; + uint64_t __user * user_address; + uint64_t * ring_buf; + + pr_debug("[%s]\n", __func__); + + bufSize = spm_cntr->ubuf[0].ubufsize; + user_address = (uint64_t*)((uint64_t)spm_cntr->ubuf[0].user_addr + spm_cntr->spm_ring_size_copied); + ring_buf = (uint64_t*)((uint64_t)spm_cntr->spm_cpu_addr + spm_cntr->spm_ring_rptr); + + if (user_address == NULL) + return -EFAULT; + + user_buf_space_left = bufSize - spm_cntr->spm_ring_size_copied; + + if (size_to_copy <= user_buf_space_left) { + ret = copy_to_user(user_address, ring_buf, size_to_copy); + if (ret) { + pr_err("copy_to_user failed, ret = %x\n", ret); + return -EFAULT; + } + spm_cntr->spm_ring_size_copied += size_to_copy; + } else { + size_to_copy = spm_cntr->spm_ring_buf_size - spm_cntr->spm_ring_size_copied; + ret = copy_to_user(user_address, ring_buf, user_buf_space_left); + if (ret) + return -EFAULT; + + spm_cntr->spm_ring_size_copied = bufSize; + spm_cntr->is_uesr_buf_filled = true; + } + + return ret; +} + +static int kfd_spm_reag_ring_buf_polling (struct kfd_spm_cntr * spm_cntr, long timeout) +{ + u32 size_to_copy; + int ret = 0; + + pr_info("[%s]\n", __func__); + + while (spm_cntr->is_uesr_buf_filled != true){ + spm_cntr->spm_ring_rptr = amdgpu_amdkfd_rlc_spm_get_rdptr(spm_cntr->kgd) & spm_cntr->spm_ring_buf_mark; +#if 1 + spm_cntr->spm_ring_wptr = amdgpu_amdkfd_rlc_spm_get_wrptr(spm_cntr->kgd) & spm_cntr->spm_ring_buf_mark; +#else + spm_cntr->spm_ring_wptr = spm_cntr->spm_cpu_addr[0] & spm_cntr->spm_ring_buf_mark; +#endif + + if ((spm_cntr->spm_ring_rptr >= 0) && (spm_cntr->spm_ring_rptr < 0x20)) + spm_cntr->spm_ring_rptr = 0x20; + + if (spm_cntr->is_uesr_buf_filled == true) + goto exit; + + if (spm_cntr->spm_ring_wptr > spm_cntr->spm_ring_rptr) { + size_to_copy = spm_cntr->spm_ring_wptr - spm_cntr->spm_ring_rptr; + ret = kfd_spm_data_cocy(spm_cntr, size_to_copy); + } else if (spm_cntr->spm_ring_wptr < spm_cntr->spm_ring_rptr) { + size_to_copy = spm_cntr->spm_ring_buf_size - spm_cntr->spm_ring_rptr; + ret = kfd_spm_data_cocy(spm_cntr, size_to_copy); + + /* correct counter start point */ + spm_cntr->spm_ring_rptr = 0x20; + size_to_copy = spm_cntr->spm_ring_wptr; + ret = kfd_spm_data_cocy(spm_cntr, size_to_copy); + } + + if (!ret) { + if (spm_cntr->is_uesr_buf_filled == true) { + /* stop */ + amdgpu_amdkfd_rlc_spm_cntl(spm_cntr->kgd, 0); + amdgpu_amdkfd_rlc_spm_set_rdptr(spm_cntr->kgd, 0); +#if 0 + amdgpu_amdkfd_rlc_spm_set_wrptr(spm_cntr->kgd, 0); +#else + spm_cntr->spm_cpu_addr[0]= 0; +#endif + return ret; + } else + amdgpu_amdkfd_rlc_spm_set_rdptr(spm_cntr->kgd, spm_cntr->spm_ring_wptr); + + if (spm_cntr->is_timeout == true) { + /* stop */ + amdgpu_amdkfd_rlc_spm_cntl(spm_cntr->kgd, 0); + amdgpu_amdkfd_rlc_spm_set_rdptr(spm_cntr->kgd, 0); +#if 0 + amdgpu_amdkfd_rlc_spm_set_wrptr(spm_cntr->kgd, 0); +#else + spm_cntr->spm_cpu_addr[0]= 0; +#endif + break; + } + } + } +exit: + return ret; +} + +static int kfd_spm_reag_ring_buf (struct kfd_spm_cntr * spm_cntr) +{ + u32 size_to_copy; + int ret = 0; + + pr_debug("[%s]\n", __func__); + + spm_cntr->spm_ring_rptr = amdgpu_amdkfd_rlc_spm_get_rdptr(spm_cntr->kgd) & spm_cntr->spm_ring_buf_mark; +#if 1 + spm_cntr->spm_ring_wptr = amdgpu_amdkfd_rlc_spm_get_wrptr(spm_cntr->kgd) & spm_cntr->spm_ring_buf_mark; +#else + spm_cntr->spm_ring_wptr = spm_cntr->spm_cpu_addr[0] & spm_cntr->spm_ring_buf_mark; +#endif + if ((spm_cntr->spm_ring_rptr >= 0) && (spm_cntr->spm_ring_rptr < 0x20)) { + spm_cntr->spm_ring_rptr = 0x20; + } + + if (spm_cntr->is_uesr_buf_filled == true) + goto exit; + + if (spm_cntr->spm_ring_wptr > spm_cntr->spm_ring_rptr) { + size_to_copy = spm_cntr->spm_ring_wptr - spm_cntr->spm_ring_rptr; + ret = kfd_spm_data_cocy(spm_cntr, size_to_copy); + + } else if (spm_cntr->spm_ring_wptr < spm_cntr->spm_ring_rptr) { + size_to_copy = spm_cntr->spm_ring_buf_size - spm_cntr->spm_ring_rptr; + ret = kfd_spm_data_cocy(spm_cntr, size_to_copy); + + spm_cntr->spm_ring_rptr = 0x20; + size_to_copy = spm_cntr->spm_ring_wptr; + ret = kfd_spm_data_cocy(spm_cntr, size_to_copy); + } + if (!ret) { + if (spm_cntr->is_uesr_buf_filled == true) { + /* stop */ + amdgpu_amdkfd_rlc_spm_cntl(spm_cntr->kgd, 0); + amdgpu_amdkfd_rlc_spm_set_rdptr(spm_cntr->kgd, 0); +#if 0 + amdgpu_amdkfd_rlc_spm_set_wrptr(spm_cntr->kgd, 0); +#else + spm_cntr->spm_cpu_addr[0]= 0; +#endif + return ret; + } else + amdgpu_amdkfd_rlc_spm_set_rdptr(spm_cntr->kgd, spm_cntr->spm_ring_wptr); + } +exit: + return ret; +} + +static int kfd_spm_sched_main(void *param) +{ + struct kfd_spm_cntr * spm_cntr = (struct kfd_spm_cntr *)param; + + while (!kthread_should_stop()) { + wait_event_interruptible(spm_cntr->spm_wq, + spm_cntr->wanted_cluster != false || + kthread_should_stop()); + + kfd_spm_reag_ring_buf (spm_cntr); + + spm_cntr->wanted_cluster = false; + } + return 0; +} + +static void transfer_data_process_worker(struct work_struct *work) +{ + struct kfd_process *p; + struct delayed_work *dwork; + + pr_debug("[%s]\n", __func__); + + dwork = to_delayed_work(work); + + /* Process termination destroys this worker thread. So during the + * lifetime of this thread, kfd_process p will be valid + */ + p = container_of(dwork, struct kfd_process, copy_to_user_work); + + p->spm_cntr->is_timeout = true; +} + + +/** + * kfd_spm_init - init driver ring struct. + * Returns 0 on success, error on failure. + */ +int kfd_spm_shed_init(struct kfd_process *p, struct kgd_dev *kgd) +{ + int ret = 0; + init_waitqueue_head(&p->spm_cntr->spm_wq); + p->spm_cntr->wanted_cluster = false; + p->spm_cntr->kgd = kgd; + INIT_DELAYED_WORK(&p->copy_to_user_work, transfer_data_process_worker); + + /* Each scheduler will run on a seperate kernel thread */ + p->spm_cntr->thread = kthread_run(kfd_spm_sched_main, p->spm_cntr, p->spm_cntr->name); + if (IS_ERR(p->spm_cntr->thread)) { + ret = PTR_ERR(p->spm_cntr->thread); + p->spm_cntr->thread = NULL; + DRM_ERROR("Failed to create scheduler for %s.\n", p->spm_cntr->name); + return ret; + } + + p->spm_cntr->thread_ready = true; + return ret; +} + +/** + * amdgpu_ring_fini - tear down the driver ring struct. + * + * @adev: amdgpu_device pointer + * @ring: amdgpu_ring structure holding ring information + * + * Tear down the driver information for the selected ring (all asics). + */ +void kfd_spm_shed_fini(struct kfd_process *p) +{ + if (p->spm_cntr->thread) + kthread_stop(p->spm_cntr->thread); + + cancel_delayed_work_sync(&p->copy_to_user_work); + p->spm_cntr->thread = NULL; + p->spm_cntr->thread_ready = false; +} + +void kfd_spm_init_process(struct kfd_process *p) +{ + mutex_init(&p->spm_mutex); + p->spm_cntr = NULL; + p->is_spm_acquired = false; +} + +static struct kfd_spm_cntr *allocate_spm_cntr_data(void) +{ + struct kfd_spm_cntr *cntr; + + cntr = kzalloc(sizeof(*cntr), GFP_KERNEL); + if (!cntr) + return NULL; + + return cntr; +} + +int kfd_acquire_spm(struct kfd_process *p, struct kgd_dev *kgd) +{ + int retval; + int count; + + pr_debug("[%s]\n", __func__); + + if(p->is_spm_acquired == true) + return EINVAL; + + if (!p->spm_cntr) { + p->spm_cntr = allocate_spm_cntr_data(); + if (!p->spm_cntr) + return -ENOMEM; + /* git spm ring buffer 128KB */ + p->spm_cntr->spm_ring_buf_size = order_base_2(128 * 1024/4); + p->spm_cntr->spm_ring_buf_size = (1 << p->spm_cntr->spm_ring_buf_size) * 4; + p->spm_cntr->spm_ring_buf_mark = p->spm_cntr->spm_ring_buf_size -1; + for (count = 0; count < SAMPLING_MAX; ++count) { + p->spm_cntr->ubuf[count].user_addr = NULL; + p->spm_cntr->ubuf[count].ubufsize = 0; + } + p->spm_cntr->buf_count = 0; + p->spm_cntr->is_uesr_buf_filled = false; + p->spm_cntr->is_nowait = false; + p->spm_cntr->thread_ready = false; + } + + retval = amdgpu_amdkfd_alloc_gtt_mem(kgd, + p->spm_cntr->spm_ring_buf_size, &p->spm_cntr->spm_obj, + &p->spm_cntr->spm_gpu_addr, (void *)&p->spm_cntr->spm_cpu_addr, + false); + + if (retval) + return EINVAL; + + memset(p->spm_cntr->spm_cpu_addr, 0, p->spm_cntr->spm_ring_buf_size); + + amdgpu_amdkfd_rlc_spm_acquire(kgd, p->spm_cntr->spm_gpu_addr, p->spm_cntr->spm_ring_buf_size); + + if (p->spm_cntr->thread_ready == false) { + p->spm_cntr->name = "spm"; + retval = kfd_spm_shed_init(p, kgd); + if (retval) { + DRM_ERROR("Failed to create spm thread %s.\n", p->spm_cntr->name); + return retval; + } + } + p->is_spm_acquired = true; + + return 0; +} + +int kfd_release_spm(struct kfd_process *p, struct kgd_dev *kgd) +{ + + kfd_spm_shed_fini(p); + + amdgpu_amdkfd_free_gtt_mem(kgd, p->spm_cntr->spm_obj); + + kfree(p->spm_cntr); + p->spm_cntr = NULL; + p->is_spm_acquired = false; + + return 0; +} + +int set_dest_buf_polling(struct kfd_spm_cntr *spm, struct kgd_dev *kgd, void __user *data) +{ + struct kfd_ioctl_spm_args __user *user_spm_data = + (struct kfd_ioctl_spm_args __user *) data; + u32 i; + int ret = 0; + unsigned long timeout; + + pr_debug ("[%s]\n", __func__); + + timeout = msecs_to_jiffies(user_spm_data->timeout) + 1; + + /* if buf = NULL, stop spm */ + if (!user_spm_data->destptr) { + amdgpu_amdkfd_rlc_spm_cntl(kgd, 0); + + user_spm_data->bytes_copied = spm->spm_ring_size_copied; + spm->spm_ring_size_copied = 0; + user_spm_data->spmtptr = (uint64_t)spm->ubuf[0].user_addr; + + for (i = 0; i < spm->buf_count; ++i) { + spm->ubuf [i] = spm->ubuf [i+1]; + } + spm->ubuf[spm->buf_count].user_addr = NULL; + spm->ubuf[spm->buf_count].ubufsize = 0; + spm->buf_count --; + if (spm->buf_count < 0) + spm->buf_count = 0; + return ret; + } + + if (!spm->buf_count) { + /* First time save user spm buffer, then start spm sampling */ + spm->ubuf[0].user_addr = (uint64_t*)user_spm_data->destptr; + spm->ubuf[0].ubufsize = user_spm_data->buf_size; + user_spm_data->bytes_copied = 0; + spm->spm_ring_size_copied = 0; + spm->buf_count ++; + spm->is_uesr_buf_filled = false; + amdgpu_amdkfd_rlc_spm_cntl(kgd, 1); + + ret = kfd_spm_reag_ring_buf_polling (spm, timeout); + user_spm_data->bytes_copied = spm->spm_ring_size_copied; + user_spm_data->spmtptr = (uint64_t)spm->ubuf[0].user_addr; + spm->buf_count --; + } else { + spm->spm_ring_size_copied = 0; + spm->is_uesr_buf_filled = false; + amdgpu_amdkfd_rlc_spm_cntl(kgd, 1); + ret = kfd_spm_reag_ring_buf_polling (spm, timeout); + + user_spm_data->bytes_copied = spm->spm_ring_size_copied; + user_spm_data->spmtptr = (uint64_t)spm->ubuf[0].user_addr; + spm->buf_count --; + + for (i = 0; i < spm->buf_count; ++i) + /* Repeated dest buf */ + if (spm->ubuf[i].user_addr == (uint64_t*)user_spm_data->destptr) + break; + if (i == spm->buf_count) { + spm->ubuf[i].user_addr = (uint64_t*)user_spm_data->destptr; + spm->ubuf[i].ubufsize = user_spm_data->buf_size; + spm->buf_count ++; + } + + for (i = 0; i < spm->buf_count; ++i) + spm->ubuf[i] = spm->ubuf[i + 1]; + spm->ubuf[spm->buf_count].user_addr = NULL; + } + + user_spm_data->bytes_copied = spm->spm_ring_size_copied; + return ret; + +} + +int kfd_set_dest_buffer(struct kfd_process *p, struct kgd_dev *kgd, void __user *data) +{ + struct kfd_ioctl_spm_args __user *user_spm_data = + (struct kfd_ioctl_spm_args __user *) data; + struct kfd_spm_cntr *spm = p->spm_cntr; + unsigned long timeout; + + pr_debug("[%s]\n", __func__); + + if(p->is_spm_acquired == false) + return -EINVAL; + + timeout = msecs_to_jiffies(user_spm_data->timeout) + 1; + spm->is_timeout = false; + schedule_delayed_work(&p->copy_to_user_work, timeout); + return set_dest_buf_polling(spm, kgd, data); +} + +int kfd_config_spm(struct kfd_process *p, struct kgd_dev *kgd,struct kfd_ioctl_spm_args *data) +{ + struct kfd_ioctl_spm_args __user *user_spm_data = + (struct kfd_ioctl_spm_args __user *) data; + struct kfd_spm_set_reg *spm_reg; + + pr_debug("[%s]\n", __func__); + + spm_reg = kvmalloc_array(1, sizeof (struct kfd_spm_set_reg), GFP_KERNEL); + + if (copy_from_user(spm_reg, (void __user *)user_spm_data->destptr, sizeof (struct kfd_spm_set_reg))) { + pr_err("copy_from_user Fail\n"); + goto exit; + } + amdgpu_amdkfd_rlc_spm_set_reg(kgd, spm_reg->reg, spm_reg->value); +exit: + kfree(spm_reg); + return 0; +} + +int kfd_rlc_spm(struct kfd_process *p, void __user *data, + uint32_t buf_size, __u32 operation) +{ + struct kfd_ioctl_spm_args *args = data; + struct kfd_dev *dev; + int r; + + dev = kfd_device_by_id(args->gpu_id); + if (!dev) + return -EINVAL; + + + switch (operation) { + case KFD_IOCTL_SPM_OP_ACQUIRE: + dev->spm_pasid = p->pasid; + r = kfd_acquire_spm(p, dev->kgd); + break; + + case KFD_IOCTL_SPM_OP_RELEASE: + r = kfd_release_spm(p, dev->kgd); + break; + + case KFD_IOCTL_SPM_OP_SET_DEST_BUF: + r = kfd_set_dest_buffer(p, dev->kgd, data); + break; + + case KFD_IOCTL_SPM_OP_CONFIG: + r = kfd_config_spm(p, dev->kgd, args); + break; + + default: + r = -EINVAL; + break; + } + return r; +} + +void kfd_spm_interrupt(unsigned int pasid) +{ + + struct kfd_process *p = kfd_lookup_process_by_pasid(pasid); + + if (!p) { + pr_info("kfd_spm_interrupt p = %p \n", p); + return; /* Presumably process exited. */ + } + + mutex_lock(&p->spm_mutex); + + p->spm_cntr->wanted_cluster = true; + wake_up_interruptible(&p->spm_cntr->spm_wq); + + mutex_unlock(&p->spm_mutex); + + kfd_unref_process(p); +} + +bool kfd_spm_interrupt_isr(struct kfd_dev *dev, + uint16_t source_id, uint32_t client_id) +{ + pr_debug("[%s]\n", __func__); + + if (source_id != GFX_9_0__SRCID__RLC_STRM_PERF_MONITOR_INTERRUPT) + return 0; + + /* Interrupt types we care about: various signals and faults. + * They will be forwarded to a work queue (see below). + */ + + return source_id == GFX_9_0__SRCID__RLC_STRM_PERF_MONITOR_INTERRUPT || + source_id == VISLANDS30_IV_SRCID_RLC_STRM_PERF_MONITOR || + client_id == AMDGPU_IRQ_CLIENTID_LEGACY || + client_id == SOC15_IH_CLIENTID_RLC; +} + +void kfd_spm_interrupt_wq(struct kfd_dev *dev, uint16_t source_id) +{ + uint16_t pasid; + + pr_debug("[%s]\n", __func__); + + pasid = dev->spm_pasid; +#if 0 + if ((source_id == GFX_9_0__SRCID__RLC_STRM_PERF_MONITOR_INTERRUPT) || + (source_id == VISLANDS30_IV_SRCID_RLC_STRM_PERF_MONITOR)) + kfd_spm_interrupt(pasid); +#endif +} diff --git a/drivers/gpu/drm/amd/amdkfd/soc15_int.h b/drivers/gpu/drm/amd/amdkfd/soc15_int.h index 0bc0b25..fb4ad60 100644 --- a/drivers/gpu/drm/amd/amdkfd/soc15_int.h +++ b/drivers/gpu/drm/amd/amdkfd/soc15_int.h @@ -30,6 +30,7 @@ #define SOC15_INTSRC_SQ_INTERRUPT_MSG 239 #define SOC15_INTSRC_VMC_FAULT 0 #define SOC15_INTSRC_SDMA_TRAP 224 +#define AMDGPU_IRQ_CLIENTID_LEGACY 0 #define SOC15_CLIENT_ID_FROM_IH_ENTRY(entry) (le32_to_cpu(entry[0]) & 0xff) diff --git a/include/uapi/linux/kfd_ioctl.h b/include/uapi/linux/kfd_ioctl.h index 4f66764..f7cb7d4 100644 --- a/include/uapi/linux/kfd_ioctl.h +++ b/include/uapi/linux/kfd_ioctl.h @@ -442,6 +442,56 @@ struct kfd_ioctl_import_dmabuf_args { __u32 dmabuf_fd; /* to KFD */ }; +/** + * kfd_ioctl_spm_op - SPM ioctl operations + * + * @KFD_IOCTL_SPM_OP_ACQUIRE: acquire exclusive access to SPM + * @KFD_IOCTL_SPM_OP_RELEASE: release exclusive access to SPM + * @KFD_IOCTL_SPM_OP_SET_DEST_BUF: set or unset destination buffer for SPM streaming + */ +enum kfd_ioctl_spm_op { + KFD_IOCTL_SPM_OP_ACQUIRE, + KFD_IOCTL_SPM_OP_RELEASE, + KFD_IOCTL_SPM_OP_SET_DEST_BUF, + KFD_IOCTL_SPM_OP_CONFIG +}; + +/* Don't wait for previous buffer to fill up */ +#define KFD_IOCTL_SPM_FLAG_POLLING 1 + +/** + * kfd_ioctl_spm_args - Arguments for SPM ioctl + * + * @op: specifies the operation to perform + * @destptr:used for the address of the destination buffer in @KFD_IOCTL_SPM_SET_DEST_BUFFER + * @buf_size:size of the destination buffer in @KFD_IOCTL_SPM_SET_DEST_BUFFER + * @timeout: timeout to wait buffer get filled + * @gpu_id: gpi ID + * @bytes_copied: byte copied from streaming performance ring buffer + * + * If @ptr is NULL, the destination buffer address is unset and copying of counters + * is stopped. + * + * Returns negative error code on failure. On success, @KFD_IOCTL_SPM_OP_ACQUIRE and + * @KFD_IOCTL_SPM_OP_RELEASE return 0, @KFD_IOCTL_SPM_OP_SET_DEST_BUF returns the fill + * level of the previous buffer. + */ +struct kfd_ioctl_spm_args { + __u64 destptr; + __u64 spmtptr; + __u32 buf_size; + __u32 op; + __u32 timeout; + __u32 gpu_id; /* to KFD */ + /* from KFD: Total amount of bytes copied */ + __u64 bytes_copied; +}; + +struct kfd_spm_set_reg { + __u64 reg; /* to KFD */ + __u32 value; +}; + /* Register offset inside the remapped mmio page */ enum kfd_mmio_remap { @@ -546,7 +596,10 @@ enum kfd_mmio_remap { #define AMDKFD_IOC_ALLOC_QUEUE_GWS \ AMDKFD_IOWR(0x1E, struct kfd_ioctl_alloc_queue_gws_args) +#define AMDKFD_IOC_RLC_SPM \ + AMDKFD_IOWR(0x1F, struct kfd_ioctl_spm_args) + #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x1F +#define AMDKFD_COMMAND_END 0x20 #endif -- 2.7.4 _______________________________________________ amd-gfx mailing list amd-gfx@lists.freedesktop.org <a href="https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.freedesktop.org%2Fmailman%2Flistinfo%2Famd-gfx&data=02%7C01%7Cyong.zhao%40amd.com%7C326c22b09293444c1a3f08d7eb7c3c2b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637236791554300056&sdata=70Ud36avjI1hPHwGX0Enfo8YiRokubqepxdL3HLsocU%3D&reserved=0">https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.freedesktop.org%2Fmailman%2Flistinfo%2Famd-gfx&data=02%7C01%7Cyong.zhao%40amd.com%7C326c22b09293444c1a3f08d7eb7c3c2b%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637236791554300056&sdata=70Ud36avjI1hPHwGX0Enfo8YiRokubqepxdL3HLsocU%3D&reserved=0</a> </div> </div> </div> </body> </html>