[PATCH] drm/amdgpu: Default disable GDS for compute VMIDs

[Kuehling, Felix]

On 2019-07-17 13:11, Greathouse, Joseph wrote:
> The GDS and GWS blocks default to allowing all VMIDs to
> access all entries. Graphics VMIDs can handle setting
> these limits when the driver launches work. However,
> compute workloads under HWS control don't go through the
> kernel driver. Instead, HWS firmware should set these
> limits when a process is put into a VMID slot.
>
> Disable access to these devices by default by turning off
> all mask bits (for OA) and setting BASE=SIZE=0 (for GDS
> and GWS) for all compute VMIDs. If a process wants to use
> these resources, they can request this from the HWS
> firmware (when such capabilities are enabled). HWS will
> then handle setting the base and limit for the process when
> it is assigned to a VMID.
>
> This will also prevent user kernels from getting 'stuck' in
> GWS by accident if they write GWS-using code but HWS
> firmware is not set up to handle GWS reset. Until HWS is
> enabled to handle GWS properly, all GWS accesses will
> MEM_VIOL fault the kernel.
>
> Change-Id: I2a709fdcb2468511754f2e5eae75546751c0e6f0
> Signed-off-by: Joseph Greathouse <Joseph.Greathouse at amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 6 ++++++
>   drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c  | 6 ++++++
>   drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c  | 6 ++++++
>   drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c  | 6 ++++++
>   4 files changed, 24 insertions(+)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
> index 618291df659b..32da5a91abfb 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
> @@ -1513,6 +1513,12 @@ static void gfx_v10_0_init_compute_vmid(struct amdgpu_device *adev)
>   		/* CP and shaders */
>   		WREG32_SOC15(GC, 0, mmSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
>   		WREG32_SOC15(GC, 0, mmSH_MEM_BASES, sh_mem_bases);
> +		/* Initialize all compute VMIDs to have no GDS, GWS, or OA
> +		   acccess. These should be enabled by FW for target VMID. */
> +		WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0x00000000);
> +		WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0x00000000);
> +		WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0x00000000);
> +		WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0x00000000);
I'd prefer these register initializations in a separate loop outside the 
srbm_mutex, because they don't depend on the srbm/grbm_select bits. Same 
for the other GFX IP versions.

Regards,
   Felix


>   	}
>   	nv_grbm_select(adev, 0, 0, 0, 0);
>   	mutex_unlock(&adev->srbm_mutex);
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
> index e1e2a44ee13c..89ea0d799c12 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
> @@ -1874,6 +1874,12 @@ static void gfx_v7_0_init_compute_vmid(struct amdgpu_device *adev)
>   		WREG32(mmSH_MEM_APE1_BASE, 1);
>   		WREG32(mmSH_MEM_APE1_LIMIT, 0);
>   		WREG32(mmSH_MEM_BASES, sh_mem_bases);
> +		/* Initialize all compute VMIDs to have no GDS, GWS, or OA
> +		   acccess. These should be enabled by FW for target VMID. */
> +		WREG32(amdgpu_gds_reg_offset[i].mem_base, 0);
> +		WREG32(amdgpu_gds_reg_offset[i].mem_size, 0);
> +		WREG32(amdgpu_gds_reg_offset[i].gws, 0);
> +		WREG32(amdgpu_gds_reg_offset[i].oa, 0);
>   	}
>   	cik_srbm_select(adev, 0, 0, 0, 0);
>   	mutex_unlock(&adev->srbm_mutex);
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
> index 8c590a554675..53d9a223f8c1 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
> @@ -3699,6 +3699,12 @@ static void gfx_v8_0_init_compute_vmid(struct amdgpu_device *adev)
>   		WREG32(mmSH_MEM_APE1_BASE, 1);
>   		WREG32(mmSH_MEM_APE1_LIMIT, 0);
>   		WREG32(mmSH_MEM_BASES, sh_mem_bases);
> +		/* Initialize all compute VMIDs to have no GDS, GWS, or OA
> +		   acccess. These should be enabled by FW for target VMID. */
> +		WREG32(amdgpu_gds_reg_offset[i].mem_base, 0);
> +		WREG32(amdgpu_gds_reg_offset[i].mem_size, 0);
> +		WREG32(amdgpu_gds_reg_offset[i].gws, 0);
> +		WREG32(amdgpu_gds_reg_offset[i].oa, 0);
>   	}
>   	vi_srbm_select(adev, 0, 0, 0, 0);
>   	mutex_unlock(&adev->srbm_mutex);
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> index 5af60e1c735a..b9ed03a8a561 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> @@ -2022,6 +2022,12 @@ static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev)
>   		/* CP and shaders */
>   		WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config);
>   		WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases);
> +		/* Initialize all compute VMIDs to have no GDS, GWS, or OA
> +		   acccess. These should be enabled by FW for target VMID. */
> +		WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0x00000000);
> +		WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0x00000000);
> +		WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0x00000000);
> +		WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0x00000000);
>   	}
>   	soc15_grbm_select(adev, 0, 0, 0, 0);
>   	mutex_unlock(&adev->srbm_mutex);