[PATCH 11/13] drm/amdgpu: add DRM_AMDGPU_ATC config option

[Felix Kuehling]

On 2018-01-26 03:13 PM, Christian König wrote:
> Move amd_iommu_v2 initialization into amdgpu when it is enabled.
>
> This is WIP and really ugly since amdgpu should not depend directly on
> amd_iommu_v2.
>
> Signed-off-by: Christian König <christian.koenig at amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/Kconfig         |  8 ++++++++
>  drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 12 ++++++++++++
>  drivers/gpu/drm/amd/amdkfd/kfd_device.c    | 11 ++++++++++-
>  3 files changed, 30 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/Kconfig b/drivers/gpu/drm/amd/amdgpu/Kconfig
> index e8af1f5e8a79..7d3fdcbf0acb 100644
> --- a/drivers/gpu/drm/amd/amdgpu/Kconfig
> +++ b/drivers/gpu/drm/amd/amdgpu/Kconfig
> @@ -40,5 +40,13 @@ config DRM_AMDGPU_GART_DEBUGFS
>  	  Selecting this option creates a debugfs file to inspect the mapped
>  	  pages. Uses more memory for housekeeping, enable only for debugging.
>  
> +config DRM_AMDGPU_ATC
> +	bool "Enable the ATC to provide SVM support"
> +	depends on DRM_AMDGPU
> +	depends on AMD_IOMMU_V2
> +	default y
> +	help
> +		This enables support for the ATC to provide a shared virtual memory implementation.
> +
>  source "drivers/gpu/drm/amd/acp/Kconfig"
>  source "drivers/gpu/drm/amd/display/Kconfig"
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index 84281ee34a25..04205236cc5e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -35,6 +35,7 @@
>  #include <linux/vgaarb.h>
>  #include <linux/vga_switcheroo.h>
>  #include <linux/efi.h>
> +#include <linux/amd-iommu.h>
>  #include "amdgpu.h"
>  #include "amdgpu_trace.h"
>  #include "amdgpu_i2c.h"
> @@ -1355,6 +1356,12 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev)
>  		adev->ip_blocks[i].status.hw = true;
>  	}
>  
> +#ifdef CONFIG_DRM_AMDGPU_ATC
> +	r = amd_iommu_init_device(adev->pdev, 0x10000);

KFD queries how many PASIDs the IOMMU can support with
amd_iommu_device_info. KFD only assigns PASIDs within that range. It can
be much smaller than the 16-bits supported by the GPU.

For a VM that uses ATC, you need to make sure it gets a PASID in the
range supported by the IOMMU. The PASID manager already supports that
and keeps smaller PASIDs for users that really need them.

Regards,
  Felix

> +	if (r)
> +		DRM_ERROR("amd_iommu_init_device error %d\n", r);
> +#endif
> +
>  	amdgpu_amdkfd_device_init(adev);
>  
>  	if (amdgpu_sriov_vf(adev))
> @@ -1428,6 +1435,11 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
>  	int i, r;
>  
>  	amdgpu_amdkfd_device_fini(adev);
> +
> +#ifdef CONFIG_DRM_AMDGPU_ATC
> +	amd_iommu_free_device(adev->pdev);
> +#endif
> +
>  	/* need to disable SMC first */
>  	for (i = 0; i < adev->num_ip_blocks; i++) {
>  		if (!adev->ip_blocks[i].status.hw)
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
> index a8fa33a08de3..7ed090f7c5f0 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
> @@ -373,7 +373,10 @@ void kgd2kfd_suspend(struct kfd_dev *kfd)
>  
>  	amd_iommu_set_invalidate_ctx_cb(kfd->pdev, NULL);
>  	amd_iommu_set_invalid_ppr_cb(kfd->pdev, NULL);
> +
> +#ifndef CONFIG_DRM_AMDGPU_ATC
>  	amd_iommu_free_device(kfd->pdev);
> +#endif
>  }
>  
>  int kgd2kfd_resume(struct kfd_dev *kfd)
> @@ -388,11 +391,15 @@ int kgd2kfd_resume(struct kfd_dev *kfd)
>  static int kfd_resume(struct kfd_dev *kfd)
>  {
>  	int err = 0;
> +
> +#ifndef CONFIG_DRM_AMDGPU_ATC
>  	unsigned int pasid_limit = kfd_get_pasid_limit();
>  
>  	err = amd_iommu_init_device(kfd->pdev, pasid_limit);
>  	if (err)
>  		return -ENXIO;
> +#endif
> +
>  	amd_iommu_set_invalidate_ctx_cb(kfd->pdev,
>  					iommu_pasid_shutdown_callback);
>  	amd_iommu_set_invalid_ppr_cb(kfd->pdev,
> @@ -414,8 +421,10 @@ static int kfd_resume(struct kfd_dev *kfd)
>  
>  dqm_start_error:
>  processes_bind_error:
> -	amd_iommu_free_device(kfd->pdev);
>  
> +#ifndef CONFIG_DRM_AMDGPU_ATC
> +	amd_iommu_free_device(kfd->pdev);
> +#endif
>  	return err;
>  }
>