[PATCH 04/13] drm/amdgpu/gmc9: Adjust xgmi offset

[Christian König]

Am 05.09.2018 um 17:29 schrieb shaoyunl:
> From: Alex Deucher <alexander.deucher at amd.com>
>
> On hives with xgmi enabled, the fb_location aperture is a size
> which defines the total framebuffer size of all nodes in the
> hive.  Each GPU in the hive has the same view via the fb_location
> aperture.  GPU0 starts at offset (0 * segment size),
> GPU1 starts at offset (1 * segment size), etc.
>
> For access to local vram on each GPU, we need to take this offset into
> account. This including on setting up GPUVM page table  and  GART table
>
> Acked-by: Huang Rui <ray.huang at amd.com>
> Acked-by: Slava Abramov <slava.abramov at amd.com>
> Signed-off-by: Shaoyun Liu <Shaoyun.Liu at amd.com>
> Signed-off-by: Alex Deucher <alexander.deucher at amd.com>
> Reviewed-by: Felix Kuehling <Felix.Kuehling at amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c  | 4 ++++
>   drivers/gpu/drm/amd/amdgpu/gfxhub_v1_1.c | 3 +++
>   drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c    | 6 ++++++
>   3 files changed, 13 insertions(+)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> index 6acdeeb..a95b615 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> @@ -158,6 +158,10 @@ void amdgpu_gmc_gart_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc)
>   	if ((size_bf >= mc->gart_size && size_bf < size_af) ||
>   	    (size_af < mc->gart_size))
>   		mc->gart_start = 0;
> +	else if (mc->xgmi.num_physical_nodes)
> +		mc->gart_start = mc->vram_start +
> +			(mc->xgmi.num_physical_nodes - mc->xgmi.physical_node_id)
> +			* mc->xgmi.node_segment_size;

That looks incomplete to me. The calculation above as well as the AGP 
code needs adjustment as well.

Probably best if some fb_start/fb_end fields to struct amdgpu_gmc and 
calculate that only once in amdgpu_gmc_vram_location.

Christian.

>   	else
>   		mc->gart_start = mc->mc_mask - mc->gart_size + 1;
>   
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_1.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_1.c
> index d4170cb..5e9ab8e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_1.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_1.c
> @@ -44,6 +44,9 @@ int gfxhub_v1_1_get_xgmi_info(struct amdgpu_device *adev)
>   			REG_GET_FIELD(xgmi_lfb_cntl, MC_VM_XGMI_LFB_CNTL, PF_LFB_REGION);
>   		if (adev->gmc.xgmi.physical_node_id > 3)
>   			return -EINVAL;
> +		adev->gmc.xgmi.node_segment_size = REG_GET_FIELD(
> +			RREG32_SOC15(GC, 0, mmMC_VM_XGMI_LFB_SIZE),
> +			MC_VM_XGMI_LFB_SIZE, PF_LFB_SIZE) << 24;
>   	}
>   
>   	return 0;
> diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
> index 3529c55..0da89ba 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
> @@ -770,12 +770,18 @@ static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev,
>   	u64 base = 0;
>   	if (!amdgpu_sriov_vf(adev))
>   		base = mmhub_v1_0_get_fb_location(adev);
> +	/* add the xgmi offset of the physical node */
> +	base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
>   	amdgpu_gmc_vram_location(adev, &adev->gmc, base);
>   	amdgpu_gmc_gart_location(adev, mc);
>   	if (!amdgpu_sriov_vf(adev))
>   		amdgpu_gmc_agp_location(adev, mc);
>   	/* base offset of vram pages */
>   	adev->vm_manager.vram_base_offset = gfxhub_v1_0_get_mc_fb_offset(adev);
> +
> +	/* XXX: add the xgmi offset of the physical node? */
> +	adev->vm_manager.vram_base_offset +=
> +		adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
>   }
>   
>   /**