[PATCH 3/8] amdgpu: resize BAR0 to the maximum available size, even if it doesn't cover VRAM (v4)

[Christian König]

Am 13.12.20 um 23:53 schrieb Darren Salt:
> This allows BAR0 resizing to be done for cards which don't advertise support
> for a size large enough to cover the VRAM but which do advertise at least
> one size larger than the default. For example, my RX 5600 XT, which
> advertises 256MB, 512MB and 1GB.
>
> [v4] use bit ops to find sizes to try
>
> [v3] don't use pci_rebar_get_current_size()
>
> [v2] rewritten to use PCI helper functions; some extra log text.
>
> Signed-off-by: Darren Salt <devspam at moreofthesa.me.uk>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 54 ++++++++++++++++++----
>   1 file changed, 44 insertions(+), 10 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index 1595b124c145..b0f8ad603147 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -908,23 +908,29 @@ void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
>    */
>   int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
>   {
> -	u64 space_needed = roundup_pow_of_two(adev->gmc.real_vram_size);
> -	u32 rbar_size = order_base_2(((space_needed >> 20) | 1)) - 1;
> +	int rbar_size, current_size;
> +	u64 current_bytes;
> +	u32 available_sizes;
>   	struct pci_bus *root;
>   	struct resource *res;
>   	unsigned i;
>   	u16 cmd;
>   	int r;
> +	bool nospc = false;
>   
>   	/* Bypass for VF */
>   	if (amdgpu_sriov_vf(adev))
>   		return 0;
>   
> -	/* skip if the bios has already enabled large BAR */
> -	if (adev->gmc.real_vram_size &&
> -	    (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size))
> +	current_bytes = pci_resource_len(adev->pdev, 0);
> +
> +	/* Skip if the BIOS has already enabled large BAR, covering the VRAM */
> +	if (current_bytes >= adev->gmc.real_vram_size)
>   		return 0;
>   
> +	current_size = current_bytes ? pci_rebar_bytes_to_size(current_bytes) : -1;
> +	rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size);
> +
>   	/* Check if the root BUS has 64bit memory resources */
>   	root = adev->pdev->bus;
>   	while (root->parent)
> @@ -940,6 +946,18 @@ int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
>   	if (!res)
>   		return 0;
>   
> +	available_sizes = pci_rebar_get_possible_sizes(adev->pdev, 0);
> +	if (available_sizes == 0)
> +		return 0;
> +
> +	/* Find the bit pos representing the smallest size covering the VRAM.
> +	 * Otherwise find the bit pos representing the largest available size.
> +	 * rbar_size may be smaller but should be unchanged.
> +	 */
> +	rbar_size = (ffs(available_sizes & (-1 << rbar_size)) ? : fls(available_sizes)) - 1;
> +	if (rbar_size < 0)
> +		return 0; /* can't happen */
> +
>   	/* Disable memory decoding while we change the BAR addresses and size */
>   	pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
>   	pci_write_config_word(adev->pdev, PCI_COMMAND,
> @@ -952,11 +970,27 @@ int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
>   
>   	pci_release_resource(adev->pdev, 0);
>   
> -	r = pci_resize_resource(adev->pdev, 0, rbar_size);
> -	if (r == -ENOSPC)
> -		DRM_INFO("Not enough PCI address space for a large BAR.");
> -	else if (r && r != -ENOTSUPP)
> -		DRM_ERROR("Problem resizing BAR0 (%d).", r);
> +	for (;
> +	     rbar_size >= 0 && rbar_size > current_size;
> +	     rbar_size = fls(available_sizes & ~(-1 << rbar_size)) - 1
> +	    ) {

Please completely remove the loop here. We should never try more than 
one size.

Regards,
Christian.

> +		r = pci_resize_resource(adev->pdev, 0, rbar_size);
> +		if (r == 0) {
> +			break;
> +		} else if (r == -ENOTSUPP) {
> +			dev_info(adev->dev, "BAR resizing not supported.");
> +			break;
> +		} else if (r == -ENOSPC) {
> +			if (!nospc) {
> +				/* Warn only the first time */
> +				dev_info(adev->dev, "Not enough PCI address space for a large BAR.");
> +				nospc = true;
> +			}
> +		} else {
> +			dev_err(adev->dev, "Problem resizing BAR0 (%d).", r);
> +			break;
> +		}
> +	}
>   
>   	pci_assign_unassigned_bus_resources(adev->pdev->bus);
>