[PATCH] drivers/pci: Decrease VF memory BAR size to Save host memory occupied by PTEs:

[Alex Deucher]

On Thu, Sep 15, 2022 at 6:34 AM Rui Ma <Rui.Ma at amd.com> wrote:
>
> The VRAM space is fixed, as the number of VFs increases, the actual BAR
> memory space used by each VF decreases. However, the BAR memory mapping is
> always based on the initial size of the VRAM. So do not map this unneeded
> memory can reduce the amount of PTEs to save host memory.
>
> Signed-off-by: Rui Ma <Rui.Ma at amd.com>
> ---
>  drivers/pci/iov.c    | 11 +++++++++--
>  drivers/pci/pci.h    | 15 +++++++++++++++
>  drivers/pci/quirks.c | 38 ++++++++++++++++++++++++++++++++++++++

Please use scripts/get_maintainer.pl to get the appropriate lists to
send this patch to for review.

Alex

>  3 files changed, 62 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c
> index 952217572113..d623f46669b6 100644
> --- a/drivers/pci/iov.c
> +++ b/drivers/pci/iov.c
> @@ -295,6 +295,11 @@ int pci_iov_add_virtfn(struct pci_dev *dev, int id)
>         struct resource *res;
>         struct pci_sriov *iov = dev->sriov;
>         struct pci_bus *bus;
> +    /*
> +     * Some SRIOV device's Bar is too large and occupy too much rmap size.
> +     * Resize the request resource of VF.
> +     */
> +       u16 shift = 1;
>
>         bus = virtfn_add_bus(dev->bus, pci_iov_virtfn_bus(dev, id));
>         if (!bus)
> @@ -328,8 +333,10 @@ int pci_iov_add_virtfn(struct pci_dev *dev, int id)
>                 virtfn->resource[i].name = pci_name(virtfn);
>                 virtfn->resource[i].flags = res->flags;
>                 size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
> +               shift = 1;
> +               shift = virtfn_get_shift(dev, iov->num_VFs, i);
>                 virtfn->resource[i].start = res->start + size * id;
> -               virtfn->resource[i].end = virtfn->resource[i].start + size - 1;
> +               virtfn->resource[i].end = virtfn->resource[i].start + (size >> (shift - 1)) - 1;
>                 rc = request_resource(res, &virtfn->resource[i]);
>                 BUG_ON(rc);
>         }
> @@ -680,12 +687,12 @@ static int sriov_enable(struct pci_dev *dev, int nr_virtfn)
>         msleep(100);
>         pci_cfg_access_unlock(dev);
>
> +       iov->num_VFs = nr_virtfn;
>         rc = sriov_add_vfs(dev, initial);
>         if (rc)
>                 goto err_pcibios;
>
>         kobject_uevent(&dev->dev.kobj, KOBJ_CHANGE);
> -       iov->num_VFs = nr_virtfn;
>
>         return 0;
>
> diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h
> index 3d60cabde1a1..befc67a280eb 100644
> --- a/drivers/pci/pci.h
> +++ b/drivers/pci/pci.h
> @@ -603,6 +603,21 @@ static inline int pci_dev_specific_reset(struct pci_dev *dev, bool probe)
>  }
>  #endif
>
> +struct virtfn_get_shift_methods {
> +       u16 vendor;
> +       u16 device;
> +       u16 (*get_shift)(struct pci_dev *dev, u16 arg, int arg2);
> +};
> +
> +#ifdef CONFIG_PCI_QUIRKS
> +u16 virtfn_get_shift(struct pci_dev *dev, u16 arg1, int arg2);
> +#else
> +static inline u16 virtfn_get_shift(struct pci_dev *dev, u16 arg1, int arg2)
> +{
> +       return (u16)1;
> +}
> +#endif
> +
>  #if defined(CONFIG_PCI_QUIRKS) && defined(CONFIG_ARM64)
>  int acpi_get_rc_resources(struct device *dev, const char *hid, u16 segment,
>                           struct resource *res);
> diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
> index da829274fc66..82502c5923e4 100644
> --- a/drivers/pci/quirks.c
> +++ b/drivers/pci/quirks.c
> @@ -4085,6 +4085,44 @@ int pci_dev_specific_reset(struct pci_dev *dev, bool probe)
>         return -ENOTTY;
>  }
>
> +static u16 divided_by_VF(struct pci_dev *dev, u16 num_VFs, int bar_num)
> +{
> +       u16 shift = 1;
> +
> +       if (bar_num == 0) {
> +               while ((1 << shift) <= num_VFs)
> +                       shift += 1;
> +       }
> +       pci_info(dev, "Bar %d get shift: %d.\n", bar_num, shift);
> +       return shift;
> +}
> +
> +static const struct virtfn_get_shift_methods virtfn_get_shift_methods[] = {
> +       { PCI_VENDOR_ID_ATI, 0x73a1, divided_by_VF},
> +       { 0 }
> +};
> +
> +/*
> + * Get shift num to calculate SRIOV device bar.
> + * Sometimes the bar size for SRIOV device is too large
> + * and we want to calculate the size to define the end
> + * of virtfn.
> + */
> +u16 virtfn_get_shift(struct pci_dev *dev, u16 arg1, int arg2)
> +{
> +       const struct virtfn_get_shift_methods *i;
> +
> +       for (i = virtfn_get_shift_methods; i->get_shift; i++) {
> +               if ((i->vendor == dev->vendor ||
> +                    i->vendor == (u16)PCI_ANY_ID) &&
> +                   (i->device == dev->device ||
> +                    i->device == (u16)PCI_ANY_ID))
> +                       return i->get_shift(dev, arg1, arg2);
> +       }
> +
> +       return (u16)1;
> +}
> +
>  static void quirk_dma_func0_alias(struct pci_dev *dev)
>  {
>         if (PCI_FUNC(dev->devfn) != 0)
> --
> 2.25.1
>