[Intel-xe] [PATCH 4/4] xe/drm/pm: manoeuvre d3cold_allowed using vram_usages

[Anshuman Gupta]

Adding support to control d3cold by using vram_usages metric from
ttm resource manager.
When root port  is capable of d3cold but xe has disallowed d3cold
due to vrame_usages above vram_d3ccold_threshol. It is required to
disable d3cold to avoid any resume failure because root port may
transition to d3cold when all of pcie endpoints and
{upstream, virtual} switch ports will transition to D3hot.

TODO: At resume vram save/restore only in case card really lost power.

Cc: Rodrigo Vivi <rodrigo.vivi at intel.com>
Signed-off-by: Anshuman Gupta <anshuman.gupta at intel.com>
---
 drivers/gpu/drm/xe/xe_pci.c | 24 ++++++++++++++++++++++++
 drivers/gpu/drm/xe/xe_pm.c  | 19 +++++++++++++++++++
 drivers/gpu/drm/xe/xe_pm.h  |  1 +
 3 files changed, 44 insertions(+)

diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c
index 137026ddee42..1d12e1b5bb62 100644
--- a/drivers/gpu/drm/xe/xe_pci.c
+++ b/drivers/gpu/drm/xe/xe_pci.c
@@ -738,6 +738,25 @@ static int xe_pci_resume(struct device *dev)
 	return 0;
 }
 
+static void xe_pci_d3cold_enable_disable(struct pci_dev *pdev, bool enable)
+{
+	struct xe_device *xe = pdev_to_xe_device(pdev);
+	struct pci_dev *root_pdev;
+
+	if (!xe->d3cold_capable)
+		return;
+
+	root_pdev = pcie_find_root_port(pdev);
+	if (!root_pdev)
+		return;
+
+	if (enable)
+		pci_d3cold_enable(root_pdev);
+	else
+		pci_d3cold_disable(root_pdev);
+
+}
+
 static int xe_pci_runtime_suspend(struct device *dev)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
@@ -755,6 +774,7 @@ static int xe_pci_runtime_suspend(struct device *dev)
 		pci_ignore_hotplug(pdev);
 		pci_set_power_state(pdev, PCI_D3cold);
 	} else {
+		xe_pci_d3cold_enable_disable(pdev, false);
 		pci_set_power_state(pdev, PCI_D3hot);
 	}
 
@@ -779,6 +799,8 @@ static int xe_pci_runtime_resume(struct device *dev)
 			return err;
 
 		pci_set_master(pdev);
+	} else {
+		xe_pci_d3cold_enable_disable(pdev, true);
 	}
 
 	return xe_pm_runtime_resume(xe);
@@ -794,6 +816,8 @@ static int xe_pci_runtime_idle(struct device *dev)
 		return 0;
 	}
 
+	xe->d3cold_allowed = xe_pm_vram_d3cold_allowed(xe);
+
 	/*
 	 * TODO: d3cold should be allowed (true) if
 	 * (IS_DGFX(xe) && !xe_device_mem_access_ongoing(xe))
diff --git a/drivers/gpu/drm/xe/xe_pm.c b/drivers/gpu/drm/xe/xe_pm.c
index ebdbdf2e0898..c864dc2ac6c2 100644
--- a/drivers/gpu/drm/xe/xe_pm.c
+++ b/drivers/gpu/drm/xe/xe_pm.c
@@ -133,6 +133,25 @@ static bool xe_pm_pci_d3cold_capable(struct pci_dev *pdev)
 	return true;
 }
 
+bool xe_pm_vram_d3cold_allowed(struct xe_device *xe)
+{
+	struct ttm_resource_manager *man;
+	u64 vram_used;
+	size_t total_vram_used_mb = 0;
+	int i;
+
+	/* TODO: Extend the logic to beyond XE_PL_VRAM1 */
+	for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) {
+		man = ttm_manager_type(&xe->ttm, i);
+		if (man) {
+			vram_used = ttm_resource_manager_usage(man);
+			total_vram_used_mb += DIV_ROUND_UP_ULL(vram_used, 1024 * 1024);
+		}
+	}
+
+	return total_vram_used_mb <= xe->vram_d3cold_threshold;
+}
+
 static ssize_t
 vram_d3cold_threshold_show(struct device *dev,
 			   struct device_attribute *attr, char *buf)
diff --git a/drivers/gpu/drm/xe/xe_pm.h b/drivers/gpu/drm/xe/xe_pm.h
index e0c4f92e27c5..6c0016470abf 100644
--- a/drivers/gpu/drm/xe/xe_pm.h
+++ b/drivers/gpu/drm/xe/xe_pm.h
@@ -13,6 +13,7 @@ struct xe_device;
 int xe_pm_suspend(struct xe_device *xe);
 int xe_pm_resume(struct xe_device *xe);
 
+bool xe_pm_vram_d3cold_allowed(struct xe_device *xe);
 void xe_pm_init(struct xe_device *xe);
 void xe_pm_runtime_fini(struct xe_device *xe);
 int xe_pm_runtime_suspend(struct xe_device *xe);
-- 
2.38.0