<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=us-ascii">
<style type="text/css" style="display:none;"> P {margin-top:0;margin-bottom:0;} </style>
</head>
<body dir="ltr">
<p style="font-family:Calibri;font-size:10pt;color:#0000FF;margin:5pt;font-style:normal;font-weight:normal;text-decoration:none;" align="Left">
[AMD Official Use Only - AMD Internal Distribution Only]<br>
</p>
<br>
<div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 11pt; color: rgb(0, 0, 0);" class="elementToProof">
Tested-by: Ce Sun <cesun102@amd.com></div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);" class="elementToProof">
<br>
</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);" class="elementToProof">
<br>
</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
<br>
</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
<br>
</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
<br>
</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
<br>
</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
<br>
</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
<br>
</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
<br>
</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
<br>
</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);" class="elementToProof">
Thanks</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);" class="elementToProof">
Ce Sun</div>
<div id="appendonsend"></div>
<hr style="display:inline-block;width:98%" tabindex="-1">
<div id="divRplyFwdMsg" dir="ltr"><font face="Calibri, sans-serif" style="font-size:11pt" color="#000000"><b>From:</b> Lazar, Lijo <Lijo.Lazar@amd.com><br>
<b>Sent:</b> Friday, August 1, 2025 3:26 PM<br>
<b>To:</b> amd-gfx@lists.freedesktop.org <amd-gfx@lists.freedesktop.org><br>
<b>Cc:</b> Zhang, Hawking <Hawking.Zhang@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Sun, Ce(Overlord) <Ce.Sun@amd.com><br>
<b>Subject:</b> [PATCH] drm/amdgpu: Save and restore switch state</font>
<div> </div>
</div>
<div class="BodyFragment"><font size="2"><span style="font-size:11pt;">
<div class="PlainText">During a DPC error kernel waits for the link to be active before<br>
notifying downstream devices. On certain platforms with Broadcom switch<br>
in synthetiic mode, switch responds with values even though the link is<br>
not fully ready. The config space restoration done by pcie port driver<br>
for SWUS/DS of dGPU is thus not effective as the switch is still doing<br>
internal enumeration.<br>
<br>
As a workaround, save state of SWUS/DS device in driver. Add additional<br>
check to see if link is active and restore the values during DPC error<br>
callbacks.<br>
<br>
Signed-off-by: Lijo Lazar <lijo.lazar@amd.com><br>
---<br>
drivers/gpu/drm/amd/amdgpu/amdgpu.h | 3 +<br>
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 78 +++++++++++++++++++++-<br>
2 files changed, 78 insertions(+), 3 deletions(-)<br>
<br>
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h<br>
index 3550c2fac184..96d772aadb04 100644<br>
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h<br>
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h<br>
@@ -904,6 +904,9 @@ struct amdgpu_pcie_reset_ctx {<br>
bool in_link_reset;<br>
bool occurs_dpc;<br>
bool audio_suspended;<br>
+ struct pci_dev *swus;<br>
+ struct pci_saved_state *swus_pcistate;<br>
+ struct pci_saved_state *swds_pcistate;<br>
};<br>
<br>
/*<br>
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c<br>
index cfd72faec16e..962c57a3ec93 100644<br>
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c<br>
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c<br>
@@ -178,6 +178,8 @@ struct amdgpu_init_level amdgpu_init_minimal_xgmi = {<br>
BIT(AMD_IP_BLOCK_TYPE_PSP)<br>
};<br>
<br>
+static void amdgpu_device_load_switch_state(struct amdgpu_device *adev);<br>
+<br>
static inline bool amdgpu_ip_member_of_hwini(struct amdgpu_device *adev,<br>
enum amd_ip_block_type block)<br>
{<br>
@@ -5006,7 +5008,8 @@ void amdgpu_device_fini_sw(struct amdgpu_device *adev)<br>
adev->reset_domain = NULL;<br>
<br>
kfree(adev->pci_state);<br>
-<br>
+ kfree(adev->pcie_reset_ctx.swds_pcistate);<br>
+ kfree(adev->pcie_reset_ctx.swus_pcistate);<br>
}<br>
<br>
/**<br>
@@ -6963,16 +6966,31 @@ pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev)<br>
struct amdgpu_device *tmp_adev;<br>
struct amdgpu_hive_info *hive;<br>
struct list_head device_list;<br>
+ struct pci_dev *link_dev;<br>
int r = 0, i;<br>
u32 memsize;<br>
+ u16 status;<br>
<br>
dev_info(adev->dev, "PCI error: slot reset callback!!\n");<br>
<br>
memset(&reset_context, 0, sizeof(reset_context));<br>
<br>
+ if (adev->pcie_reset_ctx.swus)<br>
+ link_dev = adev->pcie_reset_ctx.swus;<br>
+ else<br>
+ link_dev = adev->pdev;<br>
/* wait for asic to come out of reset */<br>
- msleep(700);<br>
-<br>
+ i = 0;<br>
+ do {<br>
+ msleep(10);<br>
+ r = pci_read_config_word(link_dev, PCI_VENDOR_ID, &status);<br>
+ if (i % 100 == 0)<br>
+ dev_info(adev->dev, "read status :%x r: %d", status, r);<br>
+ ++i;<br>
+ } while ((status != PCI_VENDOR_ID_ATI) &&<br>
+ (status != PCI_VENDOR_ID_AMD));<br>
+<br>
+ amdgpu_device_load_switch_state(adev);<br>
/* Restore PCI confspace */<br>
amdgpu_device_load_pci_state(pdev);<br>
<br>
@@ -7074,6 +7092,58 @@ void amdgpu_pci_resume(struct pci_dev *pdev)<br>
}<br>
}<br>
<br>
+static void amdgpu_device_cache_switch_state(struct amdgpu_device *adev)<br>
+{<br>
+ struct pci_dev *parent = pci_upstream_bridge(adev->pdev);<br>
+ int r;<br>
+<br>
+ if (parent->vendor != PCI_VENDOR_ID_ATI)<br>
+ return;<br>
+<br>
+ /* If already saved, return */<br>
+ if (adev->pcie_reset_ctx.swus)<br>
+ return;<br>
+ /* Upstream bridge is ATI, assume it's SWUS/DS architecture */<br>
+ r = pci_save_state(parent);<br>
+ if (r)<br>
+ return;<br>
+ adev->pcie_reset_ctx.swds_pcistate = pci_store_saved_state(parent);<br>
+<br>
+ parent = pci_upstream_bridge(parent);<br>
+ r = pci_save_state(parent);<br>
+ if (r)<br>
+ return;<br>
+ adev->pcie_reset_ctx.swus_pcistate = pci_store_saved_state(parent);<br>
+<br>
+ adev->pcie_reset_ctx.swus = parent;<br>
+}<br>
+<br>
+static void amdgpu_device_load_switch_state(struct amdgpu_device *adev)<br>
+{<br>
+ struct pci_dev *pdev;<br>
+ int r;<br>
+<br>
+ if (!adev->pcie_reset_ctx.swds_pcistate ||<br>
+ !adev->pcie_reset_ctx.swus_pcistate)<br>
+ return;<br>
+<br>
+ pdev = adev->pcie_reset_ctx.swus;<br>
+ r = pci_load_saved_state(pdev, adev->pcie_reset_ctx.swus_pcistate);<br>
+ if (!r) {<br>
+ pci_restore_state(pdev);<br>
+ } else {<br>
+ dev_warn(adev->dev, "Failed to load SWUS state, err:%d\n", r);<br>
+ return;<br>
+ }<br>
+<br>
+ pdev = pci_upstream_bridge(adev->pdev);<br>
+ r = pci_load_saved_state(pdev, adev->pcie_reset_ctx.swds_pcistate);<br>
+ if (!r)<br>
+ pci_restore_state(pdev);<br>
+ else<br>
+ dev_warn(adev->dev, "Failed to load SWDS state, err:%d\n", r);<br>
+}<br>
+<br>
bool amdgpu_device_cache_pci_state(struct pci_dev *pdev)<br>
{<br>
struct drm_device *dev = pci_get_drvdata(pdev);<br>
@@ -7098,6 +7168,8 @@ bool amdgpu_device_cache_pci_state(struct pci_dev *pdev)<br>
return false;<br>
}<br>
<br>
+ amdgpu_device_cache_switch_state(adev);<br>
+<br>
return true;<br>
}<br>
<br>
-- <br>
2.49.0<br>
<br>
</div>
</span></font></div>
</div>
</body>
</html>