[PATCH v6 1/2] drm/amdgpu: add debugfs for reset registers list
Lazar, Lijo
lijo.lazar at amd.com
Thu Feb 17 10:01:32 UTC 2022
On 2/17/2022 3:18 PM, Christian König wrote:
> Am 17.02.22 um 10:44 schrieb Lazar, Lijo:
>>
>>
>> On 2/17/2022 1:30 PM, Christian König wrote:
>>>
>>>
>>> Am 17.02.22 um 08:54 schrieb Somalapuram, Amaranath:
>>>>
>>>>
>>>> On 2/16/2022 8:26 PM, Christian König wrote:
>>>>> Am 16.02.22 um 14:11 schrieb Somalapuram, Amaranath:
>>>>>>
>>>>>> On 2/16/2022 3:41 PM, Christian König wrote:
>>>>>>
>>>>>>> Am 16.02.22 um 10:49 schrieb Somalapuram Amaranath:
>>>>>>>> List of register populated for dump collection during the GPU
>>>>>>>> reset.
>>>>>>>>
>>>>>>>> Signed-off-by: Somalapuram Amaranath
>>>>>>>> <Amaranath.Somalapuram at amd.com>
>>>>>>>> ---
>>>>>>>> drivers/gpu/drm/amd/amdgpu/amdgpu.h | 5 ++
>>>>>>>> drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c | 95
>>>>>>>> +++++++++++++++++++++
>>>>>>>> 2 files changed, 100 insertions(+)
>>>>>>>>
>>>>>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>>>>>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>>>>>>>> index b85b67a88a3d..57965316873b 100644
>>>>>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>>>>>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>>>>>>>> @@ -1097,6 +1097,11 @@ struct amdgpu_device {
>>>>>>>> struct amdgpu_reset_control *reset_cntl;
>>>>>>>> uint32_t ip_versions[HW_ID_MAX][HWIP_MAX_INSTANCE];
>>>>>>>> +
>>>>>>>> + /* reset dump register */
>>>>>>>> + uint32_t *reset_dump_reg_list;
>>>>>>>> + int n_regs;
>>>>>>>> + struct mutex reset_dump_mutex;
>>>>>>>
>>>>>>> I think we should rather use the reset lock for this instead of
>>>>>>> introducing just another mutex.
>>>>>>>
>>>>>>>> };
>>>>>>>> static inline struct amdgpu_device *drm_to_adev(struct
>>>>>>>> drm_device *ddev)
>>>>>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
>>>>>>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
>>>>>>>> index 164d6a9e9fbb..faf985c7cb93 100644
>>>>>>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
>>>>>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
>>>>>>>> @@ -1609,6 +1609,98 @@ DEFINE_DEBUGFS_ATTRIBUTE(fops_ib_preempt,
>>>>>>>> NULL,
>>>>>>>> DEFINE_DEBUGFS_ATTRIBUTE(fops_sclk_set, NULL,
>>>>>>>> amdgpu_debugfs_sclk_set, "%llu\n");
>>>>>>>> +static ssize_t amdgpu_reset_dump_register_list_read(struct
>>>>>>>> file *f,
>>>>>>>> + char __user *buf, size_t size, loff_t *pos)
>>>>>>>> +{
>>>>>>>> + struct amdgpu_device *adev = (struct amdgpu_device
>>>>>>>> *)file_inode(f)->i_private;
>>>>>>>> + char reg_offset[11];
>>>>>>>> + int i, r, len = 0;
>>>>>>>> +
>>>>>>>> + if (*pos)
>>>>>>>> + return 0;
>>>>>>>> +
>>>>>>>> + if (adev->n_regs == 0)
>>>>>>>> + return 0;
>>>>>>>> +
>>>>>>>> + for (i = 0; i < adev->n_regs; i++) {
>>>>>>>> + sprintf(reg_offset, "0x%x ",
>>>>>>>> adev->reset_dump_reg_list[i]);
>>>>>>>> + r = copy_to_user(buf + len, reg_offset,
>>>>>>>> strlen(reg_offset));
>>>>>>>> +
>>>>>>>> + if (r)
>>>>>>>> + return -EFAULT;
>>>>>>>> +
>>>>>>>> + len += strlen(reg_offset);
>>>>>>>> + }
>>>>>>>
>>>>>>> You need to hold the lock protecting adev->reset_dump_reg_list
>>>>>>> and adev->n_regs while accessing those.
>>>>>>>
>>>>>>> (BTW: num_regs instead of n_regs would match more what we use
>>>>>>> elsewhere, but is not a must have).
>>>>>>>
>>>>>> This is read function for user and returns only list of reg
>>>>>> offsets, I did not understand correctly !
>>>>>>>> +
>>>>>>>> + r = copy_to_user(buf + len, "\n", 1);
>>>>>>>> +
>>>>>>>> + if (r)
>>>>>>>> + return -EFAULT;
>>>>>>>> +
>>>>>>>> + len++;
>>>>>>>> + *pos += len;
>>>>>>>> +
>>>>>>>> + return len;
>>>>>>>> +}
>>>>>>>> +
>>>>>>>> +static ssize_t amdgpu_reset_dump_register_list_write(struct
>>>>>>>> file *f,
>>>>>>>> + const char __user *buf, size_t size, loff_t *pos)
>>>>>>>> +{
>>>>>>>> + struct amdgpu_device *adev = (struct amdgpu_device
>>>>>>>> *)file_inode(f)->i_private;
>>>>>>>> + char *reg_offset, *reg, reg_temp[11];
>>>>>>>> + static int alloc_count;
>>>>>>>> + int ret, i = 0, len = 0;
>>>>>>>> +
>>>>>>>> + do {
>>>>>>>> + reg_offset = reg_temp;
>>>>>>>> + memset(reg_offset, 0, 11);
>>>>>>>> + ret = copy_from_user(reg_offset, buf + len, min(11,
>>>>>>>> ((int)size-len)));
>>>>>>>> +
>>>>>>>> + if (ret)
>>>>>>>> + goto failed;
>>>>>>>> +
>>>>>>>> + reg = strsep(®_offset, " ");
>>>>>>>> +
>>>>>>>> + if (alloc_count <= i) {
>>>>>>>
>>>>>>>> + adev->reset_dump_reg_list = krealloc_array(
>>>>>>>> + adev->reset_dump_reg_list, 1,
>>>>>>>> + sizeof(uint32_t), GFP_KERNEL);
>>>>>>>> + alloc_count++;
>>>>>>>> + }
>>>>>>>> +
>>>>>>>> + ret = kstrtouint(reg, 16, &adev->reset_dump_reg_list[i]);
>>>>>>>
>>>>>>> This here is modifying adev->reset_dump_reg_list as well and so
>>>>>>> must be protected by a lock as well.
>>>>>>>
>>>>>>> The tricky part is that we can't allocate memory while holding
>>>>>>> this lock (because we need it during reset as well).
>>>>>>>
>>>>>>> One solution for this is to read the register list into a local
>>>>>>> array first and when that's done swap the local array with the
>>>>>>> one in adev->reset_dump_reg_list while holding the lock.
>>>>>>>
>>>> krealloc_array should be inside lock or outside lock? this may be
>>>> problem.
>>>>
>>>
>>> This *must* be outside the lock because we need to take the lock
>>> during GPU reset which has a dependency to not allocate memory or
>>> wait for locks under which memory is allocated.
>>>
>>> That's why I said you need an approach which first parses the string
>>> from userspace, build up the register list and then swap that with
>>> the existing one while holding the lock.
>>>
>>
>> Another approach would be to just protect debugfs write with
>> down_read(&adev->reset_sem) or reset domain semaphore.
>
> No, exactly that doesn't work.
>
> See the down_write(&adev->reset_sem) would then wait for this reader and
> the reader is allocating memory and allocating memory might wait for the
> reset to finish => deadlock.
I didn't get this part - allocating memory might wait for the reset to
finish.
down_write() is called as one of the first steps during device reset,
and therefore device reset hasn't started. When you say " reset to
finish", do you mean device reset or something else?
Thanks,
Lijo
>
> Regards,
> Christian.
>
>>
>> Other than that if apps are trying to read and modify the list at the
>> same time, probably we should leave that to user mode since this is
>> mainly a debug feature.
>>
>> Thanks,
>> Lijo
>>
>>> Regards,
>>> Christian.
>>>
>>>> Regards,
>>>>
>>>> S.Amarnath
>>>>
>>>>>>> Regards,
>>>>>>> Christian.
>>>>>>>
>>>>>> There are 2 situations:
>>>>>> 1st time creating list n_regs will be 0 and trace event will not
>>>>>> be triggered
>>>>>> 2nd time while updating list n_regs is already set and
>>>>>> adev->reset_dump_reg_list will have some offsets address
>>>>>> (hypothetically speaking *during reset + update* read values from
>>>>>> RREG32 will mix up of old list and new list)
>>>>>> its only critical when its freed and n_regs is not 0
>>>>>
>>>>> No, that won't work like this. See you *must* always hold a lock
>>>>> when reading or writing the array.
>>>>>
>>>>> Otherwise it is perfectly possible that one thread sees only halve
>>>>> of the updates of another thread.
>>>>>
>>>>> The only alternative would be RCU, atomic replace and manual
>>>>> barrier handling, but that would be complete overkill for that
>>>>> feature.
>>>>>
>>>>> Regards,
>>>>> Christian.
>>>>>
>>>>>>
>>>>>> Regards,
>>>>>> S.Amarnath
>>>>>>>> +
>>>>>>>> + if (ret)
>>>>>>>> + goto failed;
>>>>>>>> +
>>>>>>>> + len += strlen(reg) + 1;
>>>>>>>> + i++;
>>>>>>>> +
>>>>>>>> + } while (len < size);
>>>>>>>> +
>>>>>>>> + adev->n_regs = i;
>>>>>>>> +
>>>>>>>> + return size;
>>>>>>>> +
>>>>>>>> +failed:
>>>>>>>> + mutex_lock(&adev->reset_dump_mutex);
>>>>>>>> + kfree(adev->reset_dump_reg_list);
>>>>>>>> + adev->reset_dump_reg_list = NULL;
>>>>>>>> + alloc_count = 0;
>>>>>>>> + adev->n_regs = 0;
>>>>>>>> + mutex_unlock(&adev->reset_dump_mutex);
>>>>>>>> + return -EFAULT;
>>>>>>>> +}
>>>>>>>> +
>>>>>>>> +
>>>>>>>> +
>>>>>>>> +static const struct file_operations
>>>>>>>> amdgpu_reset_dump_register_list = {
>>>>>>>> + .owner = THIS_MODULE,
>>>>>>>> + .read = amdgpu_reset_dump_register_list_read,
>>>>>>>> + .write = amdgpu_reset_dump_register_list_write,
>>>>>>>> + .llseek = default_llseek
>>>>>>>> +};
>>>>>>>> +
>>>>>>>> int amdgpu_debugfs_init(struct amdgpu_device *adev)
>>>>>>>> {
>>>>>>>> struct dentry *root =
>>>>>>>> adev_to_drm(adev)->primary->debugfs_root;
>>>>>>>> @@ -1618,6 +1710,7 @@ int amdgpu_debugfs_init(struct
>>>>>>>> amdgpu_device *adev)
>>>>>>>> if (!debugfs_initialized())
>>>>>>>> return 0;
>>>>>>>> + mutex_init(&adev->reset_dump_mutex);
>>>>>>>> ent = debugfs_create_file("amdgpu_preempt_ib", 0600, root,
>>>>>>>> adev,
>>>>>>>> &fops_ib_preempt);
>>>>>>>> if (IS_ERR(ent)) {
>>>>>>>> @@ -1672,6 +1765,8 @@ int amdgpu_debugfs_init(struct
>>>>>>>> amdgpu_device *adev)
>>>>>>>> &amdgpu_debugfs_test_ib_fops);
>>>>>>>> debugfs_create_file("amdgpu_vm_info", 0444, root, adev,
>>>>>>>> &amdgpu_debugfs_vm_info_fops);
>>>>>>>> + debugfs_create_file("amdgpu_reset_dump_register_list", 0644,
>>>>>>>> root, adev,
>>>>>>>> + &amdgpu_reset_dump_register_list);
>>>>>>>> adev->debugfs_vbios_blob.data = adev->bios;
>>>>>>>> adev->debugfs_vbios_blob.size = adev->bios_size;
>>>>>>>
>>>>>
>>>
>
More information about the amd-gfx
mailing list