[PATCH 3/3] drm/amd/amdkfd: Surface files in Sysfs to allow users to get number of compute units that are in use.
Russell, Kent
Kent.Russell at amd.com
Mon Sep 28 10:54:39 UTC 2020
[AMD Public Use]
Some minor things inline
> -----Original Message-----
> From: amd-gfx <amd-gfx-bounces at lists.freedesktop.org> On Behalf Of Ramesh Errabolu
> Sent: Friday, September 25, 2020 6:03 PM
> To: amd-gfx at lists.freedesktop.org
> Cc: Errabolu, Ramesh <Ramesh.Errabolu at amd.com>
> Subject: [PATCH 3/3] drm/amd/amdkfd: Surface files in Sysfs to allow users to get number
> of compute units that are in use.
>
> [Why]
> Allow user to know how many compute units (CU) are in use at any given
> moment.
>
> [How]
> Surface files in Sysfs that allow user to determine the number of compute
> units that are in use for a given process. One Sysfs file is used per
> device.
>
> Signed-off-by: Ramesh Errabolu <Ramesh.Errabolu at amd.com>
> ---
> drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 25 +++++++++
> drivers/gpu/drm/amd/amdkfd/kfd_process.c | 68 +++++++++++++++++++++++-
> 2 files changed, 92 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> index 56f92cfff591..3df2b9936458 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> @@ -705,6 +705,31 @@ struct kfd_process_device {
>
> struct kobject *kobj_stats;
> unsigned int doorbell_index;
> +
> + /*
> + * @cu_occupancy: Reports occupancy of Compute Units (CU) of a process
> + * that is associated with device encoded by "this" struct instance. The
> + * value reflects CU usage by all of the waves launched by this process
> + * on this device. A very important property of occupancy parameter is
> + * that its value is a a snapshot of current use.
is a snapshot
> + *
> + * Following is to be noted regarding how this parameter is reported:
> + *
> + * The number of waves that a CU can launch is limited by couple of
> + * parameters. These are encoded by struct amdgpu_cu_info instance
> + * that is part of every device definition. For GFX9 devices this
> + * translates to 40 waves (simd_per_cu * max_waves_per_simd) when waves
> + * do not use scratch memory and 32 waves (max_scratch_slots_per_cu)
> + * when they use. This could change for future devices and therefore
when they do use scratch memory
> + * this example should be considered as a guide.
> + *
> + * All CU's of a device are available for the process. This may not be true
> + * under certain conditions - e.g. CU masking.
> + *
> + * Finally number of CU's that are occupied by a process is affected by both
> + * number of CU's a device has along with number of other competing processes
> + */
> + struct attribute attr_cu_occupancy;
> };
>
> #define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd)
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
> b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
> index 17d909c86f50..26b716b5eb23 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
> @@ -249,6 +249,52 @@ static void kfd_sdma_activity_worker(struct work_struct *work)
> }
> }
>
> +/**
> + * @kfd_get_cu_occupancy() - Collect number of waves in-flight on this device
> + * by current process. Translates acquired wave count into number of compute units
> + * that are occupied.
> + *
> + * @atr: Handle of attribute that allows reporting of wave count. The attribute
> + * handle encapsulates GPU device it is associated with, thereby allowing collection
> + * of waves in flight, etc
> + *
> + * @buffer: Handle of user provided buffer updated with wave count
> + *
> + * Return: Number of bytes written to user buffer or an error value
> + */
> +static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer)
> +{
> + int cu_cnt;
> + int wave_cnt;
> + int max_waves_per_cu;
> + struct kfd_dev *dev = NULL;
> + struct kfd_process *proc = NULL;
> + struct kfd_process_device *pdd = NULL;
> +
> + pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy);
> + dev = pdd->dev;
> + if (dev->kfd2kgd->get_cu_occupancy == NULL)
> + return -EINVAL;
> +
> + cu_cnt = 0;
> + proc = pdd->process;
> + if (pdd->qpd.queue_count == 0) {
> + pr_debug("Gpu-Id: %d has no active queues for process %d\n",
> + dev->id, proc->pasid);
> + return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
> + }
> +
> + /* Collect wave count from device if it supports */
> + wave_cnt = 0;
> + max_waves_per_cu = 0;
> + dev->kfd2kgd->get_cu_occupancy(dev->kgd, proc->pasid, &wave_cnt,
> + &max_waves_per_cu);
> +
> + /* Translate wave count to number of compute units */
> + cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu;
> + return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
> +}
> +
> static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr,
> char *buffer)
> {
> @@ -344,6 +390,7 @@ static ssize_t kfd_procfs_queue_show(struct kobject *kobj,
>
> return 0;
> }
> +
> static ssize_t kfd_procfs_stats_show(struct kobject *kobj,
> struct attribute *attr, char *buffer)
> {
> @@ -359,6 +406,10 @@ static ssize_t kfd_procfs_stats_show(struct kobject *kobj,
> PAGE_SIZE,
> "%llu\n",
> jiffies64_to_msecs(evict_jiffies));
> +
> + /* Sysfs handle that gets CU occupancy is per device */
> + } else if (strcmp(attr->name, "cu_occupancy") == 0) {
> + return kfd_get_cu_occupancy(attr, buffer);
> } else
You should use } else { here, even for a single line, see https://www.kernel.org/doc/html/v4.10/process/coding-style.html right above section 3.1. Annoyingly, checkpatch doesn't catch this.
> pr_err("Invalid attribute");
>
> @@ -466,6 +517,7 @@ static int kfd_procfs_add_sysfs_stats(struct kfd_process *p)
> * Create sysfs files for each GPU:
> * - proc/<pid>/stats_<gpuid>/
> * - proc/<pid>/stats_<gpuid>/evicted_ms
> + * - proc/<pid>/stats_<gpuid>/cu_occupancy
> */
> list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
> struct kobject *kobj_stats;
> @@ -496,6 +548,19 @@ static int kfd_procfs_add_sysfs_stats(struct kfd_process *p)
> if (ret)
> pr_warn("Creating eviction stats for gpuid %d failed",
> (int)pdd->dev->id);
> +
> + /* Add sysfs file to report compute unit occupancy */
> + if (pdd->dev->kfd2kgd->get_cu_occupancy != NULL) {
> + pdd->attr_cu_occupancy.name = "cu_occupancy";
> + pdd->attr_cu_occupancy.mode = KFD_SYSFS_FILE_MODE;
> + sysfs_attr_init(&pdd->attr_cu_occupancy);
> + ret = sysfs_create_file(kobj_stats,
> + &pdd->attr_cu_occupancy);
> + if (ret)
> + pr_warn("Creating %s failed for gpuid: %d",
> + pdd->attr_cu_occupancy.name,
> + (int)pdd->dev->id);
> + }
> }
> err:
> return ret;
> @@ -537,7 +602,6 @@ static int kfd_procfs_add_sysfs_files(struct kfd_process *p)
> return ret;
> }
>
> -
> void kfd_procfs_del_queue(struct queue *q)
> {
> if (!q)
> @@ -909,6 +973,8 @@ static void kfd_process_wq_release(struct work_struct *work)
> sysfs_remove_file(p->kobj, &pdd->attr_vram);
> sysfs_remove_file(p->kobj, &pdd->attr_sdma);
> sysfs_remove_file(p->kobj, &pdd->attr_evict);
> + if (pdd->dev->kfd2kgd->get_cu_occupancy != NULL)
> + sysfs_remove_file(p->kobj, &pdd->attr_cu_occupancy);
> kobject_del(pdd->kobj_stats);
> kobject_put(pdd->kobj_stats);
> pdd->kobj_stats = NULL;
> --
> 2.27.0
>
> _______________________________________________
> amd-gfx mailing list
> amd-gfx at lists.freedesktop.org
> https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.freedesktop.or
> g%2Fmailman%2Flistinfo%2Famd-
> gfx&data=02%7C01%7Ckent.russell%40amd.com%7C4ed773ec69b04a9d2ee308d8619
> ee024%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637366682305777071&
> ;sdata=1NPpCMCB8dfWGwIokNotPm1pJU4AMkeurhcIe0y%2FIaw%3D&reserved=0
More information about the amd-gfx
mailing list