[PATCH RFC v4 14/16] drm, cgroup: Introduce lgpu as DRM cgroup resource

Kuehling, Felix Felix.Kuehling at amd.com
Tue Oct 8 18:53:18 UTC 2019


On 2019-08-29 2:05 a.m., Kenny Ho wrote:
> drm.lgpu
>          A read-write nested-keyed file which exists on all cgroups.
>          Each entry is keyed by the DRM device's major:minor.
>
>          lgpu stands for logical GPU, it is an abstraction used to
>          subdivide a physical DRM device for the purpose of resource
>          management.
>
>          The lgpu is a discrete quantity that is device specific (i.e.
>          some DRM devices may have 64 lgpus while others may have 100
>          lgpus.)  The lgpu is a single quantity with two representations
>          denoted by the following nested keys.
>
>            =====     ========================================
>            count     Representing lgpu as anonymous resource
>            list      Representing lgpu as named resource
>            =====     ========================================
>
>          For example:
>          226:0 count=256 list=0-255
>          226:1 count=4 list=0,2,4,6
>          226:2 count=32 list=32-63
>
>          lgpu is represented by a bitmap and uses the bitmap_parselist
>          kernel function so the list key input format is a
>          comma-separated list of decimal numbers and ranges.
>
>          Consecutively set bits are shown as two hyphen-separated decimal
>          numbers, the smallest and largest bit numbers set in the range.
>          Optionally each range can be postfixed to denote that only parts
>          of it should be set.  The range will divided to groups of
>          specific size.
>          Syntax: range:used_size/group_size
>          Example: 0-1023:2/256 ==> 0,1,256,257,512,513,768,769
>
>          The count key is the hamming weight / hweight of the bitmap.
>
>          Both count and list accept the max and default keywords.
>
>          Some DRM devices may only support lgpu as anonymous resources.
>          In such case, the significance of the position of the set bits
>          in list will be ignored.
>
>          This lgpu resource supports the 'allocation' resource
>          distribution model.
>
> Change-Id: I1afcacf356770930c7f925df043e51ad06ceb98e
> Signed-off-by: Kenny Ho <Kenny.Ho at amd.com>

The description sounds reasonable to me and maps well to the CU masking 
feature in our GPUs.

It would also allow us to do more coarse-grained masking for example to 
guarantee balanced allocation of CUs across shader engines or 
partitioning of memory bandwidth or CP pipes (if that is supported by 
the hardware/firmware).

I can't comment on the code as I'm unfamiliar with the details of the 
cgroup code.

Acked-by: Felix Kuehling <Felix.Kuehling at amd.com>


> ---
>   Documentation/admin-guide/cgroup-v2.rst |  46 ++++++++
>   include/drm/drm_cgroup.h                |   4 +
>   include/linux/cgroup_drm.h              |   6 ++
>   kernel/cgroup/drm.c                     | 135 ++++++++++++++++++++++++
>   4 files changed, 191 insertions(+)
>
> diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst
> index 87a195133eaa..57f18469bd76 100644
> --- a/Documentation/admin-guide/cgroup-v2.rst
> +++ b/Documentation/admin-guide/cgroup-v2.rst
> @@ -1958,6 +1958,52 @@ DRM Interface Files
>   	Set largest allocation for /dev/dri/card1 to 4MB
>   	echo "226:1 4m" > drm.buffer.peak.max
>   
> +  drm.lgpu
> +	A read-write nested-keyed file which exists on all cgroups.
> +	Each entry is keyed by the DRM device's major:minor.
> +
> +	lgpu stands for logical GPU, it is an abstraction used to
> +	subdivide a physical DRM device for the purpose of resource
> +	management.
> +
> +	The lgpu is a discrete quantity that is device specific (i.e.
> +	some DRM devices may have 64 lgpus while others may have 100
> +	lgpus.)  The lgpu is a single quantity with two representations
> +	denoted by the following nested keys.
> +
> +	  =====     ========================================
> +	  count     Representing lgpu as anonymous resource
> +	  list      Representing lgpu as named resource
> +	  =====     ========================================
> +
> +	For example:
> +	226:0 count=256 list=0-255
> +	226:1 count=4 list=0,2,4,6
> +	226:2 count=32 list=32-63
> +
> +	lgpu is represented by a bitmap and uses the bitmap_parselist
> +	kernel function so the list key input format is a
> +	comma-separated list of decimal numbers and ranges.
> +
> +	Consecutively set bits are shown as two hyphen-separated decimal
> +	numbers, the smallest and largest bit numbers set in the range.
> +	Optionally each range can be postfixed to denote that only parts
> +	of it should be set.  The range will divided to groups of
> +	specific size.
> +	Syntax: range:used_size/group_size
> +	Example: 0-1023:2/256 ==> 0,1,256,257,512,513,768,769
> +
> +	The count key is the hamming weight / hweight of the bitmap.
> +
> +	Both count and list accept the max and default keywords.
> +
> +	Some DRM devices may only support lgpu as anonymous resources.
> +	In such case, the significance of the position of the set bits
> +	in list will be ignored.
> +
> +	This lgpu resource supports the 'allocation' resource
> +	distribution model.
> +
>   GEM Buffer Ownership
>   ~~~~~~~~~~~~~~~~~~~~
>   
> diff --git a/include/drm/drm_cgroup.h b/include/drm/drm_cgroup.h
> index 6d9707e1eb72..a8d6be0b075b 100644
> --- a/include/drm/drm_cgroup.h
> +++ b/include/drm/drm_cgroup.h
> @@ -6,6 +6,7 @@
>   
>   #include <linux/cgroup_drm.h>
>   #include <linux/workqueue.h>
> +#include <linux/types.h>
>   #include <drm/ttm/ttm_bo_api.h>
>   #include <drm/ttm/ttm_bo_driver.h>
>   
> @@ -28,6 +29,9 @@ struct drmcg_props {
>   	s64			mem_highs_default[TTM_PL_PRIV+1];
>   
>   	struct work_struct	*mem_reclaim_wq[TTM_PL_PRIV];
> +
> +	int			lgpu_capacity;
> +        DECLARE_BITMAP(lgpu_slots, MAX_DRMCG_LGPU_CAPACITY);
>   };
>   
>   #ifdef CONFIG_CGROUP_DRM
> diff --git a/include/linux/cgroup_drm.h b/include/linux/cgroup_drm.h
> index c56cfe74d1a6..7b1cfc4ce4c3 100644
> --- a/include/linux/cgroup_drm.h
> +++ b/include/linux/cgroup_drm.h
> @@ -14,6 +14,8 @@
>   /* limit defined per the way drm_minor_alloc operates */
>   #define MAX_DRM_DEV (64 * DRM_MINOR_RENDER)
>   
> +#define MAX_DRMCG_LGPU_CAPACITY 256
> +
>   enum drmcg_mem_bw_attr {
>   	DRMCG_MEM_BW_ATTR_BYTE_MOVED, /* for calulating 'instantaneous' bw */
>   	DRMCG_MEM_BW_ATTR_ACCUM_US,  /* for calulating 'instantaneous' bw */
> @@ -32,6 +34,7 @@ enum drmcg_res_type {
>   	DRMCG_TYPE_MEM_PEAK,
>   	DRMCG_TYPE_BANDWIDTH,
>   	DRMCG_TYPE_BANDWIDTH_PERIOD_BURST,
> +	DRMCG_TYPE_LGPU,
>   	__DRMCG_TYPE_LAST,
>   };
>   
> @@ -58,6 +61,9 @@ struct drmcg_device_resource {
>   	s64			mem_bw_stats[__DRMCG_MEM_BW_ATTR_LAST];
>   	s64			mem_bw_limits_bytes_in_period;
>   	s64			mem_bw_limits_avg_bytes_per_us;
> +
> +	s64			lgpu_used;
> +	DECLARE_BITMAP(lgpu_allocated, MAX_DRMCG_LGPU_CAPACITY);
>   };
>   
>   /**
> diff --git a/kernel/cgroup/drm.c b/kernel/cgroup/drm.c
> index 0ea7f0619e25..18c4368e2c29 100644
> --- a/kernel/cgroup/drm.c
> +++ b/kernel/cgroup/drm.c
> @@ -9,6 +9,7 @@
>   #include <linux/cgroup_drm.h>
>   #include <linux/ktime.h>
>   #include <linux/kernel.h>
> +#include <linux/bitmap.h>
>   #include <drm/drm_file.h>
>   #include <drm/drm_drv.h>
>   #include <drm/ttm/ttm_bo_api.h>
> @@ -52,6 +53,9 @@ static char const *mem_bw_attr_names[] = {
>   #define MEM_BW_LIMITS_NAME_AVG "avg_bytes_per_us"
>   #define MEM_BW_LIMITS_NAME_BURST "bytes_in_period"
>   
> +#define LGPU_LIMITS_NAME_LIST "list"
> +#define LGPU_LIMITS_NAME_COUNT "count"
> +
>   static struct drmcg *root_drmcg __read_mostly;
>   
>   static int drmcg_css_free_fn(int id, void *ptr, void *data)
> @@ -115,6 +119,10 @@ static inline int init_drmcg_single(struct drmcg *drmcg, struct drm_device *dev)
>   	for (i = 0; i <= TTM_PL_PRIV; i++)
>   		ddr->mem_highs[i] = dev->drmcg_props.mem_highs_default[i];
>   
> +	bitmap_copy(ddr->lgpu_allocated, dev->drmcg_props.lgpu_slots,
> +			MAX_DRMCG_LGPU_CAPACITY);
> +	ddr->lgpu_used = bitmap_weight(ddr->lgpu_allocated, MAX_DRMCG_LGPU_CAPACITY);
> +
>   	mutex_unlock(&dev->drmcg_mutex);
>   	return 0;
>   }
> @@ -280,6 +288,14 @@ static void drmcg_print_limits(struct drmcg_device_resource *ddr,
>   				MEM_BW_LIMITS_NAME_AVG,
>   				ddr->mem_bw_limits_avg_bytes_per_us);
>   		break;
> +	case DRMCG_TYPE_LGPU:
> +		seq_printf(sf, "%s=%lld %s=%*pbl\n",
> +				LGPU_LIMITS_NAME_COUNT,
> +				ddr->lgpu_used,
> +				LGPU_LIMITS_NAME_LIST,
> +				dev->drmcg_props.lgpu_capacity,
> +				ddr->lgpu_allocated);
> +		break;
>   	default:
>   		seq_puts(sf, "\n");
>   		break;
> @@ -314,6 +330,15 @@ static void drmcg_print_default(struct drmcg_props *props,
>   				MEM_BW_LIMITS_NAME_AVG,
>   				props->mem_bw_avg_bytes_per_us_default);
>   		break;
> +	case DRMCG_TYPE_LGPU:
> +		seq_printf(sf, "%s=%d %s=%*pbl\n",
> +				LGPU_LIMITS_NAME_COUNT,
> +				bitmap_weight(props->lgpu_slots,
> +					props->lgpu_capacity),
> +				LGPU_LIMITS_NAME_LIST,
> +				props->lgpu_capacity,
> +				props->lgpu_slots);
> +		break;
>   	default:
>   		seq_puts(sf, "\n");
>   		break;
> @@ -407,9 +432,21 @@ static void drmcg_value_apply(struct drm_device *dev, s64 *dst, s64 val)
>   	mutex_unlock(&dev->drmcg_mutex);
>   }
>   
> +static void drmcg_lgpu_values_apply(struct drm_device *dev,
> +		struct drmcg_device_resource *ddr, unsigned long *val)
> +{
> +
> +	mutex_lock(&dev->drmcg_mutex);
> +	bitmap_copy(ddr->lgpu_allocated, val, MAX_DRMCG_LGPU_CAPACITY);
> +	ddr->lgpu_used = bitmap_weight(ddr->lgpu_allocated, MAX_DRMCG_LGPU_CAPACITY);
> +	mutex_unlock(&dev->drmcg_mutex);
> +}
> +
>   static void drmcg_nested_limit_parse(struct kernfs_open_file *of,
>   		struct drm_device *dev, char *attrs)
>   {
> +	DECLARE_BITMAP(tmp_bitmap, MAX_DRMCG_LGPU_CAPACITY);
> +	DECLARE_BITMAP(chk_bitmap, MAX_DRMCG_LGPU_CAPACITY);
>   	enum drmcg_res_type type =
>   		DRMCG_CTF_PRIV2RESTYPE(of_cft(of)->private);
>   	struct drmcg *drmcg = css_to_drmcg(of_css(of));
> @@ -501,6 +538,83 @@ static void drmcg_nested_limit_parse(struct kernfs_open_file *of,
>   				continue;
>   			}
>   			break; /* DRMCG_TYPE_MEM */
> +		case DRMCG_TYPE_LGPU:
> +			if (strncmp(sname, LGPU_LIMITS_NAME_LIST, 256) &&
> +				strncmp(sname, LGPU_LIMITS_NAME_COUNT, 256) )
> +				continue;
> +
> +                        if (!strcmp("max", sval) ||
> +					!strcmp("default", sval)) {
> +				if (parent != NULL)
> +					drmcg_lgpu_values_apply(dev, ddr,
> +						parent->dev_resources[minor]->
> +						lgpu_allocated);
> +				else
> +					drmcg_lgpu_values_apply(dev, ddr,
> +						props->lgpu_slots);
> +
> +				continue;
> +			}
> +
> +			if (strncmp(sname, LGPU_LIMITS_NAME_COUNT, 256) == 0) {
> +				p_max = parent == NULL ? props->lgpu_capacity:
> +					bitmap_weight(
> +					parent->dev_resources[minor]->
> +					lgpu_allocated, props->lgpu_capacity);
> +
> +				rc = drmcg_process_limit_s64_val(sval,
> +					false, p_max, p_max, &val);
> +
> +				if (rc || val < 0) {
> +					drmcg_pr_cft_err(drmcg, rc, cft_name,
> +							minor);
> +					continue;
> +				}
> +
> +				bitmap_zero(tmp_bitmap,
> +						MAX_DRMCG_LGPU_CAPACITY);
> +				bitmap_set(tmp_bitmap, 0, val);
> +			}
> +
> +			if (strncmp(sname, LGPU_LIMITS_NAME_LIST, 256) == 0) {
> +				rc = bitmap_parselist(sval, tmp_bitmap,
> +						MAX_DRMCG_LGPU_CAPACITY);
> +
> +				if (rc) {
> +					drmcg_pr_cft_err(drmcg, rc, cft_name,
> +							minor);
> +					continue;
> +				}
> +
> +                        	bitmap_andnot(chk_bitmap, tmp_bitmap,
> +					props->lgpu_slots,
> +					MAX_DRMCG_LGPU_CAPACITY);
> +
> +                        	if (!bitmap_empty(chk_bitmap,
> +						MAX_DRMCG_LGPU_CAPACITY)) {
> +					drmcg_pr_cft_err(drmcg, 0, cft_name,
> +							minor);
> +					continue;
> +				}
> +			}
> +
> +
> +                        if (parent != NULL) {
> +				bitmap_and(chk_bitmap, tmp_bitmap,
> +				parent->dev_resources[minor]->lgpu_allocated,
> +				props->lgpu_capacity);
> +
> +				if (bitmap_empty(chk_bitmap,
> +						props->lgpu_capacity)) {
> +					drmcg_pr_cft_err(drmcg, 0,
> +							cft_name, minor);
> +					continue;
> +				}
> +			}
> +
> +			drmcg_lgpu_values_apply(dev, ddr, tmp_bitmap);
> +
> +			break; /* DRMCG_TYPE_LGPU */
>   		default:
>   			break;
>   		} /* switch (type) */
> @@ -606,6 +720,7 @@ static ssize_t drmcg_limit_write(struct kernfs_open_file *of, char *buf,
>   			break;
>   		case DRMCG_TYPE_BANDWIDTH:
>   		case DRMCG_TYPE_MEM:
> +		case DRMCG_TYPE_LGPU:
>   			drmcg_nested_limit_parse(of, dm->dev, sattr);
>   			break;
>   		default:
> @@ -731,6 +846,20 @@ struct cftype files[] = {
>   		.private = DRMCG_CTF_PRIV(DRMCG_TYPE_BANDWIDTH,
>   						DRMCG_FTYPE_DEFAULT),
>   	},
> +	{
> +		.name = "lgpu",
> +		.seq_show = drmcg_seq_show,
> +		.write = drmcg_limit_write,
> +		.private = DRMCG_CTF_PRIV(DRMCG_TYPE_LGPU,
> +						DRMCG_FTYPE_LIMIT),
> +	},
> +	{
> +		.name = "lgpu.default",
> +		.seq_show = drmcg_seq_show,
> +		.flags = CFTYPE_ONLY_ON_ROOT,
> +		.private = DRMCG_CTF_PRIV(DRMCG_TYPE_LGPU,
> +						DRMCG_FTYPE_DEFAULT),
> +	},
>   	{ }	/* terminate */
>   };
>   
> @@ -744,6 +873,10 @@ struct cgroup_subsys drm_cgrp_subsys = {
>   
>   static inline void drmcg_update_cg_tree(struct drm_device *dev)
>   {
> +        bitmap_zero(dev->drmcg_props.lgpu_slots, MAX_DRMCG_LGPU_CAPACITY);
> +        bitmap_fill(dev->drmcg_props.lgpu_slots,
> +			dev->drmcg_props.lgpu_capacity);
> +
>   	/* init cgroups created before registration (i.e. root cgroup) */
>   	if (root_drmcg != NULL) {
>   		struct cgroup_subsys_state *pos;
> @@ -800,6 +933,8 @@ void drmcg_device_early_init(struct drm_device *dev)
>   	for (i = 0; i <= TTM_PL_PRIV; i++)
>   		dev->drmcg_props.mem_highs_default[i] = S64_MAX;
>   
> +	dev->drmcg_props.lgpu_capacity = MAX_DRMCG_LGPU_CAPACITY;
> +
>   	drmcg_update_cg_tree(dev);
>   }
>   EXPORT_SYMBOL(drmcg_device_early_init);


More information about the dri-devel mailing list