[PATCH v2 1/1] drm/xe/eustall: Add support for EU stall sampling
Souza, Jose
jose.souza at intel.com
Fri Jul 19 20:21:43 UTC 2024
On Sun, 2024-07-07 at 15:41 -0700, Ashutosh Dixit wrote:
> From: Harish Chegondi <harish.chegondi at intel.com>
>
> A new hardware feature first introduced in PVC gives capability to
> periodically sample EU stall state and record counts for different stall
> reasons, on a per IP basis, aggregate across all EUs in a subslice and
> record the samples in a buffer in each subslice. Eventually, the aggregated
> data is written out to a buffer in the memory. This feature is also
> supported in XE2 architecture GPUs - LNL and BMG.
>
> Use an existing IOCTL DRM_IOCTL_XE_OBSERVATION as interface into the driver
> from the user space to do initial setup and obtain a file descriptor for
> the EU stall counter data stream. Input parameter to the IOCTL is a struct
> drm_xe_observation_param in which observation_type should be set to
> DRM_XE_OBSERVATION_TYPE_EU_STALL, observation_op should be
> DRM_XE_OBSERVATION_OP_STREAM_OPEN and param should point to a chain of
> drm_xe_ext_set_property structures in which each structure has a pair of
> property and value. The EU stall sampling input properties are defined in
> drm_xe_eu_stall_property_id enum.
>
> With the file descriptor obtained from DRM_IOCTL_XE_OBSERVATION, user space
> can enable and disable EU stall sampling with
> DRM_XE_OBSERVATION_IOCTL_ENABLE and DRM_XE_OBSERVATION_IOCTL_DISABLE
> IOCTLs. User space can also call poll() to check for availability of
> data. The data can be read with read(). EU stall data consists of header
> and data pairs. The header format is defined in struct
> drm_xe_eu_stall_data_header. If the user space doesn't read the EU stall
> data fast enough, it is possible that the EU stall data buffer can get
> filled up and if the hardware wants to write data, it simply drops data due
> to unavailable buffer space. In that case hardware sets a bit in a
> register. The driver sets a flag in the EU stall data header flags field
> to let the user space know that the hardware has dropped data.
>
> v2: Rename xe perf layer as xe observation layer (Ashutosh)
>
> Signed-off-by: Harish Chegondi <harish.chegondi at intel.com>
> Signed-off-by: Ashutosh Dixit <ashutosh.dixit at intel.com>
> ---
> drivers/gpu/drm/xe/Makefile | 1 +
> drivers/gpu/drm/xe/regs/xe_eu_stall_regs.h | 33 +
> drivers/gpu/drm/xe/xe_eustall_cntr.c | 1005 ++++++++++++++++++++
> drivers/gpu/drm/xe/xe_eustall_cntr.h | 62 ++
> drivers/gpu/drm/xe/xe_gt.c | 3 +
> drivers/gpu/drm/xe/xe_gt_topology.c | 9 +
> drivers/gpu/drm/xe/xe_gt_topology.h | 3 +
> drivers/gpu/drm/xe/xe_gt_types.h | 4 +
> drivers/gpu/drm/xe/xe_observation.c | 14 +
> drivers/gpu/drm/xe/xe_trace.h | 35 +
> include/uapi/drm/xe_drm.h | 77 ++
> 11 files changed, 1246 insertions(+)
> create mode 100644 drivers/gpu/drm/xe/regs/xe_eu_stall_regs.h
> create mode 100644 drivers/gpu/drm/xe/xe_eustall_cntr.c
> create mode 100644 drivers/gpu/drm/xe/xe_eustall_cntr.h
>
> diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile
> index 0eb0acc4f198..13d935ac40b3 100644
> --- a/drivers/gpu/drm/xe/Makefile
> +++ b/drivers/gpu/drm/xe/Makefile
> @@ -54,6 +54,7 @@ xe-y += xe_bb.o \
> xe_device_sysfs.o \
> xe_dma_buf.o \
> xe_drm_client.o \
> + xe_eustall_cntr.o \
> xe_exec.o \
> xe_execlist.o \
> xe_exec_queue.o \
> diff --git a/drivers/gpu/drm/xe/regs/xe_eu_stall_regs.h b/drivers/gpu/drm/xe/regs/xe_eu_stall_regs.h
> new file mode 100644
> index 000000000000..c70f35f82cc5
> --- /dev/null
> +++ b/drivers/gpu/drm/xe/regs/xe_eu_stall_regs.h
> @@ -0,0 +1,33 @@
> +/* SPDX-License-Identifier: MIT */
> +/*
> + * Copyright © 2024 Intel Corporation
> + */
> +
> +#ifndef _XE_EU_STALL_REGS_H_
> +#define _XE_EU_STALL_REGS_H_
> +
> +#include "regs/xe_reg_defs.h"
> +
> +#define XEHPC_EUSTALL_BASE XE_REG_MCR(0xe520)
> +#define XEHPC_EUSTALL_BASE_BUF_ADDR REG_GENMASK(31, 6)
> +#define XEHPC_EUSTALL_BASE_DSS_BUF_SZ REG_GENMASK(5, 3)
> +#define XEHPC_EUSTALL_BASE_ENABLE_SAMPLING REG_BIT(1)
> +#define XEHPC_EUSTALL_BASE_EVICT_TDL_STALL_BUF REG_BIT(0)
> +
> +#define XEHPC_EUSTALL_BASE_UPPER XE_REG_MCR(0xe524)
> +
> +#define XEHPC_EUSTALL_REPORT XE_REG_MCR(0xe528)
> +#define XEHPC_EUSTALL_REPORT_WRITE_PTR_MASK REG_GENMASK(15, 2)
> +#define XEHPC_EUSTALL_REPORT_WRITE_PTR_SHIFT 2
> +#define XEHPC_EUSTALL_REPORT_OVERFLOW_DROP REG_BIT(1)
> +
> +#define XEHPC_EUSTALL_REPORT1 XE_REG_MCR(0xe52c)
> +#define XEHPC_EUSTALL_REPORT1_MASK_SHIFT 16
> +#define XEHPC_EUSTALL_REPORT1_READ_PTR_MASK REG_GENMASK(15, 2)
> +#define XEHPC_EUSTALL_REPORT1_READ_PTR_SHIFT 2
> +
> +#define XEHPC_EUSTALL_CTRL XE_REG_MCR(0xe53c)
> +#define EUSTALL_MOCS REG_GENMASK(9, 3)
> +#define EUSTALL_SAMPLE_RATE REG_GENMASK(2, 0)
> +
> +#endif
> diff --git a/drivers/gpu/drm/xe/xe_eustall_cntr.c b/drivers/gpu/drm/xe/xe_eustall_cntr.c
> new file mode 100644
> index 000000000000..183638f8e1e2
> --- /dev/null
> +++ b/drivers/gpu/drm/xe/xe_eustall_cntr.c
> @@ -0,0 +1,1005 @@
> +// SPDX-License-Identifier: MIT
> +/*
> + * Copyright © 2024 Intel Corporation
> + */
> +
> +#include <linux/anon_inodes.h>
> +#include <linux/nospec.h>
> +#include <linux/poll.h>
> +#include <drm/drm_drv.h>
> +#include "xe_gt.h"
> +#include "xe_bo.h"
> +#include "xe_pm.h"
> +#include "xe_trace.h"
> +#include "xe_device.h"
> +#include "xe_gt_mcr.h"
> +#include "xe_gt_topology.h"
> +#include "xe_eustall_cntr.h"
> +#include "xe_force_wake.h"
> +#include "regs/xe_gt_regs.h"
> +
> +#define CACHELINE_BYTES 64
> +#define DEFAULT_POLL_FREQUENCY_HZ 100
> +#define DEFAULT_POLL_PERIOD_NS (NSEC_PER_SEC / DEFAULT_POLL_FREQUENCY_HZ)
> +#define MISSING_CASE(x) WARN(1, "Missing case (%s == %ld)\n", \
> + __stringify(x), (long)(x))
> +
> +extern u32 xe_observation_paranoid;
> +
> +/**
> + * struct eu_stall_open_properties
> + *
> + * @eu_stall_sampling_rate: Hardware EU stall sampling rate.
> + * @event_report_count: Minimum no of EU stall data rows for poll to set POLLIN.
> + * @eu_stall_buf_sz: Per subslice EU stall data buffer size.
> + * @open_disabled: Should EU stall sampling be disabled at open.
> + * @poll_period: The period in nanoseconds at which the CPU will check for
> + * EU stall data in the buffer.
> + * @gt_id: GT ID of the GT on which EU stall data will be captured.
> + */
> +struct eu_stall_open_properties {
> + u8 eu_stall_sampling_rate;
> + u32 event_report_count;
> + u32 eu_stall_buf_sz;
> + bool open_disabled;
> + u64 poll_period;
> + u8 gt_id;
> +};
> +
> +/**
> + * num_data_rows - Return the number of EU stall data rows of 64B each
> + * for a given data size.
> + *
> + * @data_size: EU stall data size
> + */
> +static inline u32
> +num_data_rows(u32 data_size)
> +{
> + return (data_size >> 6);
> +}
> +
> +void xe_eustall_cntr_init(struct xe_gt *gt)
> +{
> + mutex_init(>->eu_stall_cntr.lock);
> +}
> +
> +static int set_prop_eu_stall_buffer_size(struct xe_device *xe, u64 value,
> + struct eu_stall_open_properties *props)
> +{
> + if (value != SZ_128K &&
> + value != SZ_256K &&
> + value != SZ_512K) {
> + drm_dbg(&xe->drm, "Invalid EU stall buffer size %llu\n", value);
> + return -EINVAL;
> + }
> + props->eu_stall_buf_sz = value;
> + return 0;
> +}
> +
> +static int set_prop_eu_stall_sampling_rate(struct xe_device *xe, u64 value,
> + struct eu_stall_open_properties *props)
> +{
> + if (value == 0 || value > 7) {
> + drm_dbg(&xe->drm, "Invalid EU stall sampling rate %llu\n", value);
> + return -EINVAL;
> + }
> + props->eu_stall_sampling_rate = value;
> + return 0;
> +}
> +
> +static int set_prop_eu_stall_poll_period(struct xe_device *xe, u64 value,
> + struct eu_stall_open_properties *props)
> +{
> + if (value < 100000 /* 100us */) {
> + drm_dbg(&xe->drm, "EU stall data poll period %lluns less than 100us\n", value);
> + return -EINVAL;
> + }
> + props->poll_period = value;
> + return 0;
> +}
> +
> +static int set_prop_eu_stall_event_report_count(struct xe_device *xe, u64 value,
> + struct eu_stall_open_properties *props)
> +{
> + if (value == 0) {
> + drm_dbg(&xe->drm, "Invalid EU stall poll event report count %llu\n", value);
> + return -EINVAL;
> + }
> + props->event_report_count = (u32)value;
> + return 0;
> +}
> +
> +static int set_prop_eu_stall_gt_id(struct xe_device *xe, u64 value,
> + struct eu_stall_open_properties *props)
> +{
> + if (value >= XE_MAX_GT_PER_TILE) {
> + drm_dbg(&xe->drm, "Invalid GT ID %llu for EU stall sampling\n", value);
> + return -EINVAL;
> + }
> + props->gt_id = (u8)value;
> + return 0;
> +}
> +
> +static int set_prop_eu_stall_open_disabled(struct xe_device *xe, u64 value,
> + struct eu_stall_open_properties *props)
> +{
> + props->open_disabled = value;
> + return 0;
> +}
> +
> +typedef int (*set_eu_stall_property_fn)(struct xe_device *xe, u64 value,
> + struct eu_stall_open_properties *props);
> +
> +static const set_eu_stall_property_fn xe_set_eu_stall_property_funcs[] = {
> + [DRM_XE_EU_STALL_PROP_BUF_SZ] = set_prop_eu_stall_buffer_size,
> + [DRM_XE_EU_STALL_PROP_SAMPLE_RATE] = set_prop_eu_stall_sampling_rate,
> + [DRM_XE_EU_STALL_PROP_POLL_PERIOD] = set_prop_eu_stall_poll_period,
> + [DRM_XE_EU_STALL_PROP_EVENT_REPORT_COUNT] = set_prop_eu_stall_event_report_count,
> + [DRM_XE_EU_STALL_PROP_GT_ID] = set_prop_eu_stall_gt_id,
> + [DRM_XE_EU_STALL_PROP_OPEN_DISABLED] = set_prop_eu_stall_open_disabled,
> +};
> +
> +static int xe_eu_stall_user_ext_set_property(struct xe_device *xe, u64 extension,
> + struct eu_stall_open_properties *props)
> +{
> + u64 __user *address = u64_to_user_ptr(extension);
> + struct drm_xe_ext_set_property ext;
> + int err;
> + u32 idx;
> +
> + err = __copy_from_user(&ext, address, sizeof(ext));
> + if (XE_IOCTL_DBG(xe, err))
> + return -EFAULT;
> +
> + if (XE_IOCTL_DBG(xe, ext.property >= ARRAY_SIZE(xe_set_eu_stall_property_funcs)) ||
> + XE_IOCTL_DBG(xe, ext.pad))
> + return -EINVAL;
> +
> + idx = array_index_nospec(ext.property, ARRAY_SIZE(xe_set_eu_stall_property_funcs));
> + return xe_set_eu_stall_property_funcs[idx](xe, ext.value, props);
> +}
> +
> +typedef int (*xe_eu_stall_user_extension_fn)(struct xe_device *xe, u64 extension,
> + struct eu_stall_open_properties *props);
> +static const xe_eu_stall_user_extension_fn xe_eu_stall_user_extension_funcs[] = {
> + [DRM_XE_EU_STALL_EXTENSION_SET_PROPERTY] = xe_eu_stall_user_ext_set_property,
> +};
> +
> +static int xe_eu_stall_user_extensions(struct xe_device *xe, u64 extension, int ext_number,
> + struct eu_stall_open_properties *props)
> +{
> + u64 __user *address = u64_to_user_ptr(extension);
> + struct drm_xe_user_extension ext;
> + int err;
> + u32 idx;
> +
> + if (XE_IOCTL_DBG(xe, ext_number >= DRM_XE_EU_STALL_PROP_MAX))
> + return -E2BIG;
> +
> + err = __copy_from_user(&ext, address, sizeof(ext));
> + if (XE_IOCTL_DBG(xe, err))
> + return -EFAULT;
> +
> + if (XE_IOCTL_DBG(xe, ext.pad) ||
> + XE_IOCTL_DBG(xe, ext.name >= ARRAY_SIZE(xe_eu_stall_user_extension_funcs)))
> + return -EINVAL;
> +
> + idx = array_index_nospec(ext.name, ARRAY_SIZE(xe_eu_stall_user_extension_funcs));
> + err = xe_eu_stall_user_extension_funcs[idx](xe, extension, props);
> + if (XE_IOCTL_DBG(xe, err))
> + return err;
> +
> + if (ext.next_extension)
> + return xe_eu_stall_user_extensions(xe, ext.next_extension, ++ext_number, props);
> +
> + return 0;
> +}
> +
> +/**
> + * buf_data_size - Calculate the number of bytes in a circular buffer
> + * of size buf_size given the read and write pointers
> + * into the buffer.
> + *
> + * @read_ptr: Read pointer. Uses an additional overflow bit
> + * @write_ptr: Write pointer. Uses an additional overflow bit
> + *
> + * Returns: number of bytes of data in the buffer
> + */
> +static u32
> +buf_data_size(size_t buf_size, u32 read_ptr, u32 write_ptr)
> +{
> + u32 read_offset, write_offset, size = 0;
> +
> + read_offset = read_ptr & (buf_size - 1);
> + write_offset = write_ptr & (buf_size - 1);
> +
> + if (write_offset > read_offset)
> + size = write_offset - read_offset;
> + else
> + size = buf_size - read_offset + write_offset;
> +
> + return size;
> +}
> +
> +/**
> + * eu_stall_cntr_buf_check - check for data in the EU stall counter buffer
> + *
> + * @stream: xe EU stall data stream instance
> + *
> + * Returns: true if the EU stall buffer contains minimum stall data as
> + * specified by the event report count, else false.
> + */
> +static bool
> +eu_stall_cntr_buf_check(struct xe_eu_stall_cntr_stream *stream)
> +{
> + u32 read_ptr_reg, read_ptr, write_ptr_reg, write_ptr, total_data = 0;
> + u32 buf_size = stream->per_dss_buf_size;
> + struct xe_gt *gt = stream->gt;
> + struct per_dss_buf *dss_buf;
> + bool min_data_present;
> + u16 group, instance;
> + int dss;
> +
> + min_data_present = false;
> + for_each_dss_steering(dss, gt, group, instance) {
> + dss_buf = &stream->dss_buf[dss];
> + mutex_lock(&dss_buf->lock);
> + read_ptr = dss_buf->read;
> + write_ptr_reg = xe_gt_mcr_unicast_read(gt, XEHPC_EUSTALL_REPORT,
> + group, instance);
> + write_ptr = write_ptr_reg & XEHPC_EUSTALL_REPORT_WRITE_PTR_MASK;
> + write_ptr <<= (6 - XEHPC_EUSTALL_REPORT_WRITE_PTR_SHIFT);
> + write_ptr &= ((buf_size << 1) - 1);
> + /*
> + * If there has been an engine reset by GuC, and GuC doesn't restore
> + * the read and write pointer registers, the pointers will reset to 0.
> + * If so, update the cached read pointer.
> + */
> + if (unlikely((write_ptr < read_ptr) &&
> + ((read_ptr & buf_size) == (write_ptr & buf_size)))) {
> + read_ptr_reg = xe_gt_mcr_unicast_read(gt, XEHPC_EUSTALL_REPORT1,
> + group, instance);
> + read_ptr = read_ptr_reg & XEHPC_EUSTALL_REPORT1_READ_PTR_MASK;
> + read_ptr <<= (6 - XEHPC_EUSTALL_REPORT1_READ_PTR_SHIFT);
> + read_ptr &= ((buf_size << 1) - 1);
> + dss_buf->read = read_ptr;
> + }
> + if ((write_ptr != read_ptr) && !min_data_present) {
> + total_data += buf_data_size(buf_size, read_ptr, write_ptr);
> + /*
> + * Check if there are at least minimum number of stall data
> + * rows for poll() to indicate that the data is present.
> + * Each stall data row is 64B (cacheline size).
> + */
> + if (num_data_rows(total_data) >= stream->event_report_count)
> + min_data_present = true;
> + }
> + if (write_ptr_reg & XEHPC_EUSTALL_REPORT_OVERFLOW_DROP)
> + dss_buf->line_drop = true;
> + dss_buf->write = write_ptr;
> + mutex_unlock(&dss_buf->lock);
> + }
> + return min_data_present;
> +}
> +
> +static void
> +clear_dropped_eviction_line_bit(struct xe_gt *gt, u8 s, u8 ss)
> +{
> + enum xe_platform platform;
> + u32 write_ptr_reg;
> +
> + platform = gt_to_xe(gt)->info.platform;
> +
> + /* On PVC, the overflow bit has to be cleared by writing 1 to it.
> + * On other GPUs, the bit has to be cleared by writing 0 to it.
> + */
> + if (platform == XE_PVC)
> + write_ptr_reg = _MASKED_BIT_ENABLE(XEHPC_EUSTALL_REPORT_OVERFLOW_DROP);
> + else
> + write_ptr_reg = _MASKED_BIT_DISABLE(XEHPC_EUSTALL_REPORT_OVERFLOW_DROP);
> + xe_gt_mcr_unicast_write(gt, XEHPC_EUSTALL_REPORT, write_ptr_reg, s, ss);
> + trace_xe_reg_rw(gt, true, (XEHPC_EUSTALL_REPORT.__reg).addr,
> + write_ptr_reg, sizeof(write_ptr_reg));
> +}
> +
> +static int
> +__xe_eu_stall_buf_read(struct xe_eu_stall_cntr_stream *stream,
> + char __user *buf, size_t count,
> + size_t *total_size, struct xe_gt *gt,
> + u8 s, u8 ss)
> +{
> + unsigned int dss_per_grp = gt_to_xe(gt)->info.platform == XE_PVC ? 8 : 4;
> + size_t size, buf_size = stream->per_dss_buf_size;
> + u16 flags = 0, subslice = (s * dss_per_grp) + ss;
> + struct drm_xe_eu_stall_data_header header;
> + u32 read_ptr_reg, read_ptr, write_ptr;
> + u8 *dss_start_vaddr, *read_vaddr;
> + u32 read_offset, write_offset;
> + struct per_dss_buf *dss_buf;
> + bool line_drop = false;
> + int ret = 0;
> +
> + /* Hardware increments the read and write pointers such that they can
> + * overflow into one additional bit. For example, a 256KB size buffer
> + * offset pointer needs 18 bits. But HW uses 19 bits for the read and
> + * write pointers. This technique avoids wasting a slot in the buffer.
> + * Read and write offsets are calculated from the pointers in order to
> + * check if the write pointer has wrapped around the array.
> + */
> + dss_buf = &stream->dss_buf[subslice];
> + mutex_lock(&dss_buf->lock);
> + dss_start_vaddr = dss_buf->vaddr;
> + read_ptr = dss_buf->read;
> + write_ptr = dss_buf->write;
> + line_drop = dss_buf->line_drop;
> + read_offset = read_ptr & (buf_size - 1);
> + write_offset = write_ptr & (buf_size - 1);
> + /*
> + * If there has been an engine reset by GuC, and GuC doesn't restore
> + * the read and write pointer registers, the pointers will reset to 0.
> + * If so, update the cached read pointer.
> + */
> + if (unlikely((write_ptr < read_ptr) &&
> + ((read_ptr & buf_size) == (write_ptr & buf_size)))) {
> + read_ptr_reg = xe_gt_mcr_unicast_read(gt, XEHPC_EUSTALL_REPORT1,
> + s, ss);
> + read_ptr = read_ptr_reg & XEHPC_EUSTALL_REPORT1_READ_PTR_MASK;
> + read_ptr <<= (6 - XEHPC_EUSTALL_REPORT1_READ_PTR_SHIFT);
> + read_ptr &= ((buf_size << 1) - 1);
> + read_offset = read_ptr & (buf_size - 1);
> + dss_buf->read = read_ptr;
> + }
> +
> + trace_xe_eu_stall_cntr_read(s, ss, read_ptr, write_ptr,
> + read_offset, write_offset, *total_size);
> + if (write_ptr == read_ptr) {
> + mutex_unlock(&dss_buf->lock);
> + return 0;
> + }
> +
> + /* If write pointer offset is less than the read pointer offset,
> + * it means, write pointer has wrapped around the array.
> + */
> + if (write_offset > read_offset)
> + size = write_offset - read_offset;
> + else
> + size = buf_size - read_offset + write_offset;
> +
> + /* Read only the data that the user space buffer can accommodate */
> + if ((*total_size + size + sizeof(header)) > count) {
> + mutex_unlock(&dss_buf->lock);
> + return 0;
> + }
> +
> + if (line_drop)
> + flags = XE_EU_STALL_FLAG_OVERFLOW_DROP;
> +
> + /* Driver doesn't expose the number of C-slices to user space.
> + * A PVC configuration of 8 c-slices x 8 sub-slices will be
> + * exposed to the user space as 1 slice x 64 sub-slices.
> + */
> + header.subslice = subslice;
> + header.flags = flags;
> + header.record_size = CACHELINE_BYTES;
why CACHELINE_BYTES?
I would expected it to be the same size as drm_xe_eu_stall_data_xe2.
If it is for performance optimization then add reserved fields to drm_xe_eu_stall_data_xe2 until it is the same size as CACHELINE_BYTES.
> + header.num_records = size / header.record_size;
> +
> + if (copy_to_user((buf + *total_size), &header, sizeof(header))) {
> + mutex_unlock(&dss_buf->lock);
> + return -EFAULT;
> + }
> + *total_size += sizeof(header);
> +
> + read_vaddr = dss_start_vaddr + read_offset;
> +
> + if (write_offset > read_offset) {
> + if (copy_to_user((buf + *total_size), read_vaddr, size)) {
> + mutex_unlock(&dss_buf->lock);
> + return -EFAULT;
> + }
> + } else {
> + if (copy_to_user((buf + *total_size), read_vaddr, (buf_size - read_offset))) {
> + mutex_unlock(&dss_buf->lock);
> + return -EFAULT;
> + }
> + if (copy_to_user((buf + *total_size), dss_start_vaddr, write_offset)) {
> + mutex_unlock(&dss_buf->lock);
> + return -EFAULT;
> + }
> + }
> +
> + *total_size += size;
> + read_ptr += size;
> +
> + /* Read pointer can overflow into one additional bit */
> + read_ptr &= ((buf_size << 1) - 1);
> + read_ptr_reg = ((read_ptr >> 6) << XEHPC_EUSTALL_REPORT1_READ_PTR_SHIFT);
> + read_ptr_reg &= XEHPC_EUSTALL_REPORT1_READ_PTR_MASK;
> + read_ptr_reg |= (XEHPC_EUSTALL_REPORT1_READ_PTR_MASK <<
> + XEHPC_EUSTALL_REPORT1_MASK_SHIFT);
> + xe_gt_mcr_unicast_write(gt, XEHPC_EUSTALL_REPORT1, read_ptr_reg, s, ss);
> + trace_xe_reg_rw(gt, true, (XEHPC_EUSTALL_REPORT1.__reg).addr,
> + read_ptr_reg, sizeof(read_ptr_reg));
> + if (dss_buf->line_drop) {
> + clear_dropped_eviction_line_bit(gt, s, ss);
> + dss_buf->line_drop = false;
> + }
> + dss_buf->read = read_ptr;
> + mutex_unlock(&dss_buf->lock);
> + trace_xe_eu_stall_cntr_read(s, ss, read_ptr, write_ptr,
> + read_offset, write_offset, *total_size);
> + return ret;
ret is only set in the beginning of the function...
> +}
> +
> +/**
> + * xe_eu_stall_buf_read_locked - copy EU stall counters data from the
> + * per dss buffers to the userspace buffer
> + * @stream: A stream opened for EU stall count metrics
> + * @buf: destination buffer given by userspace
> + * @count: the number of bytes userspace wants to read
> + * @ppos: (inout) file seek position (unused)
> + *
> + * Returns: Number of bytes copied or a negative error code
> + * If we've successfully copied any data then reporting that takes
> + * precedence over any internal error status, so the data isn't lost.
> + */
> +static ssize_t
> +xe_eu_stall_buf_read_locked(struct xe_eu_stall_cntr_stream *stream,
> + struct file *file, char __user *buf,
> + size_t count, loff_t *ppos)
> +{
> + struct xe_gt *gt = stream->gt;
> + size_t total_size = 0;
> + u16 group, instance;
> + int ret = 0, dss;
> +
> + if (count == 0)
> + return -EINVAL;
> +
> + for_each_dss_steering(dss, gt, group, instance) {
> + ret = __xe_eu_stall_buf_read(stream, buf, count, &total_size,
> + gt, group, instance);
> + if (ret || count == total_size)
> + goto exit;
> + }
> +exit:
> + if (total_size)
> + return total_size;
> + else if (ret)
> + return ret;
> + else
> + return -EAGAIN;
> +}
> +
> +static void
> +free_eu_stall_cntr_buf(struct xe_eu_stall_cntr_stream *stream)
> +{
> + if (stream->bo) {
> + xe_bo_unpin_map_no_vm(stream->bo);
> + stream->vaddr = NULL;
> + stream->bo = NULL;
> + }
> + destroy_workqueue(stream->buf_check_wq);
> +}
> +
> +static int alloc_eu_stall_cntr_buf(struct xe_eu_stall_cntr_stream *stream,
> + u32 per_dss_buf_size)
> +{
> + struct xe_tile *tile = stream->gt->tile;
> + struct xe_gt *gt = stream->gt;
> + struct xe_bo *bo;
> + u32 size;
> + int ret = 0;
> + unsigned int last_dss;
> + xe_dss_mask_t all_dss;
> +
> + bitmap_or(all_dss, gt->fuse_topo.g_dss_mask, gt->fuse_topo.c_dss_mask,
> + XE_MAX_DSS_FUSE_BITS);
> + /*
> + * Enabled subslices can be discontiguous. Find the last subslice
> + * and calculate total buffer size based on that.
> + * intel_sseu_highest_xehp_dss returns zero based position.
> + * Therefore the result is incremented.
> + */
> + last_dss = xe_dss_mask_last_dss(all_dss);
> + size = per_dss_buf_size * (last_dss + 1);
> +
> + bo = xe_bo_create_pin_map(tile->xe, tile, NULL,
> + size, ttm_bo_type_kernel,
> + XE_BO_FLAG_VRAM_IF_DGFX(tile) |
> + XE_BO_FLAG_GGTT);
> + if (IS_ERR(bo))
> + ret = PTR_ERR(bo);
> +
> + stream->bo = bo;
> + stream->vaddr = bo->vmap.is_iomem ? bo->vmap.vaddr_iomem : bo->vmap.vaddr;
> +
> + return ret;
> +}
> +
> +static u32
> +gen_eustall_base(struct xe_eu_stall_cntr_stream *stream, bool enable)
> +{
> + u32 val = xe_bo_ggtt_addr(stream->bo);
> + u32 sz;
> +
> + XE_WARN_ON(!IS_ALIGNED(val, 64));
> +
> + switch (stream->per_dss_buf_size) {
> + case SZ_128K:
> + sz = 0;
> + break;
> + case SZ_256K:
> + sz = 1;
> + break;
> + case SZ_512K:
> + sz = 2;
> + break;
> + default:
> + MISSING_CASE(stream->per_dss_buf_size);
> + sz = 2;
> + }
> +
> + val |= REG_FIELD_PREP(XEHPC_EUSTALL_BASE_DSS_BUF_SZ, sz);
> + if (enable)
> + val |= XEHPC_EUSTALL_BASE_ENABLE_SAMPLING;
> +
> + return val;
> +}
> +
> +static void
> +xe_eu_stall_stream_enable(struct xe_eu_stall_cntr_stream *stream)
> +{
> + struct xe_gt *gt = stream->gt;
> + enum xe_platform platform;
> + u32 reg_value;
> +
> + platform = gt_to_xe(gt)->info.platform;
> +
> + /* Take runtime pm ref and forcewake to disable RC6 */
> + xe_pm_runtime_get(gt_to_xe(gt));
> + XE_WARN_ON(xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL));
> +
> + /*
> + * Wa_22016596838:pvc
> + * Disable EU DOP gating for PVC.
> + */
> + if (platform == XE_PVC)
> + xe_gt_mcr_multicast_write(gt, ROW_CHICKEN2,
> + _MASKED_BIT_ENABLE(DISABLE_DOP_GATING));
> +
> + reg_value = gen_eustall_base(stream, true);
> + xe_gt_mcr_multicast_write(gt, XEHPC_EUSTALL_BASE, reg_value);
> + trace_xe_reg_rw(gt, true, (XEHPC_EUSTALL_BASE.__reg).addr,
> + reg_value, sizeof(reg_value));
> +}
> +
> +static void
> +xe_eu_stall_stream_disable(struct xe_eu_stall_cntr_stream *stream)
> +{
> + struct xe_gt *gt = stream->gt;
> + enum xe_platform platform;
> + u16 group, instance;
> + u32 reg_value;
> + int dss;
> +
> + platform = gt_to_xe(gt)->info.platform;
> +
> + /*
> + * Before disabling EU stall sampling, check if any of the
> + * XEHPC_EUSTALL_REPORT registers have the drop bit set. If set,
> + * clear the bit. If the user space application reads all the
> + * stall data, the drop bit would be cleared during the read.
> + * But if there is any unread data and the drop bit is set for
> + * any subslice, the drop bit would continue to be set even
> + * after disabling EU stall sampling and may cause erroneous
> + * stall data in the subsequent stall data sampling run.
> + */
> + for_each_dss_steering(dss, gt, group, instance) {
> + reg_value = xe_gt_mcr_unicast_read(gt, XEHPC_EUSTALL_REPORT,
> + group, instance);
> + if (reg_value & XEHPC_EUSTALL_REPORT_OVERFLOW_DROP)
> + clear_dropped_eviction_line_bit(gt, group, instance);
> + }
> + reg_value = gen_eustall_base(stream, false);
> + xe_gt_mcr_multicast_write(gt, XEHPC_EUSTALL_BASE, reg_value);
> + trace_xe_reg_rw(gt, true, (XEHPC_EUSTALL_BASE.__reg).addr,
> + reg_value, sizeof(reg_value));
> +
> + /* Wa_22016596838:pvc */
> + if (platform == XE_PVC)
> + xe_gt_mcr_multicast_write(gt, ROW_CHICKEN2,
> + _MASKED_BIT_DISABLE(DISABLE_DOP_GATING));
> +
> + XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));
> + xe_pm_runtime_put(gt_to_xe(gt));
> +}
> +
> +static void eu_stall_buf_check_work_fn(struct work_struct *work)
> +{
> + struct xe_eu_stall_cntr_stream *stream =
> + container_of(work, typeof(*stream), buf_check_work);
> +
> + if (eu_stall_cntr_buf_check(stream)) {
> + stream->pollin = true;
> + wake_up(&stream->poll_wq);
> + }
> +}
> +
> +static enum
> +hrtimer_restart eu_stall_poll_check_timer_cb(struct hrtimer *hrtimer)
> +{
> + struct xe_eu_stall_cntr_stream *stream =
> + container_of(hrtimer, typeof(*stream), poll_check_timer);
> +
> + queue_work(stream->buf_check_wq, &stream->buf_check_work);
> + hrtimer_forward_now(hrtimer, ns_to_ktime(stream->poll_period));
> +
> + return HRTIMER_RESTART;
> +}
> +
> +static int xe_eu_stall_stream_init(struct xe_eu_stall_cntr_stream *stream,
> + struct eu_stall_open_properties *props)
> +{
> + u32 write_ptr_reg, write_ptr, read_ptr_reg;
> + u32 vaddr_offset, reg_value;
> + struct xe_gt *gt = stream->gt;
> + struct per_dss_buf *dss_buf;
> + u16 group, instance;
> + int ret, dss;
> +
> + init_waitqueue_head(&stream->poll_wq);
> + INIT_WORK(&stream->buf_check_work, eu_stall_buf_check_work_fn);
> + stream->buf_check_wq = alloc_ordered_workqueue("xe_eustall_cntr", 0);
> + if (!stream->buf_check_wq)
> + return -ENOMEM;
> + hrtimer_init(&stream->poll_check_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
> + stream->poll_check_timer.function = eu_stall_poll_check_timer_cb;
> + stream->event_report_count = props->event_report_count;
> + stream->per_dss_buf_size = props->eu_stall_buf_sz;
> + stream->poll_period = props->poll_period;
> +
> + ret = alloc_eu_stall_cntr_buf(stream, props->eu_stall_buf_sz);
> + if (ret)
> + return ret;
> +
> + xe_pm_runtime_get(gt_to_xe(gt));
> + XE_WARN_ON(xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL));
> +
> + reg_value = gen_eustall_base(stream, false);
> + xe_gt_mcr_multicast_write(gt, XEHPC_EUSTALL_BASE, reg_value);
> + trace_xe_reg_rw(gt, true, (XEHPC_EUSTALL_BASE.__reg).addr,
> + reg_value, sizeof(reg_value));
> + /* GGTT addresses can never be > 32 bits */
> + xe_gt_mcr_multicast_write(gt, XEHPC_EUSTALL_BASE_UPPER, 0);
> + reg_value = _MASKED_FIELD(EUSTALL_MOCS | EUSTALL_SAMPLE_RATE,
> + REG_FIELD_PREP(EUSTALL_MOCS, gt->mocs.uc_index << 1) |
> + REG_FIELD_PREP(EUSTALL_SAMPLE_RATE,
> + props->eu_stall_sampling_rate));
> + xe_gt_mcr_multicast_write(gt, XEHPC_EUSTALL_CTRL, reg_value);
> + trace_xe_reg_rw(gt, true, (XEHPC_EUSTALL_CTRL.__reg).addr,
> + reg_value, sizeof(reg_value));
> +
> + for_each_dss_steering(dss, gt, group, instance) {
> + write_ptr_reg = xe_gt_mcr_unicast_read(gt, XEHPC_EUSTALL_REPORT,
> + group, instance);
> + write_ptr = write_ptr_reg & XEHPC_EUSTALL_REPORT_WRITE_PTR_MASK;
> + write_ptr <<= (6 - XEHPC_EUSTALL_REPORT_WRITE_PTR_SHIFT);
> + write_ptr &= ((stream->per_dss_buf_size << 1) - 1);
> + read_ptr_reg = write_ptr >> (6 - XEHPC_EUSTALL_REPORT1_READ_PTR_SHIFT);
> + read_ptr_reg &= XEHPC_EUSTALL_REPORT1_READ_PTR_MASK;
> + read_ptr_reg |= (XEHPC_EUSTALL_REPORT1_READ_PTR_MASK <<
> + XEHPC_EUSTALL_REPORT1_MASK_SHIFT);
> + xe_gt_mcr_unicast_write(gt, XEHPC_EUSTALL_REPORT1,
> + read_ptr_reg, group, instance);
> + dss_buf = &stream->dss_buf[dss];
> + vaddr_offset = dss * props->eu_stall_buf_sz;
> + dss_buf->vaddr = stream->vaddr + vaddr_offset;
> + dss_buf->write = write_ptr;
> + dss_buf->read = write_ptr;
> + dss_buf->line_drop = false;
> + mutex_init(&dss_buf->lock);
> + }
> + XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));
> + xe_pm_runtime_put(gt_to_xe(gt));
> + return 0;
> +}
> +
> +/**
> + * xe_eu_stall_buf_read - handles read FOP for xe EU stall cntr stream FDs
> + * @file: An xe EU stall cntr stream file
> + * @buf: destination buffer given by userspace
> + * @count: the number of bytes userspace wants to read
> + * @ppos: (inout) file seek position (unused)
> + *
> + * Returns: The number of bytes copied or a negative error code on failure.
> + */
> +static ssize_t xe_eu_stall_buf_read(struct file *file, char __user *buf,
> + size_t count, loff_t *ppos)
> +{
> + struct xe_eu_stall_cntr_stream *stream = file->private_data;
> + struct xe_gt *gt = stream->gt;
> + ssize_t ret;
> +
> + if (!stream->enabled)
> + return -EIO;
> +
> + if (!(file->f_flags & O_NONBLOCK)) {
> + do {
> + if (!stream->pollin) {
> + ret = wait_event_interruptible(stream->poll_wq, stream->pollin);
> + if (ret)
> + return -EINTR;
> + }
> +
> + mutex_lock(>->eu_stall_cntr.lock);
> + ret = xe_eu_stall_buf_read_locked(stream, file, buf, count, ppos);
> + mutex_unlock(>->eu_stall_cntr.lock);
> + } while (ret == -EAGAIN);
> + } else {
> + mutex_lock(>->eu_stall_cntr.lock);
> + ret = xe_eu_stall_buf_read_locked(stream, file, buf, count, ppos);
> + mutex_unlock(>->eu_stall_cntr.lock);
> + }
> +
> + stream->pollin = false;
> +
> + return ret;
> +}
> +
> +static __poll_t
> +xe_eu_stall_buf_poll_locked(struct xe_eu_stall_cntr_stream *stream,
> + struct file *file, poll_table *wait)
> +{
> + __poll_t events = 0;
> +
> + poll_wait(file, &stream->poll_wq, wait);
> +
> + if (stream->pollin)
> + events |= EPOLLIN;
> +
> + return events;
> +}
> +
> +static __poll_t
> +xe_eu_stall_buf_poll(struct file *file, poll_table *wait)
> +{
> + struct xe_eu_stall_cntr_stream *stream = file->private_data;
> + struct xe_gt *gt = stream->gt;
> + __poll_t ret;
> +
> + mutex_lock(>->eu_stall_cntr.lock);
> + ret = xe_eu_stall_buf_poll_locked(stream, file, wait);
> + mutex_unlock(>->eu_stall_cntr.lock);
> +
> + return ret;
> +}
> +
> +static void
> +xe_eu_stall_cntr_enable_locked(struct xe_eu_stall_cntr_stream *stream)
> +{
> + if (stream->enabled)
> + return;
> +
> + stream->enabled = true;
> +
> + xe_eu_stall_stream_enable(stream);
> + hrtimer_start(&stream->poll_check_timer,
> + ns_to_ktime(stream->poll_period),
> + HRTIMER_MODE_REL);
> +}
> +
> +static void
> +xe_eu_stall_cntr_disable_locked(struct xe_eu_stall_cntr_stream *stream)
> +{
> + if (!stream->enabled)
> + return;
> +
> + stream->enabled = false;
> +
> + hrtimer_cancel(&stream->poll_check_timer);
> + flush_workqueue(stream->buf_check_wq);
> + xe_eu_stall_stream_disable(stream);
> +}
> +
> +static long
> +xe_eu_stall_cntr_ioctl_locked(struct xe_eu_stall_cntr_stream *stream,
> + unsigned int cmd, unsigned long arg)
> +{
> + switch (cmd) {
> + case DRM_XE_OBSERVATION_IOCTL_ENABLE:
> + xe_eu_stall_cntr_enable_locked(stream);
> + return 0;
> + case DRM_XE_OBSERVATION_IOCTL_DISABLE:
> + xe_eu_stall_cntr_disable_locked(stream);
> + return 0;
> + }
> +
> + return -EINVAL;
> +}
> +
> +/**
> + * xe_eu_stall_cntr_ioctl - support ioctl() usage with xe EU stall counter
> + * stream FDs
> + * @file: An xe EU stall cntr stream file
> + * @cmd: the ioctl request
> + * @arg: the ioctl data
> + *
> + * Implementation deferred to xe_eu_stall_cntr_ioctl_locked().
> + *
> + * Returns: zero on success or a negative error code. Returns -EINVAL for
> + * an unknown ioctl request.
> + */
> +static long xe_eu_stall_cntr_ioctl(struct file *file,
> + unsigned int cmd,
> + unsigned long arg)
> +{
> + struct xe_eu_stall_cntr_stream *stream = file->private_data;
> + struct xe_gt *gt = stream->gt;
> + long ret;
> +
> + mutex_lock(>->eu_stall_cntr.lock);
> + ret = xe_eu_stall_cntr_ioctl_locked(stream, cmd, arg);
> + mutex_unlock(>->eu_stall_cntr.lock);
> +
> + return ret;
> +}
> +
> +static void
> +xe_eu_stall_destroy_locked(struct xe_eu_stall_cntr_stream *stream)
> +{
> + xe_eu_stall_cntr_disable_locked(stream);
> + free_eu_stall_cntr_buf(stream);
> +}
> +
> +/**
> + * xe_eu_stall_release - handles userspace close() of a EU stall data
> + * stream file.
> + * @inode: anonymous inode associated with file
> + * @file: An xe EU stall stream file
> + *
> + * Cleans up any resources associated with an open EU stall data stream file.
> + */
> +static int xe_eu_stall_release(struct inode *inode, struct file *file)
> +{
> + struct xe_eu_stall_cntr_stream *stream = file->private_data;
> + struct xe_gt *gt = stream->gt;
> +
> + mutex_lock(>->eu_stall_cntr.lock);
> + xe_eu_stall_destroy_locked(stream);
> + kfree(stream);
> + gt->eu_stall_cntr.stream = NULL;
> + mutex_unlock(>->eu_stall_cntr.lock);
> +
> + /* Release the reference the EU stall stream kept on the driver */
> + drm_dev_put(>->tile->xe->drm);
> +
> + return 0;
> +}
> +
> +static const struct file_operations fops_eu_stall = {
> + .owner = THIS_MODULE,
> + .llseek = no_llseek,
> + .release = xe_eu_stall_release,
> + .poll = xe_eu_stall_buf_poll,
> + .read = xe_eu_stall_buf_read,
> + .unlocked_ioctl = xe_eu_stall_cntr_ioctl,
> + .compat_ioctl = xe_eu_stall_cntr_ioctl,
> +};
> +
> +/**
> + * xe_open_eu_stall_stream_locked - Open a EU stall data stream FD.
> + * @dev: drm device instance
> + * @props: individually validated u64 property value pairs
> + * @file: drm file
> + * @gt: GT from which the EU stall data will be captured
> + *
> + * Returns: zero on success or a negative error code.
> + */
> +static int
> +xe_open_eu_stall_stream_locked(struct drm_device *dev,
> + struct eu_stall_open_properties *props,
> + struct drm_file *file,
> + struct xe_gt *gt)
> +{
> + struct xe_device *xe = to_xe_device(dev);
> + struct xe_eu_stall_cntr_stream *stream;
> + unsigned long f_flags = 0;
> + xe_dss_mask_t all_dss;
> + int ret, stream_fd;
> + u32 tile_buf_size;
> +
> + bitmap_or(all_dss, gt->fuse_topo.g_dss_mask, gt->fuse_topo.c_dss_mask,
> + XE_MAX_DSS_FUSE_BITS);
> +
> + if (xe_observation_paranoid && !perfmon_capable()) {
> + drm_dbg(&xe->drm, "Insufficient privileges for EU stall monitoring\n");
> + return -EACCES;
> + }
> +
> + /* Only one session can be active at any time */
> + if (gt->eu_stall_cntr.stream) {
> + drm_dbg(&xe->drm, "EU stall cntr session already active\n");
> + return -EBUSY;
> + }
> +
> + tile_buf_size = props->eu_stall_buf_sz * (xe_dss_mask_last_dss(all_dss) + 1);
> + if (props->event_report_count > num_data_rows(tile_buf_size)) {
> + drm_dbg(&xe->drm, "Invalid EU stall data poll event report count %u\n",
> + props->event_report_count);
> + drm_dbg(&xe->drm, "Maximum event report count for the given buffer size is %u\n",
> + num_data_rows(tile_buf_size));
> + return -EINVAL;
> + }
> +
> + stream = kzalloc(sizeof(*stream), GFP_KERNEL);
> + if (!stream)
> + return -ENOMEM;
> +
> + gt->eu_stall_cntr.stream = stream;
> + stream->gt = gt;
> +
> + ret = xe_eu_stall_stream_init(stream, props);
> + if (ret) {
> + drm_dbg(&xe->drm, "EU stall stream init failed : %d\n", ret);
> + goto err_alloc;
> + }
> +
> + stream_fd = anon_inode_getfd("[xe_eu_stall]", &fops_eu_stall,
> + stream, f_flags);
> + if (stream_fd < 0) {
> + ret = stream_fd;
> + drm_dbg(&xe->drm, "EU stall inode get fd failed : %d\n", ret);
> + goto err_open;
> + }
> +
> + if (!props->open_disabled)
> + xe_eu_stall_cntr_enable_locked(stream);
> +
> + /* Take a reference on the driver that will be kept with stream_fd
> + * until its release.
> + */
> + drm_dev_get(>->tile->xe->drm);
> +
> + return stream_fd;
> +
> +err_open:
> + free_eu_stall_cntr_buf(stream);
> +err_alloc:
> + gt->eu_stall_cntr.stream = NULL;
> + kfree(stream);
> + return ret;
> +}
> +
> +int xe_open_eu_stall_stream(struct drm_device *dev,
> + u64 data,
> + struct drm_file *file)
> +{
> + struct xe_device *xe = to_xe_device(dev);
> + struct eu_stall_open_properties props;
> + struct xe_gt *gt;
> + int ret;
> +
> + memset(&props, 0, sizeof(struct eu_stall_open_properties));
> +
> + /* Set default values */
> + props.gt_id = 0;
> + props.eu_stall_buf_sz = SZ_256K;
> + props.eu_stall_sampling_rate = 4;
> + props.poll_period = DEFAULT_POLL_PERIOD_NS;
> + props.event_report_count = 1;
> +
> + ret = xe_eu_stall_user_extensions(xe, data, 0, &props);
> + if (ret)
> + return ret;
> +
> + gt = xe_device_get_gt(xe, props.gt_id);
> + if (!gt) {
> + drm_dbg(&xe->drm, "Invalid GT for EU stall sampling \n");
> + return -EINVAL;
> + }
> +
> + mutex_lock(>->eu_stall_cntr.lock);
> + ret = xe_open_eu_stall_stream_locked(dev, &props, file, gt);
> + mutex_unlock(>->eu_stall_cntr.lock);
> + return ret;
> +}
> diff --git a/drivers/gpu/drm/xe/xe_eustall_cntr.h b/drivers/gpu/drm/xe/xe_eustall_cntr.h
> new file mode 100644
> index 000000000000..7a99b8c819d3
> --- /dev/null
> +++ b/drivers/gpu/drm/xe/xe_eustall_cntr.h
> @@ -0,0 +1,62 @@
> +/* SPDX-License-Identifier: MIT */
> +/*
> + * Copyright © 2024 Intel Corporation
> + */
> +
> +#ifndef __XE_EUSTALL_CNTR_H__
> +#define __XE_EUSTALL_CNTR_H__
> +
> +#include <drm/drm_file.h>
> +#include <drm/xe_drm.h>
> +#include "regs/xe_eu_stall_regs.h"
> +
> +#define XE_MAX_DSS 128
> +
> +struct per_dss_buf {
> + u8 *vaddr;
> + u32 write;
> + u32 read;
> + bool line_drop;
> + /* lock to protect read and write pointers */
> + struct mutex lock;
> +};
> +
> +/**
> + * struct xe_eu_stall_cntr_stream - state of EU stall counter stream FD
> + */
> +struct xe_eu_stall_cntr_stream {
> + struct xe_bo *bo;
> + struct xe_gt *gt;
> +
> + bool enabled;
> + bool pollin;
> + size_t per_dss_buf_size;
> + struct hrtimer poll_check_timer;
> + struct work_struct buf_check_work;
> + struct workqueue_struct *buf_check_wq;
> + wait_queue_head_t poll_wq;
> + u32 event_report_count;
> + u64 poll_period;
> +
> + /**
> + * State of the EU stall counter buffer.
> + */
> + u8 *vaddr;
> + struct per_dss_buf dss_buf[XE_MAX_DSS];
> +};
> +
> +struct xe_eu_stall_cntr_gt {
> + /* Lock to protect stream */
> + struct mutex lock;
> +
> + /* Execution Unit (EU) stall counter stream */
> + struct xe_eu_stall_cntr_stream *stream;
> +};
> +
> +void xe_eustall_cntr_init(struct xe_gt *gt);
> +
> +int xe_open_eu_stall_stream(struct drm_device *dev,
> + u64 data,
> + struct drm_file *file);
> +
> +#endif
> diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c
> index ce8994b808fe..74f4205a6d60 100644
> --- a/drivers/gpu/drm/xe/xe_gt.c
> +++ b/drivers/gpu/drm/xe/xe_gt.c
> @@ -61,6 +61,7 @@
> #include "xe_vm.h"
> #include "xe_wa.h"
> #include "xe_wopcm.h"
> +#include "xe_eustall_cntr.h"
>
> static void gt_fini(struct drm_device *drm, void *arg)
> {
> @@ -622,6 +623,8 @@ int xe_gt_init(struct xe_gt *gt)
>
> xe_gt_record_user_engines(gt);
>
> + xe_eustall_cntr_init(gt);
> +
> return 0;
> }
>
> diff --git a/drivers/gpu/drm/xe/xe_gt_topology.c b/drivers/gpu/drm/xe/xe_gt_topology.c
> index 25ff03ab8448..54a57179a53c 100644
> --- a/drivers/gpu/drm/xe/xe_gt_topology.c
> +++ b/drivers/gpu/drm/xe/xe_gt_topology.c
> @@ -247,6 +247,15 @@ xe_dss_mask_group_ffs(const xe_dss_mask_t mask, int groupsize, int groupnum)
> return find_next_bit(mask, XE_MAX_DSS_FUSE_BITS, groupnum * groupsize);
> }
>
> +/*
> + * Used to obtain the index of the last DSS.
> + */
> +unsigned int
> +xe_dss_mask_last_dss(const xe_dss_mask_t mask)
> +{
> + return find_last_bit(mask, XE_MAX_DSS_FUSE_BITS);
> +}
> +
> bool xe_dss_mask_empty(const xe_dss_mask_t mask)
> {
> return bitmap_empty(mask, XE_MAX_DSS_FUSE_BITS);
> diff --git a/drivers/gpu/drm/xe/xe_gt_topology.h b/drivers/gpu/drm/xe/xe_gt_topology.h
> index 746b325bbf6e..7ee022784397 100644
> --- a/drivers/gpu/drm/xe/xe_gt_topology.h
> +++ b/drivers/gpu/drm/xe/xe_gt_topology.h
> @@ -28,6 +28,9 @@ void xe_gt_topology_dump(struct xe_gt *gt, struct drm_printer *p);
> unsigned int
> xe_dss_mask_group_ffs(const xe_dss_mask_t mask, int groupsize, int groupnum);
>
> +unsigned int
> +xe_dss_mask_last_dss(const xe_dss_mask_t mask);
> +
> bool xe_dss_mask_empty(const xe_dss_mask_t mask);
>
> bool
> diff --git a/drivers/gpu/drm/xe/xe_gt_types.h b/drivers/gpu/drm/xe/xe_gt_types.h
> index 6b5e0b45efb0..609db0154122 100644
> --- a/drivers/gpu/drm/xe/xe_gt_types.h
> +++ b/drivers/gpu/drm/xe/xe_gt_types.h
> @@ -16,6 +16,7 @@
> #include "xe_reg_sr_types.h"
> #include "xe_sa_types.h"
> #include "xe_uc_types.h"
> +#include "xe_eustall_cntr.h"
>
> struct xe_exec_queue_ops;
> struct xe_migrate;
> @@ -391,6 +392,9 @@ struct xe_gt {
>
> /** @oa: oa observation subsystem per gt info */
> struct xe_oa_gt oa;
> +
> + /** @eu_stall_cntr: EU stall counters subsystem per gt info */
> + struct xe_eu_stall_cntr_gt eu_stall_cntr;
> };
>
> #endif
> diff --git a/drivers/gpu/drm/xe/xe_observation.c b/drivers/gpu/drm/xe/xe_observation.c
> index fcb584b42a7d..5db9c589ce24 100644
> --- a/drivers/gpu/drm/xe/xe_observation.c
> +++ b/drivers/gpu/drm/xe/xe_observation.c
> @@ -8,6 +8,7 @@
>
> #include <drm/xe_drm.h>
>
> +#include "xe_eustall_cntr.h"
> #include "xe_oa.h"
> #include "xe_observation.h"
>
> @@ -29,6 +30,17 @@ static int xe_oa_ioctl(struct drm_device *dev, struct drm_xe_observation_param *
> }
> }
>
> +static int xe_eu_stall_ioctl(struct drm_device *dev, struct drm_xe_observation_param *arg,
> + struct drm_file *file)
> +{
> + switch (arg->observation_op) {
> + case DRM_XE_OBSERVATION_OP_STREAM_OPEN:
> + return xe_open_eu_stall_stream(dev, arg->param, file);
> + default:
> + return -EINVAL;
> + }
> +}
> +
> /**
> * xe_observation_ioctl - The top level observation layer ioctl
> * @dev: @drm_device
> @@ -51,6 +63,8 @@ int xe_observation_ioctl(struct drm_device *dev, void *data, struct drm_file *fi
> switch (arg->observation_type) {
> case DRM_XE_OBSERVATION_TYPE_OA:
> return xe_oa_ioctl(dev, arg, file);
> + case DRM_XE_OBSERVATION_TYPE_EU_STALL:
> + return xe_eu_stall_ioctl(dev, arg, file);
> default:
> return -EINVAL;
> }
> diff --git a/drivers/gpu/drm/xe/xe_trace.h b/drivers/gpu/drm/xe/xe_trace.h
> index 09ca1ad057b0..ef532eae3f88 100644
> --- a/drivers/gpu/drm/xe/xe_trace.h
> +++ b/drivers/gpu/drm/xe/xe_trace.h
> @@ -374,6 +374,41 @@ TRACE_EVENT(xe_reg_rw,
> (u32)(__entry->val >> 32))
> );
>
> +TRACE_EVENT(xe_eu_stall_cntr_read,
> + TP_PROTO(u8 slice, u8 subslice,
> + u32 read_ptr, u32 write_ptr,
> + u32 read_offset, u32 write_offset,
> + size_t total_size),
> + TP_ARGS(slice, subslice, read_ptr, write_ptr,
> + read_offset, write_offset, total_size),
> +
> + TP_STRUCT__entry(
> + __field(u8, slice)
> + __field(u8, subslice)
> + __field(u32, read_ptr)
> + __field(u32, write_ptr)
> + __field(u32, read_offset)
> + __field(u32, write_offset)
> + __field(size_t, total_size)
> + ),
> +
> + TP_fast_assign(
> + __entry->slice = slice;
> + __entry->subslice = subslice;
> + __entry->read_ptr = read_ptr;
> + __entry->write_ptr = write_ptr;
> + __entry->read_offset = read_offset;
> + __entry->write_offset = write_offset;
> + __entry->total_size = total_size;
> + ),
> +
> + TP_printk("slice:%u subslice:%u readptr:0x%x writeptr:0x%x read off:%u write off:%u size:%zu ",
> + __entry->slice, __entry->subslice,
> + __entry->read_ptr, __entry->write_ptr,
> + __entry->read_offset, __entry->write_offset,
> + __entry->total_size)
> +);
> +
> #endif
>
> /* This part must be outside protection */
> diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h
> index 19619d4952a8..343de700d10d 100644
> --- a/include/uapi/drm/xe_drm.h
> +++ b/include/uapi/drm/xe_drm.h
> @@ -1387,6 +1387,8 @@ struct drm_xe_wait_user_fence {
> enum drm_xe_observation_type {
> /** @DRM_XE_OBSERVATION_TYPE_OA: OA observation stream type */
> DRM_XE_OBSERVATION_TYPE_OA,
> + /** @DRM_XE_OBSERVATION_TYPE_EU_STALL: EU stall sampling observation stream type */
> + DRM_XE_OBSERVATION_TYPE_EU_STALL,
> };
>
> /**
> @@ -1686,6 +1688,81 @@ struct drm_xe_oa_stream_info {
> __u64 reserved[3];
> };
>
> +/**
> + * enum drm_xe_eu_stall_property_id - EU stall data stream property ids.
> + *
> + * These properties are passed to the driver as a chain of
> + * @drm_xe_ext_set_property structures with @property set to these
> + * properties' enums and @value set to the corresponding values of these
> + * properties. @drm_xe_user_extension base.name should be set to
> + * @DRM_XE_EU_STALL_EXTENSION_SET_PROPERTY.
> + */
> +enum drm_xe_eu_stall_property_id {
> +#define DRM_XE_EU_STALL_EXTENSION_SET_PROPERTY 0
> + /**
> + * @DRM_XE_EU_STALL_PROP_BUF_SZ: Per DSS Memory Buffer Size.
> + * Valid values are 128 KB, 256 KB, and 512 KB.
> + */
> + DRM_XE_EU_STALL_PROP_BUF_SZ = 1,
> +
> + /**
> + * @DRM_XE_EU_STALL_PROP_SAMPLE_RATE: Sampling rate
> + * in multiples of 251 cycles. Valid values are 1 to 7.
> + * If the value is 1, sampling interval is 251 cycles.
> + * If the value is 7, sampling interval is 7 x 251 cycles.
> + */
> + DRM_XE_EU_STALL_PROP_SAMPLE_RATE,
> +
> + /**
> + * @DRM_XE_EU_STALL_PROP_POLL_PERIOD: EU stall data
> + * poll period in nanoseconds. should be at least 100000 ns.
> + */
> + DRM_XE_EU_STALL_PROP_POLL_PERIOD,
> +
> + /**
> + * @DRM_XE_EU_STALL_PROP_EVENT_REPORT_COUNT: Minimum number of
> + * EU stall data rows to be present in the kernel buffer for
> + * poll() to set POLLIN (data present).
> + */
> + DRM_XE_EU_STALL_PROP_EVENT_REPORT_COUNT,
> +
> + /**
> + * @DRM_XE_EU_STALL_PROP_GT_ID: GT ID of the GT on which
> + * EU stall data will be captured.
> + */
> + DRM_XE_EU_STALL_PROP_GT_ID,
> +
> + /**
> + * @DRM_XE_EU_STALL_PROP_OPEN_DISABLED: A value of 1 will open
> + * the EU stall data stream without enabling EU stall sampling.
> + */
> + DRM_XE_EU_STALL_PROP_OPEN_DISABLED,
> +
> + DRM_XE_EU_STALL_PROP_MAX
> +};
> +
> +/**
> + * struct drm_xe_eu_stall_data_header - EU stall data header.
> + * Header with additional information that the driver adds
> + * before EU stall data of each subslice during read().
> + */
> +struct drm_xe_eu_stall_data_header {
> + /** @subslice: subslice number from which the following data
> + * has been captured.
> + */
> + __u16 subslice;
> + /** @flags: flags */
> + __u16 flags;
> +/* EU stall data dropped by the HW due to memory buffer being full */
> +#define XE_EU_STALL_FLAG_OVERFLOW_DROP (1 << 0)
> + /** @record_size: size of each EU stall data record */
> + __u16 record_size;
> + /** @num_records: number of records following the header */
> + __u16 num_records;
> + /** @reserved: Reserved */
> + __u16 reserved[4];
> +};
> +
> #if defined(__cplusplus)
> }
> #endif
More information about the Intel-xe
mailing list