[Intel-gfx] [PATCH v13 00/22] Add vfio_device cdev for iommufd support

Duan, Zhenzhong zhenzhong.duan at intel.com
Wed Jun 21 09:17:06 UTC 2023


>-----Original Message-----
>From: Liu, Yi L <yi.l.liu at intel.com>
>Sent: Friday, June 16, 2023 5:39 PM 
>Subject: [PATCH v13 00/22] Add vfio_device cdev for iommufd support
>...

Per Yi's suggestion, wrote a selftest app to verify functions of this patchset by referencing https://github.com/awilliam/tests/

Test pass with this app, so Tested-by: Zhenzhong Duan <zhenzhong.duan at intel.com>
Test result:
# ./iommufd-pci-device-open 0 0000:81:11.0
Using PCI device 0000:81:11.0 vfio id:  0
Bind to IOMMUFD 4 with dev_id 1
Device supports 9 regions, 5 irqs
Region 0: size 0x20000, offset 0x0, flags 0x7
[▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒]
Region 1: size 0x0, offset 0x10000000000, flags 0x0
Region 2: size 0x0, offset 0x20000000000, flags 0x0
Region 3: size 0x4000, offset 0x30000000000, flags 0xf
[▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒]
Region 4: size 0x0, offset 0x40000000000, flags 0x0
Region 5: size 0x0, offset 0x50000000000, flags 0x0
Region 6: size 0x0, offset 0x60000000000, flags 0x0
Region 7: size 0x1000, offset 0x70000000000, flags 0x3
Region 8: Failed to get info
Attached IOMMUFD 4 ioas 2 hwpt 3
Mapped user_va 7f496ed00000 size 100000 to iova 0 in ioas 2
Hot reset dependent device count: 4
0: 0000:81:00.0 devid -1
1: 0000:81:00.1 devid -1
2: 0000:81:11.0 devid 1
3: 0000:81:11.1 devid -1
Cannot reset device, depends on device 0000:81:00.0 which is not owned.
Cannot reset device, depends on device 0000:81:00.1 which is not owned.
Cannot reset device, depends on device 0000:81:11.1 which is not owned.

Source code:
# cat iommufd-pci-device-open.c
#include <errno.h>
#include <libgen.h>
#include <fcntl.h>
#include <libgen.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
#include <sys/param.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/vfs.h>

#include <linux/ioctl.h>
#include "iommufd.h"
#include "vfio.h"

void usage(char *name)
{
        printf("usage: %s <vfio device id> <ssss:bb:dd.f>\n", name);
}

int main(int argc, char **argv)
{
        int i, ret, container, group, device, vfioid, iommufd;
        char path[PATH_MAX];
        int seg, bus, dev, func;

        struct vfio_device_info device_info = {
                .argsz = sizeof(device_info)
        };
        struct vfio_region_info region_info = {
                .argsz = sizeof(region_info)
        };

        if (argc < 3) {
                usage(argv[0]);
                return -1;
        }

        ret = sscanf(argv[1], "%d", &vfioid);
        if (ret != 1) {
                usage(argv[0]);
                return -1;
        }

        ret = sscanf(argv[2], "%04x:%02x:%02x.%d", &seg, &bus, &dev, &func);
        if (ret != 4) {
                usage(argv[0]);
                return -1;
        }

        printf("Using PCI device %04x:%02x:%02x.%d vfio id:  %d\n",
               seg, bus, dev, func, vfioid);

        snprintf(path, sizeof(path), "/dev/vfio/devices/vfio%d", vfioid);
        device = open(path, O_RDWR);
        if (device < 0) {
                printf("Failed to open %s, %d (%s)\n",
                       path, device, strerror(errno));
                return device;
        }

        struct vfio_device_bind_iommufd bind = {
            .argsz = sizeof(bind),
            .flags = 0,
        };
        struct iommu_ioas_alloc alloc_data  = {
            .size = sizeof(alloc_data),
            .flags = 0,
        };
        struct vfio_device_attach_iommufd_pt attach_data = {
            .argsz = sizeof(attach_data),
            .flags = 0,
        };
        struct iommu_ioas_map map = {
            .size = sizeof(map),
            .flags = IOMMU_IOAS_MAP_READABLE |
                IOMMU_IOAS_MAP_WRITEABLE |
                IOMMU_IOAS_MAP_FIXED_IOVA,
            .__reserved = 0,
        };

        iommufd = open("/dev/iommu", O_RDWR);
        if (iommufd < 0) {
                printf("Failed to open /dev/iommufd, %d (%s)\n",
                       iommufd, strerror(errno));
                return iommufd;
        }

        bind.iommufd = iommufd; // negative value means vfio-noiommu mode
        ret = ioctl(device, VFIO_DEVICE_BIND_IOMMUFD, &bind);
        if (ret < 0) {
                printf("Failed VFIO_DEVICE_BIND_IOMMUFD %d (%s)\n",
                       ret, strerror(errno));
                return ret;
        }

        printf("Bind to IOMMUFD %d with dev_id %d\n", iommufd, bind.out_devid);

        if (ioctl(device, VFIO_DEVICE_GET_INFO, &device_info)) {
                printf("Failed to get device info\n");
                return -1;
        }

        printf("Device supports %d regions, %d irqs\n",
               device_info.num_regions, device_info.num_irqs);

        for (i = 0; i < device_info.num_regions; i++) {
                printf("Region %d: ", i);
                region_info.index = i;
                if (ioctl(device, VFIO_DEVICE_GET_REGION_INFO, &region_info)) {
                        printf("Failed to get info\n");
                        continue;
                }

                printf("size 0x%lx, offset 0x%lx, flags 0x%x\n",
                       (unsigned long)region_info.size,
                       (unsigned long)region_info.offset, region_info.flags);
                if (region_info.flags & VFIO_REGION_INFO_FLAG_MMAP) {
                        void *map = mmap(NULL, (size_t)region_info.size,
                                         PROT_READ, MAP_SHARED, device,
                                         (off_t)region_info.offset);
                        if (map == MAP_FAILED) {
                                printf("mmap failed\n");
                                continue;
                        }

                        printf("[");
                        fwrite(map, 1, region_info.size > 16 ? 16 :
                                                region_info.size, stdout);
                        printf("]\n");
                        munmap(map, (size_t)region_info.size);
                }
        }

        ret = ioctl(iommufd, IOMMU_IOAS_ALLOC, &alloc_data);
        if (ret < 0) {
                printf("Failed IOMMU_IOAS_ALLOC %d (%s)\n",
                       ret, strerror(errno));
                return ret;
        }

        attach_data.pt_id = alloc_data.out_ioas_id;
        ret = ioctl(device, VFIO_DEVICE_ATTACH_IOMMUFD_PT, &attach_data);
        if (ret < 0) {
                printf("Failed VFIO_DEVICE_ATTACH_IOMMUFD_PT ioas_id %d %d (%s)\n",
                       attach_data.pt_id, ret, strerror(errno));
                return ret;
        }

        printf("Attached IOMMUFD %d ioas %d hwpt %d\n", iommufd, alloc_data.out_ioas_id, attach_data.pt_id);

        /* Allocate some space and setup a DMA mapping */
        map.user_va = (int64_t)mmap(0, 1024 * 1024, PROT_READ | PROT_WRITE,
                MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
        map.iova = 0; /* 1MB starting at 0x0 from device view */
        map.length = 1024 * 1024;
        map.ioas_id = alloc_data.out_ioas_id;;

        ret = ioctl(iommufd, IOMMU_IOAS_MAP, &map);
        if (ret < 0) {
                printf("Failed VFIO_DEVICE_ATTACH_IOMMUFD_PT ioas_id %d %d (%s)\n",
                       attach_data.pt_id, ret, strerror(errno));
                return ret;
        }
        printf("Mapped user_va %llx size %llx to iova %llx in ioas %d\n", map.user_va, map.length, map.iova, map.ioas_id);

        struct vfio_pci_hot_reset_info *reset_info;
        struct vfio_pci_dependent_device *devices;
        struct vfio_pci_hot_reset *reset;
        reset_info = malloc(sizeof(*reset_info));
        if (!reset_info) {
                printf("Failed to alloc info struct\n");
                return -ENOMEM;
        }

        reset_info->argsz = sizeof(*reset_info);

        ret = ioctl(device, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, reset_info);
        if (ret && errno == ENODEV) {
                printf("Device does not support hot reset\n");
                return 0;
        }
        if (!ret || errno != ENOSPC) {
                printf("Expected fail/-ENOSPC, got %d/%d\n", ret, -errno);
                return -1;
        }

        printf("Hot reset dependent device count: %d\n", reset_info->count);

        reset_info = realloc(reset_info, sizeof(*reset_info) +
                        (reset_info->count * sizeof(*devices)));
        if (!reset_info) {
                printf("Failed to re-alloc info struct\n");
                return -ENOMEM;
        }

        reset_info->argsz = sizeof(*reset_info) +
                (reset_info->count * sizeof(*devices));
        ret = ioctl(device, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, reset_info);
        if (ret) {
                printf("Reset Info error\n");
                return ret;
        }

        devices = &reset_info->devices[0];

        for (i = 0; i < reset_info->count; i++)
                printf("%d: %04x:%02x:%02x.%d devid %d\n", i,
                                devices[i].segment, devices[i].bus,
                                devices[i].devfn >> 3, devices[i].devfn & 7,
                                devices[i].devid);

        if (!(reset_info->flags & VFIO_PCI_HOT_RESET_FLAG_DEV_ID)) {
                printf("VFIO_PCI_HOT_RESET_FLAG_DEV_ID should be set for IOMMUFD\n");
                return -1;
        }

        int unowned_cnt = 0;
        if (!(reset_info->flags & VFIO_PCI_HOT_RESET_FLAG_DEV_ID_OWNED)) {
                for (i = 0; i < reset_info->count; i++) {
                        if (devices[i].devid == VFIO_PCI_DEVID_NOT_OWNED) {
                                unowned_cnt++;
                                printf("Cannot reset device, "
                                        "depends on device %04x:%02x:%02x.%x "
                                        "which is not owned.\n",
                                        devices[i].segment, devices[i].bus,
                                        devices[i].devfn >> 3, devices[i].devfn & 7);
                        }
                }
                if (!unowned_cnt) {
                        printf("flags mismatch with data field, "
                                "VFIO_PCI_HOT_RESET_FLAG_DEV_ID_OWNED claimed but "
                                "no VFIO_PCI_DEVID_NOT_OWNED\n");
                        return -1;
                }
                return 0;
        }


        printf("Attempting reset: ");
        fflush(stdout);

        /* Use zero length array for hot reset with iommufd backend */
        reset = malloc(sizeof(*reset));
        reset->argsz = sizeof(*reset);

        /* Bus reset! */
        ret = ioctl(device, VFIO_DEVICE_PCI_HOT_RESET, reset);

        ret = ioctl(device, VFIO_DEVICE_PCI_HOT_RESET, reset);
        printf("Hot reset: %s\n", ret ? "Failed" : "Pass");

        printf("Press any key to exit\n");
        fgetc(stdin);

        return 0;
}

Thanks
Zhenzhong




More information about the Intel-gfx mailing list