arm64 smmu 驱动笔记（4.19）

原创

已于 2024-02-01 14:35:29 修改 · 1.6k 阅读

5 ·

CC 4.0 BY-SA版权

文章标签：

#linux

于 2023-02-01 16:31:29 首次发布

文章详细阐述了Linux内核中iommu管理设备streamID的流程，涉及arm_smmu_add_device函数、iommu_fwspec结构体、arm_smmu_iort_xlate以及arm_smmu_init_domain_context等关键步骤。在arm_smmu_add_device中，设备可能有多个sid，并通过iommu_fwspec_add_ids函数添加到iommu_fwspec中。此外，还介绍了arm_smmu_device_group如何将设备分组，并关联到iommu_group，以及如何通过streamID设置硬件寄存器。最后，讨论了地址映射的io_pgtable_ops操作及其在不同场景下的应用。

设备StreamID：sid

struct iommu_fwspec {
	const struct iommu_ops	*ops;
	struct fwnode_handle	*iommu_fwnode;
	void			*iommu_priv;
	unsigned int		num_ids;
	u32			ids[1];  //ids保存的某device使用的streamID
};

通过iommu_fwspec_add_id函数设置：

struct iommu_fwspec {
	const struct iommu_ops	*ops;
	struct fwnode_handle	*iommu_fwnode;
	void			*iommu_priv;
	unsigned int		num_ids;  //和该设备关联的总共ids/streamID
	u32			ids[1];//sid内容数组
};


int iommu_fwspec_add_ids(struct device *dev, u32 *ids, int num_ids)
{
        struct iommu_fwspec *fwspec = dev->iommu_fwspec;
        size_t size;
        int i;

        if (!fwspec)
                return -EINVAL;

        size = offsetof(struct iommu_fwspec, ids[fwspec->num_ids + num_ids]);
        if (size > sizeof(*fwspec)) {
                fwspec = krealloc(dev->iommu_fwspec, size, GFP_KERNEL);
                if (!fwspec)
                        return -ENOMEM;

                dev->iommu_fwspec = fwspec;
        }

        for (i = 0; i < num_ids; i++)
                fwspec->ids[fwspec->num_ids + i] = ids[i];

        fwspec->num_ids += num_ids;
        return 0;
}

设置流程dump_stack信息:

[ 79.499624] ===iommu_fwspec_add_ids for dev 0000:0a:00.0 num_ids 1
[ 79.505862] CPU: 29 PID: 1 Comm: swapper/0 Not tainted 4.19.0l+ #20
[ 79.512098] Hardware name: PHYTIUM LTD Phytium S2500/64/Phytium S2500/64, BIOS V2.2 Feb 9 2021
[ 79.520754] Call trace:
[ 79.523192] dump_backtrace+0x0/0x1c0
[ 79.526838] show_stack+0x24/0x30
[ 79.530138] dump_stack+0x9c/0xbc
[ 79.533439] iommu_fwspec_add_ids+0x44/0xd0
[ 79.537602] iort_iommu_xlate+0x144/0x170
[ 79.541592] iort_pci_iommu_init+0x58/0x80
[ 79.545670] pci_for_each_dma_alias+0x44/0x188
[ 79.550093] iort_iommu_configure+0xac/0x1c8
[ 79.554343] acpi_dma_configure+0x68/0xbc
[ 79.558333] pci_dma_configure+0xa8/0xc8
[ 79.562236] dma_configure+0x4c/0x64
[ 79.565795] really_probe+0x98/0x3b8
[ 79.569354] driver_probe_device+0x6c/0x138
[ 79.573516] __driver_attach+0x118/0x150
[ 79.577419] bus_for_each_dev+0x84/0xd8
[ 79.581237] driver_attach+0x30/0x40
[ 79.584795] bus_add_driver+0x20c/0x250
[ 79.588613] driver_register+0x64/0x110
[ 79.592430] __pci_register_driver+0x58/0x68
[ 79.596681] xhci_pci_init+0x58/0x64
[ 79.600240] do_one_initcall+0x68/0x1e4
[ 79.604059] kernel_init_freeable+0x2d0/0x388
[ 79.608396] kernel_init+0x18/0x108
[ 79.611867] ret_from_fork+0x10/0x1c

iort_iommu_configure函数：

//drivers/acpi/arm64/iort.c
const struct iommu_ops *iort_iommu_configure(struct device *dev)
{
	struct acpi_iort_node *node, *parent;
	const struct iommu_ops *ops;
	u32 streamid = 0;
	int err = -ENODEV;

	/*
	 * If we already translated the fwspec there
	 * is nothing left to do, return the iommu_ops.
	 */
	ops = iort_fwspec_iommu_ops(dev->iommu_fwspec);
	if (ops)
		return ops;

	if (dev_is_pci(dev)) { //PCI设备的sid 使用的是BDF号。
		struct pci_bus *bus = to_pci_dev(dev)->bus;
		struct iort_pci_alias_info info = { .dev = dev };

		node = iort_scan_node(ACPI_IORT_NODE_PCI_ROOT_COMPLEX,
				      iort_match_node_callback, &bus->dev);
		if (!node)
			return NULL;

		info.node = node;
//计算sid
		err = pci_for_each_dma_alias(to_pci_dev(dev),
					     iort_pci_iommu_init, &info); 
	} else {
		int i = 0;

		node = iort_scan_node(ACPI_IORT_NODE_NAMED_COMPONENT,
				      iort_match_node_callback, dev);
		if (!node)
			return NULL;

		do {
			parent = iort_node_map_platform_id(node, &streamid,
							   IORT_IOMMU_TYPE,
							   i++);

			if (parent)
				err = iort_iommu_xlate(dev, parent, streamid);
		} while (parent && !err);
	}

	/*
	 * If we have reason to believe the IOMMU driver missed the initial
	 * add_device callback for dev, replay it to get things in order.
	 */
	if (!err) {
		ops = iort_fwspec_iommu_ops(dev->iommu_fwspec);
		err = iort_add_device_replay(ops, dev);
	}

	/* Ignore all other errors apart from EPROBE_DEFER */
	if (err == -EPROBE_DEFER) {
		ops = ERR_PTR(err);
	} else if (err) {
		dev_dbg(dev, "Adding to IOMMU failed: %d\n", err);
		ops = NULL;
	}

	return ops;
}



 */
int pci_for_each_dma_alias(struct pci_dev *pdev,
			   int (*fn)(struct pci_dev *pdev,
				     u16 alias, void *data), void *data)
{
    .....
	struct pci_bus *bus;
	int ret;

	ret = fn(pdev, PCI_DEVID(pdev->bus->number, pdev->devfn), data);
    .....
}


static int iort_pci_iommu_init(struct pci_dev *pdev, u16 alias, void *data)
{
	struct iort_pci_alias_info *info = data;
	struct acpi_iort_node *parent;
	u32 streamid;

	parent = iort_node_map_id(info->node, alias, &streamid,
				  IORT_IOMMU_TYPE);
	return iort_iommu_xlate(info->dev, parent, streamid);
}

最终：
static int arm_smmu_iort_xlate(struct device *dev, u32 streamid,
			       struct fwnode_handle *fwnode,
			       const struct iommu_ops *ops)
{
	int ret = iommu_fwspec_init(dev, fwnode, ops);

	if (!ret)
		ret = iommu_fwspec_add_ids(dev, &streamid, 1);  //添加streamid

	return ret;
}

arm_smmu_add_device

在drivers/iommu/arm-smmu.c的 arm_smmu_add_device函数中：

#ifdef CONFIG_ARCH_PHYTIUM
#define FWID_READ(id) (((u16)(id) >> 3) | (((id) >> SMR_MASK_SHIFT | 0x7000) << SMR_MASK_SHIFT))
#endif
//默认streamID会使用pci的bdf，但ft2000+ 硬件实现上不同，该值会通过FWID_READ函数进行调整。iommu_fwspec_add_ids加入到fwspec中。因此一个设备会有两个sid。

static int arm_smmu_add_device(struct device *dev)
{
        struct arm_smmu_device *smmu;
        struct arm_smmu_master_cfg *cfg;
        struct iommu_fwspec *fwspec = dev->iommu_fwspec;
        int i, ret;

#ifdef CONFIG_ARCH_PHYTIUM
        /* FT2000PLUS workaround patch */
        if ((read_cpuid_id() & MIDR_CPU_MODEL_MASK) == MIDR_PHYTIUM_FT2000PLUS) {
                printk("===arm_smmu_add_device ft2000+ \n");
                int num = fwspec->num_ids;
                for (i = 0; i < num; i++) {
                        u32 fwid = FWID_READ(fwspec->ids[i]);
                        printk("===arm_smmu_add_device ft2000+ ids %llx to fwid %llx \n",fwspec->ids[i],fwid);
                        iommu_fwspec_add_ids(dev, &fwid, 1);
                }
        }
#endif

..............
打印一下设备的sid
       for (i = 0; i < fwspec->num_ids; i++) {
                u16 sid = fwspec->ids[i];
                u16 mask = fwspe