[Freedreno] [PATCH 3/7] iommu/arm-smmu: Add support for TTBR1
Jordan Crouse
jcrouse at codeaurora.org
Tue Mar 7 16:39:51 UTC 2017
Allow a SMMU device to opt into allocating a TTBR1 pagetable.
The size of the TTBR1 region will be the same as
the TTBR0 size with the sign extension bit set on the highest
bit in the region unless the upstream size is 49 bits and then
the sign-extension bit will be set on the 49th bit.
The map/unmap operations will automatically use the appropriate
pagetable based on the specified iova and the existing mask.
Signed-off-by: Jordan Crouse <jcrouse at codeaurora.org>
---
drivers/iommu/arm-smmu.c | 19 ++++-
drivers/iommu/io-pgtable-arm.c | 168 +++++++++++++++++++++++++++++++++++++----
drivers/iommu/io-pgtable.h | 6 ++
3 files changed, 173 insertions(+), 20 deletions(-)
diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
index c47f883..2e3879f 100644
--- a/drivers/iommu/arm-smmu.c
+++ b/drivers/iommu/arm-smmu.c
@@ -256,9 +256,6 @@ enum arm_smmu_s2cr_privcfg {
#define RESUME_RETRY (0 << 0)
#define RESUME_TERMINATE (1 << 0)
-#define TTBCR2_SEP_SHIFT 15
-#define TTBCR2_SEP_UPSTREAM (0x7 << TTBCR2_SEP_SHIFT)
-
#define TTBRn_ASID_SHIFT 48
#define FSR_MULTI (1 << 31)
@@ -414,6 +411,7 @@ struct arm_smmu_domain {
struct arm_smmu_cfg cfg;
enum arm_smmu_domain_stage stage;
struct mutex init_mutex; /* Protects smmu pointer */
+ u32 attributes;
struct iommu_domain domain;
};
@@ -803,7 +801,6 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
} else {
reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
reg2 = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
- reg2 |= TTBCR2_SEP_UPSTREAM;
}
if (smmu->version > ARM_SMMU_V1)
writel_relaxed(reg2, cb_base + ARM_SMMU_CB_TTBCR2);
@@ -844,6 +841,9 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
enum io_pgtable_fmt fmt;
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
+ unsigned int quirks =
+ smmu_domain->attributes & (1 << DOMAIN_ATTR_ENABLE_TTBR1) ?
+ IO_PGTABLE_QUIRK_ARM_TTBR1 : 0;
mutex_lock(&smmu_domain->init_mutex);
if (smmu_domain->smmu)
@@ -953,6 +953,7 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
}
smmu_domain->pgtbl_cfg = (struct io_pgtable_cfg) {
+ .quirks = quirks,
.pgsize_bitmap = smmu->pgsize_bitmap,
.ias = ias,
.oas = oas,
@@ -1539,6 +1540,10 @@ static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
case DOMAIN_ATTR_NESTING:
*(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
return 0;
+ case DOMAIN_ATTR_ENABLE_TTBR1:
+ *((int *)data) = !!(smmu_domain->attributes
+ & (1 << DOMAIN_ATTR_ENABLE_TTBR1));
+ return 0;
default:
return -ENODEV;
}
@@ -1565,6 +1570,12 @@ static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
break;
+ case DOMAIN_ATTR_ENABLE_TTBR1:
+ if (*((int *)data))
+ smmu_domain->attributes |=
+ 1 << DOMAIN_ATTR_ENABLE_TTBR1;
+ ret = 0;
+ break;
default:
ret = -ENODEV;
}
diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c
index f5c90e1..110a691 100644
--- a/drivers/iommu/io-pgtable-arm.c
+++ b/drivers/iommu/io-pgtable-arm.c
@@ -124,14 +124,21 @@
#define ARM_LPAE_TCR_TG0_64K (1 << 14)
#define ARM_LPAE_TCR_TG0_16K (2 << 14)
+#define ARM_LPAE_TCR_TG1_16K 1ULL
+#define ARM_LPAE_TCR_TG1_4K 2ULL
+#define ARM_LPAE_TCR_TG1_64K 3ULL
+
#define ARM_LPAE_TCR_SH0_SHIFT 12
#define ARM_LPAE_TCR_SH0_MASK 0x3
+#define ARM_LPAE_TCR_SH1_SHIFT 28
#define ARM_LPAE_TCR_SH_NS 0
#define ARM_LPAE_TCR_SH_OS 2
#define ARM_LPAE_TCR_SH_IS 3
#define ARM_LPAE_TCR_ORGN0_SHIFT 10
+#define ARM_LPAE_TCR_ORGN1_SHIFT 26
#define ARM_LPAE_TCR_IRGN0_SHIFT 8
+#define ARM_LPAE_TCR_IRGN1_SHIFT 24
#define ARM_LPAE_TCR_RGN_MASK 0x3
#define ARM_LPAE_TCR_RGN_NC 0
#define ARM_LPAE_TCR_RGN_WBWA 1
@@ -144,6 +151,9 @@
#define ARM_LPAE_TCR_T0SZ_SHIFT 0
#define ARM_LPAE_TCR_SZ_MASK 0xf
+#define ARM_LPAE_TCR_T1SZ_SHIFT 16
+#define ARM_LPAE_TCR_T1SZ_MASK 0x3f
+
#define ARM_LPAE_TCR_PS_SHIFT 16
#define ARM_LPAE_TCR_PS_MASK 0x7
@@ -157,6 +167,16 @@
#define ARM_LPAE_TCR_PS_44_BIT 0x4ULL
#define ARM_LPAE_TCR_PS_48_BIT 0x5ULL
+#define ARM_LPAE_TCR_SEP_SHIFT (15 + 32)
+
+#define ARM_LPAE_TCR_SEP_31 0x0ULL
+#define ARM_LPAE_TCR_SEP_35 0x1ULL
+#define ARM_LPAE_TCR_SEP_39 0x2ULL
+#define ARM_LPAE_TCR_SEP_41 0x3ULL
+#define ARM_LPAE_TCR_SEP_43 0x4ULL
+#define ARM_LPAE_TCR_SEP_47 0x5ULL
+#define ARM_LPAE_TCR_SEP_UPSTREAM 0x7ULL
+
#define ARM_LPAE_MAIR_ATTR_SHIFT(n) ((n) << 3)
#define ARM_LPAE_MAIR_ATTR_MASK 0xff
#define ARM_LPAE_MAIR_ATTR_DEVICE 0x04
@@ -195,7 +215,7 @@ struct arm_lpae_io_pgtable {
unsigned long pg_shift;
unsigned long bits_per_level;
- void *pgd;
+ void *pgd[2];
};
typedef u64 arm_lpae_iopte;
@@ -381,14 +401,38 @@ static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data,
return pte;
}
+static inline arm_lpae_iopte *arm_lpae_get_table(
+ struct arm_lpae_io_pgtable *data, unsigned long iova)
+{
+ struct io_pgtable_cfg *cfg = &data->iop.cfg;
+
+ if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) {
+ unsigned long mask;
+
+ /*
+ * IAS of 48 means sign extension bit 48, otherwise it is the
+ * ias - 1 (32 -> bit 31, etc)
+ */
+ mask = (cfg->ias == 48) ? (1UL << 48) :
+ (1UL << (cfg->ias - 1));
+
+ if (iova & mask)
+ return data->pgd[1];
+ }
+
+ return data->pgd[0];
+}
+
static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova,
phys_addr_t paddr, size_t size, int iommu_prot)
{
struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
- arm_lpae_iopte *ptep = data->pgd;
+ arm_lpae_iopte *ptep;
int ret, lvl = ARM_LPAE_START_LVL(data);
arm_lpae_iopte prot;
+ ptep = arm_lpae_get_table(data, iova);
+
/* If no access, then nothing to do */
if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE)))
return 0;
@@ -439,7 +483,10 @@ static void arm_lpae_free_pgtable(struct io_pgtable *iop)
{
struct arm_lpae_io_pgtable *data = io_pgtable_to_data(iop);
- __arm_lpae_free_pgtable(data, ARM_LPAE_START_LVL(data), data->pgd);
+ __arm_lpae_free_pgtable(data, ARM_LPAE_START_LVL(data), data->pgd[0]);
+ if (data->pgd[1])
+ __arm_lpae_free_pgtable(data, ARM_LPAE_START_LVL(data),
+ data->pgd[1]);
kfree(data);
}
@@ -535,9 +582,11 @@ static int arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova,
{
size_t unmapped;
struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
- arm_lpae_iopte *ptep = data->pgd;
+ arm_lpae_iopte *ptep;
int lvl = ARM_LPAE_START_LVL(data);
+ ptep = arm_lpae_get_table(data, iova);
+
unmapped = __arm_lpae_unmap(data, iova, size, lvl, ptep);
if (unmapped)
io_pgtable_tlb_sync(&data->iop);
@@ -549,9 +598,11 @@ static phys_addr_t arm_lpae_iova_to_phys(struct io_pgtable_ops *ops,
unsigned long iova)
{
struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
- arm_lpae_iopte pte, *ptep = data->pgd;
+ arm_lpae_iopte pte, *ptep;
int lvl = ARM_LPAE_START_LVL(data);
+ ptep = arm_lpae_get_table(data, iova);
+
do {
/* Valid IOPTE pointer? */
if (!ptep)
@@ -660,13 +711,81 @@ static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg)
return data;
}
+static u64 arm64_lpae_setup_ttbr1(struct io_pgtable_cfg *cfg,
+ struct arm_lpae_io_pgtable *data)
+
+{
+ u64 reg;
+
+ /* If TTBR1 is disabled, disable speculative walks through the TTBR1 */
+ if (!(cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)) {
+ reg = ARM_LPAE_TCR_EPD1;
+ reg |= (ARM_LPAE_TCR_SEP_UPSTREAM << ARM_LPAE_TCR_SEP_SHIFT);
+ return reg;
+ }
+
+ reg = (ARM_LPAE_TCR_SH_IS << ARM_LPAE_TCR_SH1_SHIFT) |
+ (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_IRGN1_SHIFT) |
+ (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN1_SHIFT);
+
+ switch (1 << data->pg_shift) {
+ case SZ_4K:
+ reg |= (ARM_LPAE_TCR_TG1_4K << 30);
+ break;
+ case SZ_16K:
+ reg |= (ARM_LPAE_TCR_TG1_16K << 30);
+ break;
+ case SZ_64K:
+ reg |= (ARM_LPAE_TCR_TG1_64K << 30);
+ break;
+ }
+
+ /* Set T1SZ */
+ reg |= (64ULL - cfg->ias) << ARM_LPAE_TCR_T1SZ_SHIFT;
+
+ /* Set the SEP bit based on the size */
+ switch (cfg->ias) {
+ case 32:
+ reg |= (ARM_LPAE_TCR_SEP_31 << ARM_LPAE_TCR_SEP_SHIFT);
+ break;
+ case 36:
+ reg |= (ARM_LPAE_TCR_SEP_35 << ARM_LPAE_TCR_SEP_SHIFT);
+ break;
+ case 40:
+ reg |= (ARM_LPAE_TCR_SEP_39 << ARM_LPAE_TCR_SEP_SHIFT);
+ break;
+ case 42:
+ reg |= (ARM_LPAE_TCR_SEP_41 << ARM_LPAE_TCR_SEP_SHIFT);
+ break;
+ case 44:
+ reg |= (ARM_LPAE_TCR_SEP_43 << ARM_LPAE_TCR_SEP_SHIFT);
+ break;
+ case 48:
+ /*
+ * If ias is 48 then that probably means that the UBS on the
+ * device was 0101b (49) which is a special case that assumes
+ * bit 48 is the sign extension bit. In this case we are
+ * expected to use ARM_LPAE_TCR_SEP_UPSTREAM to use bit 48 as
+ * the extension bit. One might be confused because there is
+ * also an option to set the SEP to bit 47 but this is probably
+ * not what the arm-smmu driver intended.
+ */
+ default:
+ reg |= (ARM_LPAE_TCR_SEP_UPSTREAM << ARM_LPAE_TCR_SEP_SHIFT);
+ break;
+ }
+
+ return reg;
+}
+
static struct io_pgtable *
arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)
{
u64 reg;
struct arm_lpae_io_pgtable *data;
- if (cfg->quirks & ~IO_PGTABLE_QUIRK_ARM_NS)
+ if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS |
+ IO_PGTABLE_QUIRK_ARM_TTBR1))
return NULL;
data = arm_lpae_alloc_pgtable(cfg);
@@ -715,8 +834,9 @@ static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg)
reg |= (64ULL - cfg->ias) << ARM_LPAE_TCR_T0SZ_SHIFT;
- /* Disable speculative walks through TTBR1 */
- reg |= ARM_LPAE_TCR_EPD1;
+ /* Bring in the TTBR1 configuration */
+ reg |= arm64_lpae_setup_ttbr1(cfg, data);
+
cfg->arm_lpae_s1_cfg.tcr = reg;
/* MAIRs */
@@ -731,16 +851,32 @@ static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg)
cfg->arm_lpae_s1_cfg.mair[1] = 0;
/* Looking good; allocate a pgd */
- data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg);
- if (!data->pgd)
+ data->pgd[0] = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg);
+ if (!data->pgd[0])
goto out_free_data;
+
+ if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) {
+ data->pgd[1] = __arm_lpae_alloc_pages(data->pgd_size,
+ GFP_KERNEL, cfg);
+ if (!data->pgd[1]) {
+ __arm_lpae_free_pages(data->pgd[0], data->pgd_size,
+ cfg);
+ goto out_free_data;
+ }
+ } else {
+ data->pgd[1] = NULL;
+ }
+
/* Ensure the empty pgd is visible before any actual TTBR write */
wmb();
/* TTBRs */
- cfg->arm_lpae_s1_cfg.ttbr[0] = virt_to_phys(data->pgd);
- cfg->arm_lpae_s1_cfg.ttbr[1] = 0;
+ cfg->arm_lpae_s1_cfg.ttbr[0] = virt_to_phys(data->pgd[0]);
+
+ if (data->pgd[1])
+ cfg->arm_lpae_s1_cfg.ttbr[1] = virt_to_phys(data->pgd[1]);
+
return &data->iop;
out_free_data:
@@ -825,15 +961,15 @@ static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg)
cfg->arm_lpae_s2_cfg.vtcr = reg;
/* Allocate pgd pages */
- data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg);
- if (!data->pgd)
+ data->pgd[0] = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg);
+ if (!data->pgd[0])
goto out_free_data;
/* Ensure the empty pgd is visible before any actual TTBR write */
wmb();
/* VTTBR */
- cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd);
+ cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd[0]);
return &data->iop;
out_free_data:
@@ -931,7 +1067,7 @@ static void __init arm_lpae_dump_ops(struct io_pgtable_ops *ops)
cfg->pgsize_bitmap, cfg->ias);
pr_err("data: %d levels, 0x%zx pgd_size, %lu pg_shift, %lu bits_per_level, pgd @ %p\n",
data->levels, data->pgd_size, data->pg_shift,
- data->bits_per_level, data->pgd);
+ data->bits_per_level, data->pgd[0]);
}
#define __FAIL(ops, i) ({ \
diff --git a/drivers/iommu/io-pgtable.h b/drivers/iommu/io-pgtable.h
index 969d82c..fd2cd34 100644
--- a/drivers/iommu/io-pgtable.h
+++ b/drivers/iommu/io-pgtable.h
@@ -65,11 +65,17 @@ struct io_pgtable_cfg {
* PTEs, for Mediatek IOMMUs which treat it as a 33rd address bit
* when the SoC is in "4GB mode" and they can only access the high
* remap of DRAM (0x1_00000000 to 0x1_ffffffff).
+ *
+ * IO_PGTABLE_QUIRK_ARM_TTBR1: Specifies that TTBR1 has been enabled on
+ * this domain. Set up the configuration registers and dyanmically
+ * choose which pagetable (TTBR0 or TTBR1) a mapping should go into
+ * based on the address.
*/
#define IO_PGTABLE_QUIRK_ARM_NS BIT(0)
#define IO_PGTABLE_QUIRK_NO_PERMS BIT(1)
#define IO_PGTABLE_QUIRK_TLBI_ON_MAP BIT(2)
#define IO_PGTABLE_QUIRK_ARM_MTK_4GB BIT(3)
+ #define IO_PGTABLE_QUIRK_ARM_TTBR1 BIT(4)
unsigned long quirks;
unsigned long pgsize_bitmap;
unsigned int ias;
--
1.9.1
More information about the Freedreno
mailing list