diff --git a/arch/arm64/include/asm/cache.h b/arch/arm64/include/asm/cache.h
index b2e6ece237130b50e8a0554cdeea8fb1f61dd73b..5df5cfe1c1431a763657a19339150b0f5b788159 100644
--- a/arch/arm64/include/asm/cache.h
+++ b/arch/arm64/include/asm/cache.h
@@ -20,8 +20,12 @@
#define CTR_L1IP_SHIFT 14
#define CTR_L1IP_MASK 3
+#define CTR_DMINLINE_SHIFT 16
+#define CTR_ERG_SHIFT 20
#define CTR_CWG_SHIFT 24
#define CTR_CWG_MASK 15
+#define CTR_IDC_SHIFT 28
+#define CTR_DIC_SHIFT 29
#define CTR_L1IP(ctr) (((ctr) >> CTR_L1IP_SHIFT) & CTR_L1IP_MASK)
diff --git a/arch/arm64/include/asm/cacheflush.h b/arch/arm64/include/asm/cacheflush.h
index bef9f418f08986830e68f6bc2c41c88eb5333a29..7dfcec4700fef0355372cbf7d4c2f11176c914d6 100644
--- a/arch/arm64/include/asm/cacheflush.h
+++ b/arch/arm64/include/asm/cacheflush.h
@@ -133,6 +133,9 @@ extern void flush_dcache_page(struct page *);
static inline void __flush_icache_all(void)
{
+ if (cpus_have_const_cap(ARM64_HAS_CACHE_DIC))
+ return;
+
asm("ic ialluis");
dsb(ish);
}
diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h
index 39134c46bb136d23a15bdedc4672838b62cef053..ff9fb3aba17bab52ddb0e0b92eb1cab282354ded 100644
--- a/arch/arm64/include/asm/cpucaps.h
+++ b/arch/arm64/include/asm/cpucaps.h
@@ -46,7 +46,9 @@
#define ARM64_HARDEN_BP_POST_GUEST_EXIT 25
#define ARM64_HAS_RAS_EXTN 26
#define ARM64_WORKAROUND_843419 27
+#define ARM64_HAS_CACHE_IDC 28
+#define ARM64_HAS_CACHE_DIC 29
-#define ARM64_NCAPS 28
+#define ARM64_NCAPS 30
#endif /* __ASM_CPUCAPS_H */
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index f96b3449034beb713620081789cc95ed283d7a81..bdab5522386613b8365d21cd52ebb761b4f24f7d 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -199,12 +199,12 @@ static const struct arm64_ftr_bits ftr_id_aa64mmfr2[] = {
};
static const struct arm64_ftr_bits ftr_ctr[] = {
- ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, 31, 1, 1), /* RES1 */
- ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, 29, 1, 1), /* DIC */
- ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, 28, 1, 1), /* IDC */
- ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_HIGHER_SAFE, 24, 4, 0), /* CWG */
- ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_HIGHER_SAFE, 20, 4, 0), /* ERG */
- ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, 16, 4, 1), /* DminLine */
+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, 31, 1, 1), /* RES1 */
+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_DIC_SHIFT, 1, 1),
+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_IDC_SHIFT, 1, 1),
+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_HIGHER_SAFE, CTR_CWG_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_HIGHER_SAFE, CTR_ERG_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_DMINLINE_SHIFT, 4, 1),
/*
* Linux can handle differing I-cache policies. Userspace JITs will
* make use of *minLine.
@@ -852,6 +852,18 @@ static bool has_no_fpsimd(const struct arm64_cpu_capabilities *entry, int __unus
ID_AA64PFR0_FP_SHIFT) < 0;
}
+static bool has_cache_idc(const struct arm64_cpu_capabilities *entry,
+ int __unused)
+{
+ return read_sanitised_ftr_reg(SYS_CTR_EL0) & BIT(CTR_IDC_SHIFT);
+}
+
+static bool has_cache_dic(const struct arm64_cpu_capabilities *entry,
+ int __unused)
+{
+ return read_sanitised_ftr_reg(SYS_CTR_EL0) & BIT(CTR_DIC_SHIFT);
+}
+
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
static int __kpti_forced; /* 0: not forced, >0: forced on, <0: forced off */
@@ -1088,6 +1100,18 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
.enable = cpu_clear_disr,
},
#endif /* CONFIG_ARM64_RAS_EXTN */
+ {
+ .desc = "Data cache clean to the PoU not required for I/D coherence",
+ .capability = ARM64_HAS_CACHE_IDC,
+ .def_scope = SCOPE_SYSTEM,
+ .matches = has_cache_idc,
+ },
+ {
+ .desc = "Instruction cache invalidation not required for I/D coherence",
+ .capability = ARM64_HAS_CACHE_DIC,
+ .def_scope = SCOPE_SYSTEM,
+ .matches = has_cache_dic,
+ },
{},
};
diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S
index 758bde7e2fa68a9d1a241857ad593b39a4d6e705..30334d81b0215347129898913d89fb5f26721513 100644
--- a/arch/arm64/mm/cache.S
+++ b/arch/arm64/mm/cache.S
@@ -50,6 +50,10 @@ ENTRY(flush_icache_range)
*/
ENTRY(__flush_cache_user_range)
uaccess_ttbr0_enable x2, x3, x4
+alternative_if ARM64_HAS_CACHE_IDC
+ dsb ishst
+ b 7f
+alternative_else_nop_endif
dcache_line_size x2, x3
sub x3, x2, #1
bic x4, x0, x3
@@ -60,8 +64,13 @@ user_alt 9f, "dc cvau, x4", "dc civac, x4", ARM64_WORKAROUND_CLEAN_CACHE
b.lo 1b
dsb ish
+7:
+alternative_if ARM64_HAS_CACHE_DIC
+ isb
+ b 8f
+alternative_else_nop_endif
invalidate_icache_by_line x0, x1, x2, x3, 9f
- mov x0, #0
+8: mov x0, #0
1:
uaccess_ttbr0_disable x1, x2
ret
@@ -80,6 +89,12 @@ ENDPROC(__flush_cache_user_range)
* - end - virtual end address of region
*/
ENTRY(invalidate_icache_range)
+alternative_if ARM64_HAS_CACHE_DIC
+ mov x0, xzr
+ isb
+ ret
+alternative_else_nop_endif
+
uaccess_ttbr0_enable x2, x3, x4
invalidate_icache_by_line x0, x1, x2, x3, 2f
@@ -116,6 +131,10 @@ ENDPIPROC(__flush_dcache_area)
* - size - size in question
*/
ENTRY(__clean_dcache_area_pou)
+alternative_if ARM64_HAS_CACHE_IDC
+ dsb ishst
+ ret
+alternative_else_nop_endif
dcache_by_line_op cvau, ish, x0, x1, x2, x3
ret
ENDPROC(__clean_dcache_area_pou)