Skip to content

Commit a69483e

Browse files
willdeaconwilldeacon
committed
Merge branch 'for-next/8.6-timers' into for-next/core
* for-next/8.6-timers: arm64: Add HWCAP for self-synchronising virtual counter arm64: Add handling of CNTVCTSS traps arm64: Add CNT{P,V}CTSS_EL0 alternatives to cnt{p,v}ct_el0 arm64: Add a capability for FEAT_ECV clocksource/drivers/arch_arm_timer: Move workaround synchronisation around clocksource/drivers/arm_arch_timer: Fix masking for high freq counters clocksource/drivers/arm_arch_timer: Drop unnecessary ISB on CVAL programming clocksource/drivers/arm_arch_timer: Remove any trace of the TVAL programming interface clocksource/drivers/arm_arch_timer: Work around broken CVAL implementations clocksource/drivers/arm_arch_timer: Advertise 56bit timer to the core code clocksource/drivers/arm_arch_timer: Move MMIO timer programming over to CVAL clocksource/drivers/arm_arch_timer: Fix MMIO base address vs callback ordering issue clocksource/drivers/arm_arch_timer: Move drop _tval from erratum function names clocksource/drivers/arm_arch_timer: Move system register timer programming over to CVAL clocksource/drivers/arm_arch_timer: Extend write side of timer register accessors to u64 clocksource/drivers/arm_arch_timer: Drop CNT*_TVAL read accessors clocksource/arm_arch_timer: Add build-time guards for unhandled register accesses
2 parents 5816b3e + fee29f0 commit a69483e

File tree

14 files changed

+264
-149
lines changed

14 files changed

+264
-149
lines changed

‎Documentation/arm64/cpu-feature-registers.rst

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -235,15 +235,23 @@ infrastructure:
235235
| DPB | [3-0] | y |
236236
+------------------------------+---------+---------+
237237

238-
6) ID_AA64MMFR2_EL1 - Memory model feature register 2
238+
6) ID_AA64MMFR0_EL1 - Memory model feature register 0
239+
240+
+------------------------------+---------+---------+
241+
| Name | bits | visible |
242+
+------------------------------+---------+---------+
243+
| ECV | [63-60] | y |
244+
+------------------------------+---------+---------+
245+
246+
7) ID_AA64MMFR2_EL1 - Memory model feature register 2
239247

240248
+------------------------------+---------+---------+
241249
| Name | bits | visible |
242250
+------------------------------+---------+---------+
243251
| AT | [35-32] | y |
244252
+------------------------------+---------+---------+
245253

246-
7) ID_AA64ZFR0_EL1 - SVE feature ID register 0
254+
8) ID_AA64ZFR0_EL1 - SVE feature ID register 0
247255

248256
+------------------------------+---------+---------+
249257
| Name | bits | visible |

‎Documentation/arm64/elf_hwcaps.rst

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,10 @@ HWCAP2_MTE
247247
Functionality implied by ID_AA64PFR1_EL1.MTE == 0b0010, as described
248248
by Documentation/arm64/memory-tagging-extension.rst.
249249

250+
HWCAP2_ECV
251+
252+
Functionality implied by ID_AA64MMFR0_EL1.ECV == 0b0001.
253+
250254
4. Unused AT_HWCAP bits
251255
-----------------------
252256

‎arch/arm/include/asm/arch_timer.h

Lines changed: 22 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
#include <asm/hwcap.h>
88
#include <linux/clocksource.h>
99
#include <linux/init.h>
10+
#include <linux/io-64-nonatomic-lo-hi.h>
1011
#include <linux/types.h>
1112

1213
#include <clocksource/arm_arch_timer.h>
@@ -24,29 +25,35 @@ int arch_timer_arch_init(void);
2425
* the code. At least it does so with a recent GCC (4.6.3).
2526
*/
2627
static __always_inline
27-
void arch_timer_reg_write_cp15(int access, enum arch_timer_reg reg, u32 val)
28+
void arch_timer_reg_write_cp15(int access, enum arch_timer_reg reg, u64 val)
2829
{
2930
if (access == ARCH_TIMER_PHYS_ACCESS) {
3031
switch (reg) {
3132
case ARCH_TIMER_REG_CTRL:
32-
asm volatile("mcr p15, 0, %0, c14, c2, 1" : : "r" (val));
33+
asm volatile("mcr p15, 0, %0, c14, c2, 1" : : "r" ((u32)val));
34+
isb();
3335
break;
34-
case ARCH_TIMER_REG_TVAL:
35-
asm volatile("mcr p15, 0, %0, c14, c2, 0" : : "r" (val));
36+
case ARCH_TIMER_REG_CVAL:
37+
asm volatile("mcrr p15, 2, %Q0, %R0, c14" : : "r" (val));
3638
break;
39+
default:
40+
BUILD_BUG();
3741
}
3842
} else if (access == ARCH_TIMER_VIRT_ACCESS) {
3943
switch (reg) {
4044
case ARCH_TIMER_REG_CTRL:
41-
asm volatile("mcr p15, 0, %0, c14, c3, 1" : : "r" (val));
45+
asm volatile("mcr p15, 0, %0, c14, c3, 1" : : "r" ((u32)val));
46+
isb();
4247
break;
43-
case ARCH_TIMER_REG_TVAL:
44-
asm volatile("mcr p15, 0, %0, c14, c3, 0" : : "r" (val));
48+
case ARCH_TIMER_REG_CVAL:
49+
asm volatile("mcrr p15, 3, %Q0, %R0, c14" : : "r" (val));
4550
break;
51+
default:
52+
BUILD_BUG();
4653
}
54+
} else {
55+
BUILD_BUG();
4756
}
48-
49-
isb();
5057
}
5158

5259
static __always_inline
@@ -59,19 +66,19 @@ u32 arch_timer_reg_read_cp15(int access, enum arch_timer_reg reg)
5966
case ARCH_TIMER_REG_CTRL:
6067
asm volatile("mrc p15, 0, %0, c14, c2, 1" : "=r" (val));
6168
break;
62-
case ARCH_TIMER_REG_TVAL:
63-
asm volatile("mrc p15, 0, %0, c14, c2, 0" : "=r" (val));
64-
break;
69+
default:
70+
BUILD_BUG();
6571
}
6672
} else if (access == ARCH_TIMER_VIRT_ACCESS) {
6773
switch (reg) {
6874
case ARCH_TIMER_REG_CTRL:
6975
asm volatile("mrc p15, 0, %0, c14, c3, 1" : "=r" (val));
7076
break;
71-
case ARCH_TIMER_REG_TVAL:
72-
asm volatile("mrc p15, 0, %0, c14, c3, 0" : "=r" (val));
73-
break;
77+
default:
78+
BUILD_BUG();
7479
}
80+
} else {
81+
BUILD_BUG();
7582
}
7683

7784
return val;

‎arch/arm64/include/asm/arch_timer.h

Lines changed: 44 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
({ \
3333
const struct arch_timer_erratum_workaround *__wa; \
3434
__wa = __this_cpu_read(timer_unstable_counter_workaround); \
35-
(__wa && __wa->h) ? __wa->h : arch_timer_##h; \
35+
(__wa && __wa->h) ? ({ isb(); __wa->h;}) : arch_timer_##h; \
3636
})
3737

3838
#else
@@ -52,8 +52,6 @@ struct arch_timer_erratum_workaround {
5252
enum arch_timer_erratum_match_type match_type;
5353
const void *id;
5454
const char *desc;
55-
u32 (*read_cntp_tval_el0)(void);
56-
u32 (*read_cntv_tval_el0)(void);
5755
u64 (*read_cntpct_el0)(void);
5856
u64 (*read_cntvct_el0)(void);
5957
int (*set_next_event_phys)(unsigned long, struct clock_event_device *);
@@ -64,25 +62,28 @@ struct arch_timer_erratum_workaround {
6462
DECLARE_PER_CPU(const struct arch_timer_erratum_workaround *,
6563
timer_unstable_counter_workaround);
6664

67-
/* inline sysreg accessors that make erratum_handler() work */
68-
static inline notrace u32 arch_timer_read_cntp_tval_el0(void)
65+
static inline notrace u64 arch_timer_read_cntpct_el0(void)
6966
{
70-
return read_sysreg(cntp_tval_el0);
71-
}
67+
u64 cnt;
7268

73-
static inline notrace u32 arch_timer_read_cntv_tval_el0(void)
74-
{
75-
return read_sysreg(cntv_tval_el0);
76-
}
69+
asm volatile(ALTERNATIVE("isb\n mrs %0, cntpct_el0",
70+
"nop\n" __mrs_s("%0", SYS_CNTPCTSS_EL0),
71+
ARM64_HAS_ECV)
72+
: "=r" (cnt));
7773

78-
static inline notrace u64 arch_timer_read_cntpct_el0(void)
79-
{
80-
return read_sysreg(cntpct_el0);
74+
return cnt;
8175
}
8276

8377
static inline notrace u64 arch_timer_read_cntvct_el0(void)
8478
{
85-
return read_sysreg(cntvct_el0);
79+
u64 cnt;
80+
81+
asm volatile(ALTERNATIVE("isb\n mrs %0, cntvct_el0",
82+
"nop\n" __mrs_s("%0", SYS_CNTVCTSS_EL0),
83+
ARM64_HAS_ECV)
84+
: "=r" (cnt));
85+
86+
return cnt;
8687
}
8788

8889
#define arch_timer_reg_read_stable(reg) \
@@ -102,51 +103,58 @@ static inline notrace u64 arch_timer_read_cntvct_el0(void)
102103
* the code.
103104
*/
104105
static __always_inline
105-
void arch_timer_reg_write_cp15(int access, enum arch_timer_reg reg, u32 val)
106+
void arch_timer_reg_write_cp15(int access, enum arch_timer_reg reg, u64 val)
106107
{
107108
if (access == ARCH_TIMER_PHYS_ACCESS) {
108109
switch (reg) {
109110
case ARCH_TIMER_REG_CTRL:
110111
write_sysreg(val, cntp_ctl_el0);
112+
isb();
111113
break;
112-
case ARCH_TIMER_REG_TVAL:
113-
write_sysreg(val, cntp_tval_el0);
114+
case ARCH_TIMER_REG_CVAL:
115+
write_sysreg(val, cntp_cval_el0);
114116
break;
117+
default:
118+
BUILD_BUG();
115119
}
116120
} else if (access == ARCH_TIMER_VIRT_ACCESS) {
117121
switch (reg) {
118122
case ARCH_TIMER_REG_CTRL:
119123
write_sysreg(val, cntv_ctl_el0);
124+
isb();
120125
break;
121-
case ARCH_TIMER_REG_TVAL:
122-
write_sysreg(val, cntv_tval_el0);
126+
case ARCH_TIMER_REG_CVAL:
127+
write_sysreg(val, cntv_cval_el0);
123128
break;
129+
default:
130+
BUILD_BUG();
124131
}
132+
} else {
133+
BUILD_BUG();
125134
}
126-
127-
isb();
128135
}
129136

130137
static __always_inline
131-
u32 arch_timer_reg_read_cp15(int access, enum arch_timer_reg reg)
138+
u64 arch_timer_reg_read_cp15(int access, enum arch_timer_reg reg)
132139
{
133140
if (access == ARCH_TIMER_PHYS_ACCESS) {
134141
switch (reg) {
135142
case ARCH_TIMER_REG_CTRL:
136143
return read_sysreg(cntp_ctl_el0);
137-
case ARCH_TIMER_REG_TVAL:
138-
return arch_timer_reg_read_stable(cntp_tval_el0);
144+
default:
145+
BUILD_BUG();
139146
}
140147
} else if (access == ARCH_TIMER_VIRT_ACCESS) {
141148
switch (reg) {
142149
case ARCH_TIMER_REG_CTRL:
143150
return read_sysreg(cntv_ctl_el0);
144-
case ARCH_TIMER_REG_TVAL:
145-
return arch_timer_reg_read_stable(cntv_tval_el0);
151+
default:
152+
BUILD_BUG();
146153
}
147154
}
148155

149-
BUG();
156+
BUILD_BUG();
157+
unreachable();
150158
}
151159

152160
static inline u32 arch_timer_get_cntfrq(void)
@@ -169,7 +177,6 @@ static __always_inline u64 __arch_counter_get_cntpct_stable(void)
169177
{
170178
u64 cnt;
171179

172-
isb();
173180
cnt = arch_timer_reg_read_stable(cntpct_el0);
174181
arch_counter_enforce_ordering(cnt);
175182
return cnt;
@@ -179,8 +186,10 @@ static __always_inline u64 __arch_counter_get_cntpct(void)
179186
{
180187
u64 cnt;
181188

182-
isb();
183-
cnt = read_sysreg(cntpct_el0);
189+
asm volatile(ALTERNATIVE("isb\n mrs %0, cntpct_el0",
190+
"nop\n" __mrs_s("%0", SYS_CNTPCTSS_EL0),
191+
ARM64_HAS_ECV)
192+
: "=r" (cnt));
184193
arch_counter_enforce_ordering(cnt);
185194
return cnt;
186195
}
@@ -189,7 +198,6 @@ static __always_inline u64 __arch_counter_get_cntvct_stable(void)
189198
{
190199
u64 cnt;
191200

192-
isb();
193201
cnt = arch_timer_reg_read_stable(cntvct_el0);
194202
arch_counter_enforce_ordering(cnt);
195203
return cnt;
@@ -199,8 +207,10 @@ static __always_inline u64 __arch_counter_get_cntvct(void)
199207
{
200208
u64 cnt;
201209

202-
isb();
203-
cnt = read_sysreg(cntvct_el0);
210+
asm volatile(ALTERNATIVE("isb\n mrs %0, cntvct_el0",
211+
"nop\n" __mrs_s("%0", SYS_CNTVCTSS_EL0),
212+
ARM64_HAS_ECV)
213+
: "=r" (cnt));
204214
arch_counter_enforce_ordering(cnt);
205215
return cnt;
206216
}

‎arch/arm64/include/asm/esr.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,9 @@
227227
#define ESR_ELx_SYS64_ISS_SYS_CNTVCT (ESR_ELx_SYS64_ISS_SYS_VAL(3, 3, 2, 14, 0) | \
228228
ESR_ELx_SYS64_ISS_DIR_READ)
229229

230+
#define ESR_ELx_SYS64_ISS_SYS_CNTVCTSS (ESR_ELx_SYS64_ISS_SYS_VAL(3, 3, 6, 14, 0) | \
231+
ESR_ELx_SYS64_ISS_DIR_READ)
232+
230233
#define ESR_ELx_SYS64_ISS_SYS_CNTFRQ (ESR_ELx_SYS64_ISS_SYS_VAL(3, 3, 0, 14, 0) | \
231234
ESR_ELx_SYS64_ISS_DIR_READ)
232235

@@ -317,6 +320,9 @@
317320
#define ESR_ELx_CP15_64_ISS_SYS_CNTVCT (ESR_ELx_CP15_64_ISS_SYS_VAL(1, 14) | \
318321
ESR_ELx_CP15_64_ISS_DIR_READ)
319322

323+
#define ESR_ELx_CP15_64_ISS_SYS_CNTVCTSS (ESR_ELx_CP15_64_ISS_SYS_VAL(9, 14) | \
324+
ESR_ELx_CP15_64_ISS_DIR_READ)
325+
320326
#define ESR_ELx_CP15_32_ISS_SYS_CNTFRQ (ESR_ELx_CP15_32_ISS_SYS_VAL(0, 0, 14, 0) |\
321327
ESR_ELx_CP15_32_ISS_DIR_READ)
322328

‎arch/arm64/include/asm/hwcap.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@
105105
#define KERNEL_HWCAP_RNG __khwcap2_feature(RNG)
106106
#define KERNEL_HWCAP_BTI __khwcap2_feature(BTI)
107107
#define KERNEL_HWCAP_MTE __khwcap2_feature(MTE)
108+
#define KERNEL_HWCAP_ECV __khwcap2_feature(ECV)
108109

109110
/*
110111
* This yields a mask that user programs can use to figure out what

‎arch/arm64/include/asm/sysreg.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -507,6 +507,9 @@
507507

508508
#define SYS_CNTFRQ_EL0 sys_reg(3, 3, 14, 0, 0)
509509

510+
#define SYS_CNTPCTSS_EL0 sys_reg(3, 3, 14, 0, 5)
511+
#define SYS_CNTVCTSS_EL0 sys_reg(3, 3, 14, 0, 6)
512+
510513
#define SYS_CNTP_TVAL_EL0 sys_reg(3, 3, 14, 2, 0)
511514
#define SYS_CNTP_CTL_EL0 sys_reg(3, 3, 14, 2, 1)
512515
#define SYS_CNTP_CVAL_EL0 sys_reg(3, 3, 14, 2, 2)

‎arch/arm64/include/uapi/asm/hwcap.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,5 +75,6 @@
7575
#define HWCAP2_RNG (1 << 16)
7676
#define HWCAP2_BTI (1 << 17)
7777
#define HWCAP2_MTE (1 << 18)
78+
#define HWCAP2_ECV (1 << 19)
7879

7980
#endif /* _UAPI__ASM_HWCAP_H */

‎arch/arm64/kernel/cpufeature.c

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -279,7 +279,7 @@ static const struct arm64_ftr_bits ftr_id_aa64zfr0[] = {
279279
};
280280

281281
static const struct arm64_ftr_bits ftr_id_aa64mmfr0[] = {
282-
ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_ECV_SHIFT, 4, 0),
282+
ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_ECV_SHIFT, 4, 0),
283283
ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_FGT_SHIFT, 4, 0),
284284
ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EXS_SHIFT, 4, 0),
285285
/*
@@ -1930,6 +1930,16 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
19301930
.sign = FTR_UNSIGNED,
19311931
.min_field_value = 1,
19321932
},
1933+
{
1934+
.desc = "Enhanced Counter Virtualization",
1935+
.capability = ARM64_HAS_ECV,
1936+
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
1937+
.matches = has_cpuid_feature,
1938+
.sys_reg = SYS_ID_AA64MMFR0_EL1,
1939+
.field_pos = ID_AA64MMFR0_ECV_SHIFT,
1940+
.sign = FTR_UNSIGNED,
1941+
.min_field_value = 1,
1942+
},
19331943
#ifdef CONFIG_ARM64_PAN
19341944
{
19351945
.desc = "Privileged Access Never",
@@ -2451,6 +2461,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
24512461
#ifdef CONFIG_ARM64_MTE
24522462
HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_MTE_SHIFT, FTR_UNSIGNED, ID_AA64PFR1_MTE, CAP_HWCAP, KERNEL_HWCAP_MTE),
24532463
#endif /* CONFIG_ARM64_MTE */
2464+
HWCAP_CAP(SYS_ID_AA64MMFR0_EL1, ID_AA64MMFR0_ECV_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ECV),
24542465
{},
24552466
};
24562467

‎arch/arm64/kernel/cpuinfo.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ static const char *const hwcap_str[] = {
9494
[KERNEL_HWCAP_RNG] = "rng",
9595
[KERNEL_HWCAP_BTI] = "bti",
9696
[KERNEL_HWCAP_MTE] = "mte",
97+
[KERNEL_HWCAP_ECV] = "ecv",
9798
};
9899

99100
#ifdef CONFIG_COMPAT

0 commit comments

Comments
 (0)