Skip to content

Commit 6c9b7d7

Browse files
KAGA-KOKOsuryasaimadhu
KAGA-KOKO
authored and
suryasaimadhu
committed
x86/tlb: Move PCID helpers where they are used
Aside of the fact that they are used only in the TLB code, especially having the comment close to the actual implementation makes a lot of sense. Signed-off-by: Thomas Gleixner <[email protected]> Signed-off-by: Borislav Petkov <[email protected]> Reviewed-by: Alexandre Chartre <[email protected]> Acked-by: Peter Zijlstra (Intel) <[email protected]> Link: https://lkml.kernel.org/r/[email protected]
1 parent af5c40c commit 6c9b7d7

File tree

2 files changed

+126
-127
lines changed

2 files changed

+126
-127
lines changed

‎arch/x86/include/asm/tlbflush.h

Lines changed: 6 additions & 127 deletions
Original file line numberDiff line numberDiff line change
@@ -13,133 +13,6 @@
1313
#include <asm/pti.h>
1414
#include <asm/processor-flags.h>
1515

16-
/*
17-
* The x86 feature is called PCID (Process Context IDentifier). It is similar
18-
* to what is traditionally called ASID on the RISC processors.
19-
*
20-
* We don't use the traditional ASID implementation, where each process/mm gets
21-
* its own ASID and flush/restart when we run out of ASID space.
22-
*
23-
* Instead we have a small per-cpu array of ASIDs and cache the last few mm's
24-
* that came by on this CPU, allowing cheaper switch_mm between processes on
25-
* this CPU.
26-
*
27-
* We end up with different spaces for different things. To avoid confusion we
28-
* use different names for each of them:
29-
*
30-
* ASID - [0, TLB_NR_DYN_ASIDS-1]
31-
* the canonical identifier for an mm
32-
*
33-
* kPCID - [1, TLB_NR_DYN_ASIDS]
34-
* the value we write into the PCID part of CR3; corresponds to the
35-
* ASID+1, because PCID 0 is special.
36-
*
37-
* uPCID - [2048 + 1, 2048 + TLB_NR_DYN_ASIDS]
38-
* for KPTI each mm has two address spaces and thus needs two
39-
* PCID values, but we can still do with a single ASID denomination
40-
* for each mm. Corresponds to kPCID + 2048.
41-
*
42-
*/
43-
44-
/* There are 12 bits of space for ASIDS in CR3 */
45-
#define CR3_HW_ASID_BITS 12
46-
47-
/*
48-
* When enabled, PAGE_TABLE_ISOLATION consumes a single bit for
49-
* user/kernel switches
50-
*/
51-
#ifdef CONFIG_PAGE_TABLE_ISOLATION
52-
# define PTI_CONSUMED_PCID_BITS 1
53-
#else
54-
# define PTI_CONSUMED_PCID_BITS 0
55-
#endif
56-
57-
#define CR3_AVAIL_PCID_BITS (X86_CR3_PCID_BITS - PTI_CONSUMED_PCID_BITS)
58-
59-
/*
60-
* ASIDs are zero-based: 0->MAX_AVAIL_ASID are valid. -1 below to account
61-
* for them being zero-based. Another -1 is because PCID 0 is reserved for
62-
* use by non-PCID-aware users.
63-
*/
64-
#define MAX_ASID_AVAILABLE ((1 << CR3_AVAIL_PCID_BITS) - 2)
65-
66-
/*
67-
* 6 because 6 should be plenty and struct tlb_state will fit in two cache
68-
* lines.
69-
*/
70-
#define TLB_NR_DYN_ASIDS 6
71-
72-
/*
73-
* Given @asid, compute kPCID
74-
*/
75-
static inline u16 kern_pcid(u16 asid)
76-
{
77-
VM_WARN_ON_ONCE(asid > MAX_ASID_AVAILABLE);
78-
79-
#ifdef CONFIG_PAGE_TABLE_ISOLATION
80-
/*
81-
* Make sure that the dynamic ASID space does not confict with the
82-
* bit we are using to switch between user and kernel ASIDs.
83-
*/
84-
BUILD_BUG_ON(TLB_NR_DYN_ASIDS >= (1 << X86_CR3_PTI_PCID_USER_BIT));
85-
86-
/*
87-
* The ASID being passed in here should have respected the
88-
* MAX_ASID_AVAILABLE and thus never have the switch bit set.
89-
*/
90-
VM_WARN_ON_ONCE(asid & (1 << X86_CR3_PTI_PCID_USER_BIT));
91-
#endif
92-
/*
93-
* The dynamically-assigned ASIDs that get passed in are small
94-
* (<TLB_NR_DYN_ASIDS). They never have the high switch bit set,
95-
* so do not bother to clear it.
96-
*
97-
* If PCID is on, ASID-aware code paths put the ASID+1 into the
98-
* PCID bits. This serves two purposes. It prevents a nasty
99-
* situation in which PCID-unaware code saves CR3, loads some other
100-
* value (with PCID == 0), and then restores CR3, thus corrupting
101-
* the TLB for ASID 0 if the saved ASID was nonzero. It also means
102-
* that any bugs involving loading a PCID-enabled CR3 with
103-
* CR4.PCIDE off will trigger deterministically.
104-
*/
105-
return asid + 1;
106-
}
107-
108-
/*
109-
* Given @asid, compute uPCID
110-
*/
111-
static inline u16 user_pcid(u16 asid)
112-
{
113-
u16 ret = kern_pcid(asid);
114-
#ifdef CONFIG_PAGE_TABLE_ISOLATION
115-
ret |= 1 << X86_CR3_PTI_PCID_USER_BIT;
116-
#endif
117-
return ret;
118-
}
119-
120-
struct pgd_t;
121-
static inline unsigned long build_cr3(pgd_t *pgd, u16 asid)
122-
{
123-
if (static_cpu_has(X86_FEATURE_PCID)) {
124-
return __sme_pa(pgd) | kern_pcid(asid);
125-
} else {
126-
VM_WARN_ON_ONCE(asid != 0);
127-
return __sme_pa(pgd);
128-
}
129-
}
130-
131-
static inline unsigned long build_cr3_noflush(pgd_t *pgd, u16 asid)
132-
{
133-
VM_WARN_ON_ONCE(asid > MAX_ASID_AVAILABLE);
134-
/*
135-
* Use boot_cpu_has() instead of this_cpu_has() as this function
136-
* might be called during early boot. This should work even after
137-
* boot because all CPU's the have same capabilities:
138-
*/
139-
VM_WARN_ON_ONCE(!boot_cpu_has(X86_FEATURE_PCID));
140-
return __sme_pa(pgd) | kern_pcid(asid) | CR3_NOFLUSH;
141-
}
142-
14316
struct flush_tlb_info;
14417

14518
void __flush_tlb_all(void);
@@ -153,6 +26,12 @@ void flush_tlb_others(const struct cpumask *cpumask,
15326
#include <asm/paravirt.h>
15427
#endif
15528

29+
/*
30+
* 6 because 6 should be plenty and struct tlb_state will fit in two cache
31+
* lines.
32+
*/
33+
#define TLB_NR_DYN_ASIDS 6
34+
15635
struct tlb_context {
15736
u64 ctx_id;
15837
u64 tlb_gen;

‎arch/x86/mm/tlb.c

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,126 @@
4848
*/
4949
#define LAST_USER_MM_IBPB 0x1UL
5050

51+
/*
52+
* The x86 feature is called PCID (Process Context IDentifier). It is similar
53+
* to what is traditionally called ASID on the RISC processors.
54+
*
55+
* We don't use the traditional ASID implementation, where each process/mm gets
56+
* its own ASID and flush/restart when we run out of ASID space.
57+
*
58+
* Instead we have a small per-cpu array of ASIDs and cache the last few mm's
59+
* that came by on this CPU, allowing cheaper switch_mm between processes on
60+
* this CPU.
61+
*
62+
* We end up with different spaces for different things. To avoid confusion we
63+
* use different names for each of them:
64+
*
65+
* ASID - [0, TLB_NR_DYN_ASIDS-1]
66+
* the canonical identifier for an mm
67+
*
68+
* kPCID - [1, TLB_NR_DYN_ASIDS]
69+
* the value we write into the PCID part of CR3; corresponds to the
70+
* ASID+1, because PCID 0 is special.
71+
*
72+
* uPCID - [2048 + 1, 2048 + TLB_NR_DYN_ASIDS]
73+
* for KPTI each mm has two address spaces and thus needs two
74+
* PCID values, but we can still do with a single ASID denomination
75+
* for each mm. Corresponds to kPCID + 2048.
76+
*
77+
*/
78+
79+
/* There are 12 bits of space for ASIDS in CR3 */
80+
#define CR3_HW_ASID_BITS 12
81+
82+
/*
83+
* When enabled, PAGE_TABLE_ISOLATION consumes a single bit for
84+
* user/kernel switches
85+
*/
86+
#ifdef CONFIG_PAGE_TABLE_ISOLATION
87+
# define PTI_CONSUMED_PCID_BITS 1
88+
#else
89+
# define PTI_CONSUMED_PCID_BITS 0
90+
#endif
91+
92+
#define CR3_AVAIL_PCID_BITS (X86_CR3_PCID_BITS - PTI_CONSUMED_PCID_BITS)
93+
94+
/*
95+
* ASIDs are zero-based: 0->MAX_AVAIL_ASID are valid. -1 below to account
96+
* for them being zero-based. Another -1 is because PCID 0 is reserved for
97+
* use by non-PCID-aware users.
98+
*/
99+
#define MAX_ASID_AVAILABLE ((1 << CR3_AVAIL_PCID_BITS) - 2)
100+
101+
/*
102+
* Given @asid, compute kPCID
103+
*/
104+
static inline u16 kern_pcid(u16 asid)
105+
{
106+
VM_WARN_ON_ONCE(asid > MAX_ASID_AVAILABLE);
107+
108+
#ifdef CONFIG_PAGE_TABLE_ISOLATION
109+
/*
110+
* Make sure that the dynamic ASID space does not confict with the
111+
* bit we are using to switch between user and kernel ASIDs.
112+
*/
113+
BUILD_BUG_ON(TLB_NR_DYN_ASIDS >= (1 << X86_CR3_PTI_PCID_USER_BIT));
114+
115+
/*
116+
* The ASID being passed in here should have respected the
117+
* MAX_ASID_AVAILABLE and thus never have the switch bit set.
118+
*/
119+
VM_WARN_ON_ONCE(asid & (1 << X86_CR3_PTI_PCID_USER_BIT));
120+
#endif
121+
/*
122+
* The dynamically-assigned ASIDs that get passed in are small
123+
* (<TLB_NR_DYN_ASIDS). They never have the high switch bit set,
124+
* so do not bother to clear it.
125+
*
126+
* If PCID is on, ASID-aware code paths put the ASID+1 into the
127+
* PCID bits. This serves two purposes. It prevents a nasty
128+
* situation in which PCID-unaware code saves CR3, loads some other
129+
* value (with PCID == 0), and then restores CR3, thus corrupting
130+
* the TLB for ASID 0 if the saved ASID was nonzero. It also means
131+
* that any bugs involving loading a PCID-enabled CR3 with
132+
* CR4.PCIDE off will trigger deterministically.
133+
*/
134+
return asid + 1;
135+
}
136+
137+
/*
138+
* Given @asid, compute uPCID
139+
*/
140+
static inline u16 user_pcid(u16 asid)
141+
{
142+
u16 ret = kern_pcid(asid);
143+
#ifdef CONFIG_PAGE_TABLE_ISOLATION
144+
ret |= 1 << X86_CR3_PTI_PCID_USER_BIT;
145+
#endif
146+
return ret;
147+
}
148+
149+
static inline unsigned long build_cr3(pgd_t *pgd, u16 asid)
150+
{
151+
if (static_cpu_has(X86_FEATURE_PCID)) {
152+
return __sme_pa(pgd) | kern_pcid(asid);
153+
} else {
154+
VM_WARN_ON_ONCE(asid != 0);
155+
return __sme_pa(pgd);
156+
}
157+
}
158+
159+
static inline unsigned long build_cr3_noflush(pgd_t *pgd, u16 asid)
160+
{
161+
VM_WARN_ON_ONCE(asid > MAX_ASID_AVAILABLE);
162+
/*
163+
* Use boot_cpu_has() instead of this_cpu_has() as this function
164+
* might be called during early boot. This should work even after
165+
* boot because all CPU's the have same capabilities:
166+
*/
167+
VM_WARN_ON_ONCE(!boot_cpu_has(X86_FEATURE_PCID));
168+
return __sme_pa(pgd) | kern_pcid(asid) | CR3_NOFLUSH;
169+
}
170+
51171
/*
52172
* We get here when we do something requiring a TLB invalidation
53173
* but could not go invalidate all of the contexts. We do the

0 commit comments

Comments
 (0)