1313
1414static pgd_t kasan_pg_dir [PTRS_PER_PGD ] __initdata __aligned (PAGE_SIZE );
1515
16+ #ifdef __PAGETABLE_P4D_FOLDED
17+ #define __pgd_none (early , pgd ) (0)
18+ #else
19+ #define __pgd_none (early , pgd ) (early ? (pgd_val(pgd) == 0) : \
20+ (__pa(pgd_val(pgd)) == (unsigned long)__pa(kasan_early_shadow_p4d)))
21+ #endif
22+
1623#ifdef __PAGETABLE_PUD_FOLDED
1724#define __p4d_none (early , p4d ) (0)
1825#else
@@ -142,6 +149,19 @@ static pud_t *__init kasan_pud_offset(p4d_t *p4dp, unsigned long addr, int node,
142149 return pud_offset (p4dp , addr );
143150}
144151
152+ static p4d_t * __init kasan_p4d_offset (pgd_t * pgdp , unsigned long addr , int node , bool early )
153+ {
154+ if (__pgd_none (early , pgdp_get (pgdp ))) {
155+ phys_addr_t p4d_phys = early ?
156+ __pa_symbol (kasan_early_shadow_p4d ) : kasan_alloc_zeroed_page (node );
157+ if (!early )
158+ memcpy (__va (p4d_phys ), kasan_early_shadow_p4d , sizeof (kasan_early_shadow_p4d ));
159+ pgd_populate (& init_mm , pgdp , (p4d_t * )__va (p4d_phys ));
160+ }
161+
162+ return p4d_offset (pgdp , addr );
163+ }
164+
145165static void __init kasan_pte_populate (pmd_t * pmdp , unsigned long addr ,
146166 unsigned long end , int node , bool early )
147167{
@@ -178,19 +198,19 @@ static void __init kasan_pud_populate(p4d_t *p4dp, unsigned long addr,
178198 do {
179199 next = pud_addr_end (addr , end );
180200 kasan_pmd_populate (pudp , addr , next , node , early );
181- } while (pudp ++ , addr = next , addr != end );
201+ } while (pudp ++ , addr = next , addr != end && __pud_none ( early , READ_ONCE ( * pudp )) );
182202}
183203
184204static void __init kasan_p4d_populate (pgd_t * pgdp , unsigned long addr ,
185205 unsigned long end , int node , bool early )
186206{
187207 unsigned long next ;
188- p4d_t * p4dp = p4d_offset (pgdp , addr );
208+ p4d_t * p4dp = kasan_p4d_offset (pgdp , addr , node , early );
189209
190210 do {
191211 next = p4d_addr_end (addr , end );
192212 kasan_pud_populate (p4dp , addr , next , node , early );
193- } while (p4dp ++ , addr = next , addr != end );
213+ } while (p4dp ++ , addr = next , addr != end && __p4d_none ( early , READ_ONCE ( * p4dp )) );
194214}
195215
196216static void __init kasan_pgd_populate (unsigned long addr , unsigned long end ,
@@ -218,7 +238,7 @@ static void __init kasan_map_populate(unsigned long start, unsigned long end,
218238asmlinkage void __init kasan_early_init (void )
219239{
220240 BUILD_BUG_ON (!IS_ALIGNED (KASAN_SHADOW_START , PGDIR_SIZE ));
221- BUILD_BUG_ON (!IS_ALIGNED (KASAN_SHADOW_END , PGDIR_SIZE ));
241+ BUILD_BUG_ON (!IS_ALIGNED (KASAN_SHADOW_END + 1 , PGDIR_SIZE ));
222242}
223243
224244static inline void kasan_set_pgd (pgd_t * pgdp , pgd_t pgdval )
@@ -233,7 +253,7 @@ static void __init clear_pgds(unsigned long start, unsigned long end)
233253 * swapper_pg_dir. pgd_clear() can't be used
234254 * here because it's nop on 2,3-level pagetable setups
235255 */
236- for (; start < end ; start += PGDIR_SIZE )
256+ for (; start < end ; start = pgd_addr_end ( start , end ) )
237257 kasan_set_pgd ((pgd_t * )pgd_offset_k (start ), __pgd (0 ));
238258}
239259
@@ -242,6 +262,17 @@ void __init kasan_init(void)
242262 u64 i ;
243263 phys_addr_t pa_start , pa_end ;
244264
265+ /*
266+ * If PGDIR_SIZE is too large for cpu_vabits, KASAN_SHADOW_END will
267+ * overflow UINTPTR_MAX and then looks like a user space address.
268+ * For example, PGDIR_SIZE of CONFIG_4KB_4LEVEL is 2^39, which is too
269+ * large for Loongson-2K series whose cpu_vabits = 39.
270+ */
271+ if (KASAN_SHADOW_END < vm_map_base ) {
272+ pr_warn ("PGDIR_SIZE too large for cpu_vabits, KernelAddressSanitizer disabled.\n" );
273+ return ;
274+ }
275+
245276 /*
246277 * PGD was populated as invalid_pmd_table or invalid_pud_table
247278 * in pagetable_init() which depends on how many levels of page
0 commit comments