arch: replace _current_cpu with arch_curr_cpu()

Use the `arch_curr_cpu()` directly instead of the `_current_cpu`
abstraction.

Guard the `z_smp_cpu_mobile()` usage validation with a new Kconfig
`CONFIG_VALIDATE_ARCH_CURR_CPU`, and move it into the
`arch_curr_cpu()` function.

Signed-off-by: Yong Cong Sin <[email protected]>
Signed-off-by: Yong Cong Sin <[email protected]>

Author: ycsin

arch/arc/core/thread.c

@@ -334,7 +334,7 @@ int arc_vpx_lock(k_timeout_t timeout)
 
 	key = k_spin_lock(&lock);
 
-	id = _current_cpu->id;
+	id = arch_curr_cpu()->id;
 #if (CONFIG_MP_MAX_NUM_CPUS > 1) && defined(CONFIG_SCHED_CPU_MASK)
 	__ASSERT(!arch_is_in_isr() && (_current->base.cpu_mask == BIT(id)), "");
 #endif
@@ -357,7 +357,7 @@ void arc_vpx_unlock(void)
 #if (CONFIG_MP_MAX_NUM_CPUS > 1) && defined(CONFIG_SCHED_CPU_MASK)
 	__ASSERT(!arch_is_in_isr() && (_current->base.cpu_mask == BIT(id)), "");
 #endif
-	id = _current_cpu->id;
+	id = arch_curr_cpu()->id;
 	k_spin_unlock(&lock, key);
 
 	/*

arch/arm/core/cortex_a_r/fault.c

@@ -153,21 +153,21 @@ bool z_arm_fault_undef_instruction_fp(void)
 
 	__set_FPEXC(FPEXC_EN);
 
-	if (_current_cpu->nested > 1) {
+	if (arch_curr_cpu()->nested > 1) {
 		/*
 		 * If the nested count is greater than 1, the undefined
 		 * instruction exception came from an irq/svc context.  (The
 		 * irq/svc handler would have the nested count at 1 and then
 		 * the undef exception would increment it to 2).
 		 */
 		struct __fpu_sf *spill_esf =
-			(struct __fpu_sf *)_current_cpu->fp_ctx;
+			(struct __fpu_sf *)arch_curr_cpu()->fp_ctx;
 
 		if (spill_esf == NULL) {
 			return false;
 		}
 
-		_current_cpu->fp_ctx = NULL;
+		arch_curr_cpu()->fp_ctx = NULL;
 
 		/*
 		 * If the nested count is 2 and the current thread has used the
@@ -177,9 +177,9 @@ bool z_arm_fault_undef_instruction_fp(void)
 		 * saved exception stack frame, then save the floating point
 		 * context because it is about to be overwritten.
 		 */
-		if (((_current_cpu->nested == 2)
+		if (((arch_curr_cpu()->nested == 2)
 				&& (_current->base.user_options & K_FP_REGS))
-			|| ((_current_cpu->nested > 2)
+			|| ((arch_curr_cpu()->nested > 2)
 				&& (spill_esf->undefined & FPEXC_EN))) {
 			/*
 			 * Spill VFP registers to specified exception stack

arch/arm64/core/fpu.c

@@ -34,7 +34,7 @@ static void DBG(char *msg, struct k_thread *th)
 	unsigned int v;
 
 	strcpy(buf, "CPU# exc# ");
-	buf[3] = '0' + _current_cpu->id;
+	buf[3] = '0' + arch_curr_cpu()->id;
 	buf[8] = '0' + arch_exception_depth();
 	strcat(buf, _current->name);
 	strcat(buf, ": ");
@@ -68,7 +68,7 @@ void arch_flush_local_fpu(void)
 {
 	__ASSERT(read_daif() & DAIF_IRQ_BIT, "must be called with IRQs disabled");
 
-	struct k_thread *owner = atomic_ptr_get(&_current_cpu->arch.fpu_owner);
+	struct k_thread *owner = atomic_ptr_get(&arch_curr_cpu()->arch.fpu_owner);
 
 	if (owner != NULL) {
 		uint64_t cpacr = read_cpacr_el1();
@@ -82,7 +82,7 @@ void arch_flush_local_fpu(void)
 		/* make sure content made it to memory before releasing */
 		barrier_dsync_fence_full();
 		/* release ownership */
-		atomic_ptr_clear(&_current_cpu->arch.fpu_owner);
+		atomic_ptr_clear(&arch_curr_cpu()->arch.fpu_owner);
 		DBG("disable", owner);
 
 		/* disable FPU access */
@@ -106,7 +106,7 @@ static void flush_owned_fpu(struct k_thread *thread)
 			continue;
 		}
 		/* we found it live on CPU i */
-		if (i == _current_cpu->id) {
+		if (i == arch_curr_cpu()->id) {
 			arch_flush_local_fpu();
 		} else {
 			/* the FPU context is live on another CPU */
@@ -235,12 +235,12 @@ void z_arm64_fpu_trap(struct arch_esf *esf)
 	barrier_isync_fence_full();
 
 	/* save current owner's content  if any */
-	struct k_thread *owner = atomic_ptr_get(&_current_cpu->arch.fpu_owner);
+	struct k_thread *owner = atomic_ptr_get(&arch_curr_cpu()->arch.fpu_owner);
 
 	if (owner) {
 		z_arm64_fpu_save(&owner->arch.saved_fp_context);
 		barrier_dsync_fence_full();
-		atomic_ptr_clear(&_current_cpu->arch.fpu_owner);
+		atomic_ptr_clear(&arch_curr_cpu()->arch.fpu_owner);
 		DBG("save", owner);
 	}
 
@@ -264,7 +264,7 @@ void z_arm64_fpu_trap(struct arch_esf *esf)
 #endif
 
 	/* become new owner */
-	atomic_ptr_set(&_current_cpu->arch.fpu_owner, _current);
+	atomic_ptr_set(&arch_curr_cpu()->arch.fpu_owner, _current);
 
 	/* restore our content */
 	z_arm64_fpu_restore(&_current->arch.saved_fp_context);
@@ -287,7 +287,7 @@ static void fpu_access_update(unsigned int exc_update_level)
 
 	if (arch_exception_depth() == exc_update_level) {
 		/* We're about to execute non-exception code */
-		if (atomic_ptr_get(&_current_cpu->arch.fpu_owner) == _current) {
+		if (atomic_ptr_get(&arch_curr_cpu()->arch.fpu_owner) == _current) {
 			/* turn on FPU access */
 			write_cpacr_el1(cpacr | CPACR_EL1_FPEN_NOTRAP);
 		} else {
@@ -333,7 +333,7 @@ int arch_float_disable(struct k_thread *thread)
 #ifdef CONFIG_SMP
 		flush_owned_fpu(thread);
 #else
-		if (thread == atomic_ptr_get(&_current_cpu->arch.fpu_owner)) {
+		if (thread == atomic_ptr_get(&arch_curr_cpu()->arch.fpu_owner)) {
 			arch_flush_local_fpu();
 		}
 #endif

arch/arm64/core/isr_wrapper.S

@@ -32,7 +32,7 @@ GDATA(_sw_isr_table)
 GTEXT(_isr_wrapper)
 SECTION_FUNC(TEXT, _isr_wrapper)
 
-	/* ++_current_cpu->nested to be checked by arch_is_in_isr() */
+	/* ++arch_curr_cpu()->nested to be checked by arch_is_in_isr() */
 	get_cpu	x0
 	ldr	w1, [x0, #___cpu_t_nested_OFFSET]
 	add	w2, w1, #1
@@ -113,7 +113,7 @@ spurious_continue:
 
 GTEXT(z_arm64_irq_done)
 z_arm64_irq_done:
-	/* if (--_current_cpu->nested != 0) exit */
+	/* if (--arch_curr_cpu()->nested != 0) exit */
 	get_cpu	x0
 	ldr	w1, [x0, #___cpu_t_nested_OFFSET]
 	subs	w1, w1, #1

arch/arm64/core/smp.c

@@ -283,7 +283,7 @@ void arch_spin_relax(void)
 		 * We may not be in IRQ context here hence cannot use
 		 * arch_flush_local_fpu() directly.
 		 */
-		arch_float_disable(_current_cpu->arch.fpu_owner);
+		arch_float_disable(arch_curr_cpu()->arch.fpu_owner);
 	}
 }
 #endif

arch/arm64/core/switch.S

@@ -179,7 +179,7 @@ offload:
 	 */
 	ldp	x1, x0, [sp, ___esf_t_x0_x1_OFFSET]
 
-	/* ++_current_cpu->nested to be checked by arch_is_in_isr() */
+	/* ++arch_curr_cpu()->nested to be checked by arch_is_in_isr() */
 	get_cpu	x2
 	ldr	w3, [x2, #___cpu_t_nested_OFFSET]
 	add	w4, w3, #1

arch/mips/core/irq_manage.c

@@ -62,7 +62,7 @@ int arch_irq_is_enabled(unsigned int irq)
 
 void z_mips_enter_irq(uint32_t ipending)
 {
-	_current_cpu->nested++;
+	arch_curr_cpu()->nested++;
 
 #ifdef CONFIG_IRQ_OFFLOAD
 	z_irq_do_offload();
@@ -88,7 +88,7 @@ void z_mips_enter_irq(uint32_t ipending)
 		}
 	}
 
-	_current_cpu->nested--;
+	arch_curr_cpu()->nested--;
 
 	if (IS_ENABLED(CONFIG_STACK_SENTINEL)) {
 		z_check_stack_sentinel();

arch/mips/include/kernel_arch_func.h

@@ -39,7 +39,7 @@ FUNC_NORETURN void z_mips_fatal_error(unsigned int reason,
 
 static inline bool arch_is_in_isr(void)
 {
-	return _current_cpu->nested != 0U;
+	return arch_curr_cpu()->nested != 0U;
 }
 
 #ifdef CONFIG_IRQ_OFFLOAD

arch/riscv/core/fpu.c

@@ -35,7 +35,7 @@ static void DBG(char *msg, struct k_thread *th)
 	unsigned int v;
 
 	strcpy(buf, "CPU# exc# ");
-	buf[3] = '0' + _current_cpu->id;
+	buf[3] = '0' + arch_curr_cpu()->id;
 	buf[8] = '0' + _current->arch.exception_depth;
 	strcat(buf, _current->name);
 	strcat(buf, ": ");
@@ -70,7 +70,7 @@ static void z_riscv_fpu_disable(void)
 		csr_clear(mstatus, MSTATUS_FS);
 
 		/* remember its clean/dirty state */
-		_current_cpu->arch.fpu_state = (status & MSTATUS_FS);
+		arch_curr_cpu()->arch.fpu_state = (status & MSTATUS_FS);
 	}
 }
 
@@ -82,7 +82,7 @@ static void z_riscv_fpu_load(void)
 		 "must be called with FPU access disabled");
 
 	/* become new owner */
-	atomic_ptr_set(&_current_cpu->arch.fpu_owner, _current);
+	atomic_ptr_set(&arch_curr_cpu()->arch.fpu_owner, _current);
 
 	/* restore our content */
 	csr_set(mstatus, MSTATUS_FS_INIT);
@@ -105,10 +105,10 @@ void arch_flush_local_fpu(void)
 	__ASSERT((csr_read(mstatus) & MSTATUS_FS) == 0,
 		 "must be called with FPU access disabled");
 
-	struct k_thread *owner = atomic_ptr_get(&_current_cpu->arch.fpu_owner);
+	struct k_thread *owner = atomic_ptr_get(&arch_curr_cpu()->arch.fpu_owner);
 
 	if (owner != NULL) {
-		bool dirty = (_current_cpu->arch.fpu_state == MSTATUS_FS_DIRTY);
+		bool dirty = (arch_curr_cpu()->arch.fpu_state == MSTATUS_FS_DIRTY);
 
 		if (dirty) {
 			/* turn on FPU access */
@@ -124,7 +124,7 @@ void arch_flush_local_fpu(void)
 		csr_clear(mstatus, MSTATUS_FS);
 
 		/* release ownership */
-		atomic_ptr_clear(&_current_cpu->arch.fpu_owner);
+		atomic_ptr_clear(&arch_curr_cpu()->arch.fpu_owner);
 		DBG("disable", owner);
 	}
 }
@@ -147,7 +147,7 @@ static void flush_owned_fpu(struct k_thread *thread)
 			continue;
 		}
 		/* we found it live on CPU i */
-		if (i == _current_cpu->id) {
+		if (i == arch_curr_cpu()->id) {
 			z_riscv_fpu_disable();
 			arch_flush_local_fpu();
 			break;
@@ -258,7 +258,7 @@ static bool fpu_access_allowed(unsigned int exc_update_level)
 
 	if (_current->arch.exception_depth == exc_update_level) {
 		/* We're about to execute non-exception code */
-		if (_current_cpu->arch.fpu_owner == _current) {
+		if (arch_curr_cpu()->arch.fpu_owner == _current) {
 			/* everything is already in place */
 			return true;
 		}
@@ -276,7 +276,7 @@ static bool fpu_access_allowed(unsigned int exc_update_level)
 			flush_owned_fpu(_current);
 #endif
 			z_riscv_fpu_load();
-			_current_cpu->arch.fpu_state = MSTATUS_FS_CLEAN;
+			arch_curr_cpu()->arch.fpu_state = MSTATUS_FS_CLEAN;
 			return true;
 		}
 		return false;
@@ -297,7 +297,7 @@ void z_riscv_fpu_exit_exc(struct arch_esf *esf)
 {
 	if (fpu_access_allowed(1)) {
 		esf->mstatus &= ~MSTATUS_FS;
-		esf->mstatus |= _current_cpu->arch.fpu_state;
+		esf->mstatus |= arch_curr_cpu()->arch.fpu_state;
 	} else {
 		esf->mstatus &= ~MSTATUS_FS;
 	}
@@ -313,7 +313,7 @@ void z_riscv_fpu_thread_context_switch(void)
 {
 	if (fpu_access_allowed(0)) {
 		csr_clear(mstatus, MSTATUS_FS);
-		csr_set(mstatus, _current_cpu->arch.fpu_state);
+		csr_set(mstatus, arch_curr_cpu()->arch.fpu_state);
 	} else {
 		z_riscv_fpu_disable();
 	}
@@ -327,7 +327,7 @@ int arch_float_disable(struct k_thread *thread)
 #ifdef CONFIG_SMP
 		flush_owned_fpu(thread);
 #else
-		if (thread == _current_cpu->arch.fpu_owner) {
+		if (thread == arch_curr_cpu()->arch.fpu_owner) {
 			z_riscv_fpu_disable();
 			arch_flush_local_fpu();
 		}

arch/riscv/core/isr.S

@@ -137,7 +137,7 @@ SECTION_FUNC(exception.entry, _isr_wrapper)
 .balign CONFIG_RISCV_TRAP_HANDLER_ALIGNMENT
 
 #ifdef CONFIG_USERSPACE
-	/* retrieve address of _current_cpu preserving s0 */
+	/* retrieve address of arch_curr_cpu() preserving s0 */
 	csrrw s0, mscratch, s0
 
 	/* preserve t0 and t1 temporarily */
@@ -172,7 +172,7 @@ SECTION_FUNC(exception.entry, _isr_wrapper)
 	lr t0, _curr_cpu_arch_user_exc_tmp0(s0)
 	lr t1, _curr_cpu_arch_user_exc_tmp1(s0)
 
-	/* retrieve original s0 and restore _current_cpu in mscratch */
+	/* retrieve original s0 and restore arch_curr_cpu() in mscratch */
 	csrrw s0, mscratch, s0
 #endif
 
@@ -184,7 +184,7 @@ SECTION_FUNC(exception.entry, _isr_wrapper)
 	DO_CALLER_SAVED(sr)		;
 #endif /* CONFIG_RISCV_SOC_HAS_ISR_STACKING */
 
-	/* Save s0 in the esf and load it with &_current_cpu. */
+	/* Save s0 in the esf and load it with &arch_curr_cpu(). */
 	sr s0, __struct_arch_esf_s0_OFFSET(sp)
 	get_current_cpu s0
 
@@ -476,7 +476,7 @@ do_irq_offload:
 	lr a1, __struct_arch_esf_a0_OFFSET(sp)
 	lr a0, __struct_arch_esf_a1_OFFSET(sp)
 
-	/* Increment _current_cpu->nested */
+	/* Increment arch_curr_cpu()->nested */
 	lw t1, ___cpu_t_nested_OFFSET(s0)
 	addi t2, t1, 1
 	sw t2, ___cpu_t_nested_OFFSET(s0)
@@ -592,7 +592,7 @@ is_interrupt:
 2:
 #endif
 
-	/* Increment _current_cpu->nested */
+	/* Increment arch_curr_cpu()->nested */
 	lw t1, ___cpu_t_nested_OFFSET(s0)
 	addi t2, t1, 1
 	sw t2, ___cpu_t_nested_OFFSET(s0)
@@ -654,7 +654,7 @@ on_irq_stack:
 #endif
 
 irq_done:
-	/* Decrement _current_cpu->nested */
+	/* Decrement arch_curr_cpu()->nested */
 	lw t2, ___cpu_t_nested_OFFSET(s0)
 	addi t2, t2, -1
 	sw t2, ___cpu_t_nested_OFFSET(s0)
@@ -698,7 +698,7 @@ reschedule:
 
 z_riscv_thread_start:
 might_have_rescheduled:
-	/* reload s0 with &_current_cpu as it might have changed or be unset */
+	/* reload s0 with &arch_curr_cpu() as it might have changed or be unset */
 	get_current_cpu s0
 
 #endif /* CONFIG_MULTITHREADING */