full AP initialisation into proc loop. theyll now wait in HLT until someone calls wake_cpu with their logical id

Author: braindigitalis

include/apic.h

@@ -1,31 +1,98 @@
 /**
  * @file apic.h
- * @author Craig Edwards (craigedwards@brainbox.cc)
+ * @author Craig Edwards
  * @copyright Copyright (c) 2012-2025
  * @brief Local APIC (Advanced Programmable Interrupt Controller) interface
  *
  * This header provides low-level routines and register definitions to interface with
  * the Local APIC (LAPIC) on x86_64 systems. It allows for reading and writing LAPIC registers,
- * detecting the current CPU's LAPIC ID, and querying the LAPIC base address.
+ * detecting the current CPU's LAPIC ID, sending inter-processor interrupts (IPIs),
+ * and querying the LAPIC base address.
  *
- * This interface is used during system initialisation to configure per-core interrupt handling,
- * timer setup, and inter-processor signalling on SMP systems.
+ * The LAPIC is a per-core interrupt controller responsible for:
+ * - Handling local interrupts (e.g., timer, thermal, performance counters).
+ * - Providing per-core programmable timer functionality.
+ * - Supporting Inter-Processor Interrupts (IPIs) for SMP coordination.
+ * - Forwarding external interrupts via the I/O APIC.
+ *
+ * Both xAPIC (MMIO) and x2APIC (MSR-based) modes are supported by this interface.
  */
 
 #pragma once
 
-/** Default virtual memory mapping address for the Local APIC (used by the kernel) */
-#define APIC_ADDRESS 0x4000
+/** @brief Default virtual memory mapping address for the Local APIC MMIO page. */
+#define APIC_ADDRESS    0x4000
+
+/** @brief Model Specific Register (MSR) index for LAPIC base address (IA32_APIC_BASE). */
+#define APIC_BASE_MSR   0x1B
+
+/** @brief Bitmask in IA32_APIC_BASE MSR to enable the Local APIC. */
+#define APIC_BASE_MSR_ENABLE   0x800
+
+/* -------------------------------------------------------------------------- */
+/* LAPIC Register Offsets (xAPIC mode, relative to MMIO base)                  */
+/* -------------------------------------------------------------------------- */
+
+/** @brief Local APIC ID Register (read-only). */
+#define APIC_ID         0x0020
+
+/** @brief Local APIC Version Register (read-only). */
+#define APIC_VERSION    0x0030
+
+/** @brief Task Priority Register (TPR). Controls interrupt priority threshold. */
+#define APIC_TPR        0x0080
+
+/** @brief Interrupt Command Register (ICR), low dword (vector + control flags). */
+#define APIC_ICR_LOW    0x0300
+
+/** @brief Interrupt Command Register (ICR), high dword (destination LAPIC ID). */
+#define APIC_ICR_HIGH   0x0310
+
+/* -------------------------------------------------------------------------- */
+/* Delivery Modes for Inter-Processor Interrupts (IPI)                        */
+/* -------------------------------------------------------------------------- */
+
+/** @brief Fixed delivery mode: deliver interrupt using vector field. */
+#define APIC_DM_FIXED       0x000
+
+/** @brief Non-Maskable Interrupt (NMI) delivery mode. */
+#define APIC_DM_NMI         0x400
 
-/** Model Specific Register for the LAPIC base address */
-#define APIC_BASE_MSR 0x1B
+/** @brief INIT delivery mode: reset target CPU to INIT state. */
+#define APIC_DM_INIT        0x500
 
-/** Bitmask to enable the LAPIC via the APIC_BASE_MSR */
-#define APIC_BASE_MSR_ENABLE 0x800
+/** @brief Startup IPI delivery mode: start target CPU at specified vector. */
+#define APIC_DM_STARTUP     0x600
 
-/* LAPIC register offsets (relative to the LAPIC base address) */
-#define APIC_ID        0x0020  /**< LAPIC ID Register (read-only) */
-#define APIC_VERSION   0x0030  /**< LAPIC Version Register (read-only) */
+/* -------------------------------------------------------------------------- */
+/* x2APIC Registers (MSR-based access only)                                   */
+/* -------------------------------------------------------------------------- */
+
+/** @brief MSR index for the x2APIC Interrupt Command Register (ICR). */
+#define IA32_X2APIC_ICR     0x830
+
+/* -------------------------------------------------------------------------- */
+/* Miscellaneous constants                                                    */
+/* -------------------------------------------------------------------------- */
+
+/** @brief Vector number used for custom AP wake-up IPIs. */
+#define APIC_WAKE_IPI       240
+
+/** @brief Destination shorthand: no shorthand, use explicit LAPIC ID. */
+#define APIC_DEST_NO_SHORTHAND (0 << 18)
+
+/** @brief Destination mode: physical addressing. */
+#define APIC_DEST_PHYSICAL  (0 << 11)
+
+/** @brief Assert level for IPI (used to trigger delivery). */
+#define APIC_LEVEL_ASSERT   (1 << 14)
+
+/** @brief Edge-triggered delivery mode for IPI. */
+#define APIC_TRIGGER_EDGE   (0 << 15)
+
+/* -------------------------------------------------------------------------- */
+/* Function prototypes                                                        */
+/* -------------------------------------------------------------------------- */
 
 /**
  * @brief Read a Local APIC register.
@@ -34,7 +101,7 @@
  * - In xAPIC mode, the register is memory‑mapped at the LAPIC base address.
  * - In x2APIC mode, registers are accessed via MSRs at 0x800 + (reg >> 4).
  *
- * @param reg Register offset (e.g. APIC_EOI = 0xB0).
+ * @param reg Register offset (e.g. APIC_TPR = 0x80).
  * @return uint32_t Value read from the register.
  */
 uint32_t apic_read(uint64_t reg);
@@ -46,8 +113,8 @@ uint32_t apic_read(uint64_t reg);
  * - In xAPIC mode, the register is memory‑mapped at the LAPIC base address.
  * - In x2APIC mode, registers are written via MSRs at 0x800 + (reg >> 4).
  *
- * @param reg Register offset (e.g. APIC_EOI = 0xB0).
- * @param val Value to write.
+ * @param reg Register offset (e.g. APIC_TPR = 0x80).
+ * @param value Value to write.
  */
 void apic_write(uint64_t reg, uint32_t value);
 
@@ -118,4 +185,68 @@ uint32_t get_lapic_id_from_cpu_id(uint8_t cpu_id);
  */
 uint8_t get_cpu_id_from_lapic_id(uint32_t lapic_id);
 
-void boot_aps();
+/**
+ * @brief Initialise and bring Application Processors (APs) online via Limine SMP.
+ *
+ * This function uses the Limine SMP response to discover all CPUs in the system,
+ * map their LAPIC IDs to logical CPU IDs, and assign each AP an entry point
+ * (`kmain_ap`). The Bootstrap Processor (BSP) is detected and skipped, as are
+ * CPUs with processor IDs >= 255 (reserved for broadcast or outside kernel limits).
+ *
+ * Once all eligible APs are assigned their startup entry point, this function
+ * spins until each AP reports online by incrementing @ref aps_online.
+ *
+ * @note This does not directly issue INIT/SIPI IPIs; Limine handles the actual
+ *       startup of APs. The kernel only assigns the entry function and waits.
+ * @warning Systems with more than 254 CPUs are truncated to 254 usable cores.
+ *
+ * @see kmain_ap, aps_online, set_lapic_id_for_cpu_id
+ */
+void boot_aps();
+
+
+/**
+ * @brief Send an Inter-Processor Interrupt (IPI) to a specific LAPIC ID.
+ *
+ * Supports both xAPIC (ICR registers) and x2APIC (IA32_X2APIC_ICR MSR).
+ *
+ * @param lapic_id Destination LAPIC ID.
+ * @param vector Interrupt vector to deliver.
+ */
+void apic_send_ipi(uint32_t lapic_id, uint8_t vector);
+
+/**
+ * @brief Send a wake-up IPI to a target logical CPU.
+ *
+ * This function is used to wake an Application Processor (AP) from a halted or
+ * waiting state. It resolves the kernel-assigned logical CPU ID into the
+ * hardware LAPIC ID and then sends a fixed-mode IPI using the predefined
+ * wake-up vector (APIC_WAKE_IPI).
+ *
+ * @param logical_cpu_id Logical CPU ID (zero-based, contiguous index assigned
+ *                       by the kernel).
+ *
+ * @note The Local APIC on the target CPU must already be enabled for the wake-up
+ *       IPI to be accepted. This function assumes that LAPIC initialisation has
+ *       been performed during CPU bring-up.
+ */
+void wake_cpu(uint8_t logical_cpu_id);
+
+/**
+ * @brief Initialise the Local APIC on an Application Processor (AP).
+ *
+ * This function enables the Local APIC by setting the spurious interrupt
+ * vector register's enable bit, and lowers the Task Priority Register (TPR)
+ * to zero so that the AP can receive all interrupts.
+ *
+ * It is typically invoked during AP startup after IDT installation, allowing
+ * the processor to handle interrupts and inter-processor IPIs.
+ *
+ * @note This does not configure LAPIC timers or IPI handlers; it only ensures
+ *       that the APIC is enabled and accepting interrupts.
+ *
+ * @see apic_write
+ * @see APIC_SVR
+ * @see APIC_TPR
+ */
+void apic_setup_ap();

include/idt.h

@@ -150,3 +150,17 @@ int alloc_msi_vector(uint8_t cpu);
  */
 void free_msi_vector(uint8_t cpu, int vec);
 
+/**
+ * @brief Load the shared IDT and enable interrupts on an Application Processor (AP).
+ *
+ * This function loads the global Interrupt Descriptor Table (IDT) pointer into
+ * the AP's IDTR register using the `lidt` instruction. After the IDT is loaded,
+ * it enables external interrupts via `sti` (wrapped by interrupts_on()).
+ *
+ * It is typically called during AP startup before entering the scheduler loop,
+ * ensuring that the AP can handle interrupts using the same IDT as the BSP.
+ *
+ * @note All APs share the same IDT (idt64) in this design.
+ * @see lidt, interrupts_on
+ */
+void load_ap_shared_idt();

include/lapic_timer.h

@@ -6,6 +6,7 @@
 #ifndef __LAPIC_TIMER_H__
 #define __LAPIC_TIMER_H__
 
+#define APIC_SVR         0xF0
 #define APIC_TASKPRIOR	 0x80
 #define APIC_EOI	 0x0B0
 #define APIC_LDR	 0x0D0
@@ -30,6 +31,6 @@
 #define TMR_PERIODIC	 0x20000
 #define TMR_BASEDIV	 (1<<20)
 
-void init_lapic_timer();
+void init_lapic_timer(uint64_t quantum);
 
 #endif

src/ap.c

@@ -1,5 +1,11 @@
+/**
+ * @file ap.c
+ * @brief Bring up and enter the process loop for an AP
+ * @author Craig Edwards (craigedwards@brainbox.cc)
+ * @copyright Copyright (c) 2012-2025
+ */
+
 #include <kernel.h>
-#include <stdatomic.h>
 
 volatile struct limine_smp_request smp_request = {
 	.id = LIMINE_SMP_REQUEST,
@@ -9,36 +15,15 @@ volatile struct limine_smp_request smp_request = {
 size_t aps_online = 0;
 simple_cv_t boot_condition;
 
-void kmain_ap(struct limine_smp_info *info)
-{
-	// Load the shared IDT
-	__asm__ volatile("lidtq (%0)" :: "r"(&idt64));
-	interrupts_on();
-
+void wait_for_interpreter_start(struct limine_smp_info *info) {
 	kprintf("CPU: %u online; ID: %u\n", info->processor_id, info->lapic_id);
 	atomic_fetch_add(&aps_online, 1);
-
 	simple_cv_wait(&boot_condition);
-	/**
-	 * @todo Insert cpu-local scheduler loop here.
-	 * Each AP will run its own list of executing BASIC processes. Accessing
-	 * the list of other APs and the BSP will be strictly controlled via a
-	 * marshalled lookup system using a spinlock, e.g. if AP 1 wants to start a new
-	 * process on AP 2, it signals via this system.
-	 *
-	 * This will be done as follows:
-	 *
-	 * 1) Each AP can be instructed via a command queue to launch, query or kill a process.
-	 * 2) Each AP will have its own command queue
-	 * 3) Any AP can push a command onto the command queue for one or more other APs to action
-	 * 4) Initially AP's will wait for a start command in their queue,
-	 *    they won't run their scheduler until they receive this command. This allows them all to
-	 *    gracefully wait until the first BASIC process is ready to be loaded (/programs/init).
-	 * 5) A shared process list will be used for enumerating processes and checking process
-	 *    state, so we dont have to peek at another scheduler instance's command queue to
-	 *    see if a process ours is waiting on still lives.
-	 */
-	 dprintf("Got start signal on cpu #%d\n", info->processor_id);
+}
 
-	 proc_loop();
+void kmain_ap(struct limine_smp_info *info) {
+	load_ap_shared_idt();
+	wait_for_interpreter_start(info);
+	apic_setup_ap();
+	proc_loop();
 }

src/apic.c

@@ -68,17 +68,48 @@ uint64_t rdmsr(uint32_t msr) {
 int x2apic_supported(void) {
 	unsigned int eax, ebx, ecx, edx;
 
-	if (!__get_cpuid(1, &eax, &ebx, &ecx, &edx))
+	if (!__get_cpuid(1, &eax, &ebx, &ecx, &edx)) {
 		return 0;
+	}
 
 	return (ecx & (1u << 21)) != 0; // Bit 21 of ECX = x2APIC support
 }
 
 int x2apic_enabled(void) {
-	 if (!x2apic_supported()) {
-		 return 0; // CPU doesn't even support x2APIC
-	 }
-	 // Read APIC base MSR to see if x2APIC mode is active
-	 uint64_t apic_base = rdmsr(IA32_APIC_BASE_MSR);
-	 return (apic_base & APIC_BASE_X2APIC_ENABLE) != 0;
- }
+	if (!x2apic_supported()) {
+		return 0; // CPU doesn't even support x2APIC
+	}
+	// Read APIC base MSR to see if x2APIC mode is active
+	uint64_t apic_base = rdmsr(IA32_APIC_BASE_MSR);
+	return (apic_base & APIC_BASE_X2APIC_ENABLE) != 0;
+}
+
+void apic_send_ipi(uint32_t lapic_id, uint8_t vector) {
+	if (x2apic_enabled()) {
+		uint64_t icr = 0;
+		icr |= vector;                  // vector
+		icr |= (0ULL << 8);             // delivery mode = fixed
+		icr |= (0ULL << 11);            // physical dest
+		icr |= (1ULL << 14);            // level = assert
+		icr |= ((uint64_t)lapic_id << 32);
+		wrmsr(IA32_X2APIC_ICR, icr);
+	} else {
+		apic_write(APIC_ICR_HIGH, ((uint32_t) lapic_id) << 24);
+		apic_write(APIC_ICR_LOW,
+			   vector | APIC_DM_FIXED | APIC_DEST_NO_SHORTHAND |
+			   APIC_DEST_PHYSICAL | APIC_LEVEL_ASSERT | APIC_TRIGGER_EDGE);
+		// Wait for delivery to complete (bit 12 = Delivery Status)
+		while (apic_read(APIC_ICR_LOW) & (1 << 12)) {
+			__builtin_ia32_pause();
+		}
+	}
+}
+
+void wake_cpu(uint8_t logical_cpu_id) {
+	apic_send_ipi(get_lapic_id_from_cpu_id(logical_cpu_id), APIC_WAKE_IPI);
+}
+
+void apic_setup_ap() {
+	apic_write(APIC_SVR, apic_read(APIC_SVR) | 0x100); // Set APIC enable bit
+	apic_write(APIC_TPR, 0);
+}

src/idt.c

@@ -104,7 +104,7 @@ void init_idt() {
 #ifdef USE_IOAPIC
 	pic_disable();
 	remap_irqs_to_ioapic();
-	init_lapic_timer();
+	init_lapic_timer(100);
 #else
 	pic_enable();
 #endif
@@ -134,3 +134,8 @@ void free_msi_vector(uint8_t cpu, int vec) {
 	int bit = vec % 64;
 	msi_bitmap[cpu][w] &= ~(1ULL << bit);
 }
+
+void load_ap_shared_idt() {
+	__asm__ volatile("lidtq (%0)" :: "r"(&idt64));
+	interrupts_on();
+}