Kernel: Rewrite WaitQueue

Kernel/Bus/USB/xHCI/xHCIController.h

@@ -12,6 +12,7 @@
 #include <Kernel/Bus/USB/xHCI/xHCIRootHub.h>
 #include <Kernel/Interrupts/GenericInterruptHandler.h>
 #include <Kernel/Memory/TypedMapping.h>
+#include <Kernel/Tasks/DeprecatedWaitQueue.h>
 
 namespace Kernel::USB {
 
@@ -1062,7 +1063,7 @@ class xHCIController
         u32 end_index { 0 };
     };
     struct SyncPendingTransfer : public PendingTransfer {
-        WaitQueue wait_queue;
+        DeprecatedWaitQueue wait_queue;
         TransferRequestBlock::CompletionCode completion_code { TransferRequestBlock::CompletionCode::Invalid };
         u32 remainder { 0 };
     };
@@ -1169,7 +1170,7 @@ class xHCIController
     DoorbellRegisters volatile& m_doorbell_registers;
 
     RefPtr<Process> m_process;
-    WaitQueue m_event_queue;
+    DeprecatedWaitQueue m_event_queue;
 
     bool m_using_message_signalled_interrupts { false };
     bool m_large_contexts { false };
@@ -1181,7 +1182,7 @@ class xHCIController
     Vector<NonnullOwnPtr<PeriodicPendingTransfer>> m_active_periodic_transfers;
 
     Spinlock<LockRank::None> m_command_lock;
-    WaitQueue m_command_completion_queue;
+    DeprecatedWaitQueue m_command_completion_queue;
     TransferRequestBlock m_command_result_transfer_request_block {};
     u32 m_command_ring_enqueue_index { 0 };
     u8 m_command_ring_producer_cycle_state { 1 };

Kernel/CMakeLists.txt

@@ -380,6 +380,7 @@ set(KERNEL_SOURCES
     Devices/TTY/VirtualConsole.cpp
     Tasks/Coredump.cpp
     Tasks/CrashHandler.cpp
+    Tasks/DeprecatedWaitQueue.cpp
     Tasks/FinalizerTask.cpp
     Tasks/FutexQueue.cpp
     Tasks/HostnameContext.cpp

Kernel/Devices/AsyncDeviceRequest.h

@@ -10,9 +10,9 @@
 #include <Kernel/Library/NonnullLockRefPtr.h>
 #include <Kernel/Library/UserOrKernelBuffer.h>
 #include <Kernel/Memory/ScopedAddressSpaceSwitcher.h>
+#include <Kernel/Tasks/DeprecatedWaitQueue.h>
 #include <Kernel/Tasks/Process.h>
 #include <Kernel/Tasks/Thread.h>
-#include <Kernel/Tasks/WaitQueue.h>
 
 namespace Kernel {
 
@@ -160,7 +160,7 @@ class AsyncDeviceRequest : public AtomicRefCounted<AsyncDeviceRequest> {
 
     AsyncDeviceSubRequestList m_sub_requests_pending;
     AsyncDeviceSubRequestList m_sub_requests_complete;
-    WaitQueue m_queue;
+    DeprecatedWaitQueue m_queue;
     LockWeakPtr<Process> const m_process;
     void* m_private { nullptr };
     mutable Spinlock<LockRank::None> m_lock {};

Kernel/Devices/Audio/AC97/AC97.h

@@ -182,7 +182,7 @@ class AC97 final
     bool m_double_rate_pcm_enabled { false };
     NonnullOwnPtr<IOWindow> m_mixer_io_window;
     NonnullOwnPtr<IOWindow> m_bus_io_window;
-    WaitQueue m_irq_queue;
+    DeprecatedWaitQueue m_irq_queue;
     OwnPtr<Memory::Region> m_output_buffer;
     u8 m_output_buffer_page_count { 4 };
     u8 m_output_buffer_page_index { 0 };

Kernel/Devices/Audio/IntelHDA/Stream.h

@@ -14,7 +14,7 @@
 #include <Kernel/Devices/Audio/IntelHDA/Format.h>
 #include <Kernel/Library/IOWindow.h>
 #include <Kernel/Locking/SpinlockProtected.h>
-#include <Kernel/Tasks/WaitQueue.h>
+#include <Kernel/Tasks/DeprecatedWaitQueue.h>
 
 namespace Kernel::Audio::IntelHDA {
 
@@ -91,7 +91,7 @@ class Stream {
     OwnPtr<Memory::Region> m_buffer_descriptor_list;
     RecursiveSpinlockProtected<OwnPtr<Memory::Region>, LockRank::None> m_buffers;
     size_t m_buffer_position { 0 };
-    WaitQueue m_irq_queue;
+    DeprecatedWaitQueue m_irq_queue;
     bool m_running { false };
     FormatParameters m_format_parameters;
 };

Kernel/Devices/Serial/VirtIO/Console.h

@@ -73,7 +73,7 @@ class Console
     OwnPtr<Memory::RingBuffer> m_control_transmit_buffer;
     OwnPtr<Memory::RingBuffer> m_control_receive_buffer;
 
-    WaitQueue m_control_wait_queue;
+    DeprecatedWaitQueue m_control_wait_queue;
 
     static unsigned next_device_id;
 };

Kernel/Devices/Storage/AHCI/InterruptHandler.h

@@ -17,7 +17,6 @@
 #include <Kernel/Memory/PhysicalRAMPage.h>
 #include <Kernel/Sections.h>
 #include <Kernel/Security/Random.h>
-#include <Kernel/Tasks/WaitQueue.h>
 
 namespace Kernel {
 

Kernel/Devices/Storage/AHCI/Port.h

@@ -23,7 +23,6 @@
 #include <Kernel/Memory/ScatterGatherList.h>
 #include <Kernel/Sections.h>
 #include <Kernel/Security/Random.h>
-#include <Kernel/Tasks/WaitQueue.h>
 
 namespace Kernel {
 

Kernel/Devices/Storage/NVMe/NVMeQueue.cpp

@@ -160,7 +160,7 @@ u16 NVMeQueue::submit_sync_sqe(NVMeSubmission& sub)
     });
     submit_sqe(sub);
 
-    // FIXME: Only sync submissions (usually used for admin commands) use a WaitQueue based IO. Eventually we need to
+    // FIXME: Only sync submissions (usually used for admin commands) use a DeprecatedWaitQueue based IO. Eventually we need to
     //  move this logic into the block layer instead of sprinkling them in the driver code.
     m_sync_wait_queue.wait_forever("NVMe sync submit"sv);
     return cmd_status;

Kernel/Devices/Storage/NVMe/NVMeQueue.h

@@ -18,6 +18,7 @@
 #include <Kernel/Locking/Spinlock.h>
 #include <Kernel/Memory/MemoryManager.h>
 #include <Kernel/Memory/TypedMapping.h>
+#include <Kernel/Tasks/DeprecatedWaitQueue.h>
 
 namespace Kernel {
 
@@ -129,7 +130,7 @@ class NVMeQueue : public AtomicRefCounted<NVMeQueue> {
     Span<NVMeSubmission> m_sqe_array;
     OwnPtr<Memory::Region> m_sq_dma_region;
     Span<NVMeCompletion> m_cqe_array;
-    WaitQueue m_sync_wait_queue;
+    DeprecatedWaitQueue m_sync_wait_queue;
     Doorbell m_db_regs;
     NonnullRefPtr<Memory::PhysicalRAMPage const> const m_rw_dma_page;
 };

Kernel/Devices/Storage/StorageController.h

@@ -15,7 +15,6 @@
 #include <Kernel/Memory/PhysicalAddress.h>
 #include <Kernel/Memory/PhysicalRAMPage.h>
 #include <Kernel/Security/Random.h>
-#include <Kernel/Tasks/WaitQueue.h>
 
 namespace Kernel {
 

Kernel/FileSystem/FIFO.h

@@ -9,7 +9,7 @@
 #include <Kernel/FileSystem/File.h>
 #include <Kernel/Library/DoubleBuffer.h>
 #include <Kernel/Locking/Mutex.h>
-#include <Kernel/Tasks/WaitQueue.h>
+#include <Kernel/Tasks/DeprecatedWaitQueue.h>
 #include <Kernel/UnixTypes.h>
 
 namespace Kernel {
@@ -54,8 +54,8 @@ class FIFO final : public File {
 
     int m_fifo_id { 0 };
 
-    WaitQueue m_read_open_queue;
-    WaitQueue m_write_open_queue;
+    DeprecatedWaitQueue m_read_open_queue;
+    DeprecatedWaitQueue m_write_open_queue;
     Mutex m_open_lock;
 };
 

Kernel/Forward.h

@@ -19,6 +19,7 @@ class Coredump;
 class Credentials;
 class CustodyBase;
 class Custody;
+class DeprecatedWaitQueue;
 class Device;
 class DeviceControlDevice;
 class DiskCache;

Kernel/Locking/Mutex.h

@@ -14,7 +14,8 @@
 #include <Kernel/Forward.h>
 #include <Kernel/Locking/LockLocation.h>
 #include <Kernel/Locking/LockMode.h>
-#include <Kernel/Tasks/WaitQueue.h>
+#include <Kernel/Locking/SpinlockProtected.h>
+#include <Kernel/Tasks/Thread.h>
 
 namespace Kernel {
 

Kernel/Locking/SpinlockProtectedBase.h

@@ -10,10 +10,13 @@
 
 namespace Kernel {
 
+class WaitQueue;
+
 template<typename T, typename Lock>
 class SpinlockProtectedBase {
     AK_MAKE_NONCOPYABLE(SpinlockProtectedBase);
     AK_MAKE_NONMOVABLE(SpinlockProtectedBase);
+    friend class WaitQueue;
 
 private:
     template<typename U>

Kernel/Net/Intel/E1000NetworkAdapter.h

@@ -105,6 +105,6 @@ class E1000NetworkAdapter : public NetworkAdapter
     bool m_link_up { false };
     EntropySource m_entropy_source;
 
-    WaitQueue m_wait_queue;
+    DeprecatedWaitQueue m_wait_queue;
 };
 }

Kernel/Net/NetworkTask.cpp

@@ -62,7 +62,7 @@ void NetworkTask_main(void*)
 {
     delayed_ack_sockets = new HashTable<NonnullRefPtr<TCPSocket>>;
 
-    WaitQueue packet_wait_queue;
+    DeprecatedWaitQueue packet_wait_queue;
     int pending_packets = 0;
     NetworkingManagement::the().for_each([&](auto& adapter) {
         dmesgln("NetworkTask: {} network adapter found: hw={}", adapter.class_name(), adapter.mac_address().to_string());

Kernel/Net/Realtek/RTL8168NetworkAdapter.h

@@ -219,6 +219,6 @@ class RTL8168NetworkAdapter final : public NetworkAdapter
     u16 m_tx_free_index { 0 };
     bool m_link_up { false };
     EntropySource m_entropy_source;
-    WaitQueue m_wait_queue;
+    DeprecatedWaitQueue m_wait_queue;
 };
 }

Kernel/Security/Random.h

@@ -13,6 +13,7 @@
 #include <Kernel/Arch/Processor.h>
 #include <Kernel/Library/StdLib.h>
 #include <Kernel/Locking/Mutex.h>
+#include <Kernel/Tasks/DeprecatedWaitQueue.h>
 #include <LibCrypto/Cipher/AES.h>
 #include <LibCrypto/Cipher/Cipher.h>
 #include <LibCrypto/Hash/SHA2.h>
@@ -131,7 +132,7 @@ class KernelRng : public FortunaPRNG<Crypto::Cipher::AESCipher, Crypto::Hash::SH
     void wake_if_ready();
 
 private:
-    WaitQueue m_seed_queue;
+    DeprecatedWaitQueue m_seed_queue;
 };
 
 class EntropySource {

Kernel/Tasks/DeprecatedWaitQueue.cpp

@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) 2020, the SerenityOS developers.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <Kernel/Debug.h>
+#include <Kernel/Tasks/DeprecatedWaitQueue.h>
+#include <Kernel/Tasks/Thread.h>
+
+namespace Kernel {
+
+bool DeprecatedWaitQueue::should_add_blocker(Thread::Blocker& b, void*)
+{
+    VERIFY(m_lock.is_locked());
+    VERIFY(b.blocker_type() == Thread::Blocker::Type::Queue);
+    if (m_wake_requested) {
+        m_wake_requested = false;
+        dbgln_if(WAITQUEUE_DEBUG, "DeprecatedWaitQueue @ {}: do not block thread {}", this, b.thread());
+        return false;
+    }
+    dbgln_if(WAITQUEUE_DEBUG, "DeprecatedWaitQueue @ {}: should block thread {}", this, b.thread());
+    return true;
+}
+
+u32 DeprecatedWaitQueue::wake_one()
+{
+    u32 did_wake = 0;
+    SpinlockLocker lock(m_lock);
+    dbgln_if(WAITQUEUE_DEBUG, "DeprecatedWaitQueue @ {}: wake_one", this);
+    bool did_unblock_one = unblock_all_blockers_whose_conditions_are_met_locked([&](Thread::Blocker& b, void*, bool& stop_iterating) {
+        VERIFY(b.blocker_type() == Thread::Blocker::Type::Queue);
+        auto& blocker = static_cast<Thread::DeprecatedWaitQueueBlocker&>(b);
+        dbgln_if(WAITQUEUE_DEBUG, "DeprecatedWaitQueue @ {}: wake_one unblocking {}", this, blocker.thread());
+        if (blocker.unblock()) {
+            stop_iterating = true;
+            did_wake = 1;
+            return true;
+        }
+        return false;
+    });
+    m_wake_requested = !did_unblock_one;
+    dbgln_if(WAITQUEUE_DEBUG, "DeprecatedWaitQueue @ {}: wake_one woke {} threads", this, did_wake);
+    return did_wake;
+}
+
+u32 DeprecatedWaitQueue::wake_n(u32 wake_count)
+{
+    if (wake_count == 0)
+        return 0; // should we assert instead?
+    SpinlockLocker lock(m_lock);
+    dbgln_if(WAITQUEUE_DEBUG, "DeprecatedWaitQueue @ {}: wake_n({})", this, wake_count);
+    u32 did_wake = 0;
+
+    bool did_unblock_some = unblock_all_blockers_whose_conditions_are_met_locked([&](Thread::Blocker& b, void*, bool& stop_iterating) {
+        VERIFY(b.blocker_type() == Thread::Blocker::Type::Queue);
+        auto& blocker = static_cast<Thread::DeprecatedWaitQueueBlocker&>(b);
+        dbgln_if(WAITQUEUE_DEBUG, "DeprecatedWaitQueue @ {}: wake_n unblocking {}", this, blocker.thread());
+        VERIFY(did_wake < wake_count);
+        if (blocker.unblock()) {
+            if (++did_wake >= wake_count)
+                stop_iterating = true;
+            return true;
+        }
+        return false;
+    });
+    m_wake_requested = !did_unblock_some;
+    dbgln_if(WAITQUEUE_DEBUG, "DeprecatedWaitQueue @ {}: wake_n({}) woke {} threads", this, wake_count, did_wake);
+    return did_wake;
+}
+
+u32 DeprecatedWaitQueue::wake_all()
+{
+    SpinlockLocker lock(m_lock);
+
+    dbgln_if(WAITQUEUE_DEBUG, "DeprecatedWaitQueue @ {}: wake_all", this);
+    u32 did_wake = 0;
+
+    bool did_unblock_any = unblock_all_blockers_whose_conditions_are_met_locked([&](Thread::Blocker& b, void*, bool&) {
+        VERIFY(b.blocker_type() == Thread::Blocker::Type::Queue);
+        auto& blocker = static_cast<Thread::DeprecatedWaitQueueBlocker&>(b);
+
+        dbgln_if(WAITQUEUE_DEBUG, "DeprecatedWaitQueue @ {}: wake_all unblocking {}", this, blocker.thread());
+
+        if (blocker.unblock()) {
+            did_wake++;
+            return true;
+        }
+        return false;
+    });
+    m_wake_requested = !did_unblock_any;
+    dbgln_if(WAITQUEUE_DEBUG, "DeprecatedWaitQueue @ {}: wake_all woke {} threads", this, did_wake);
+    return did_wake;
+}
+
+}

Kernel/Tasks/DeprecatedWaitQueue.h

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2020, the SerenityOS developers.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/Atomic.h>
+#include <Kernel/Locking/Spinlock.h>
+#include <Kernel/Tasks/Thread.h>
+
+namespace Kernel {
+
+class DeprecatedWaitQueue final : public Thread::BlockerSet {
+public:
+    u32 wake_one();
+    u32 wake_n(u32 wake_count);
+    u32 wake_all();
+
+    template<class... Args>
+    Thread::BlockResult wait_on(Thread::BlockTimeout const& timeout, Args&&... args)
+    {
+        return Thread::current()->block<Thread::DeprecatedWaitQueueBlocker>(timeout, *this, forward<Args>(args)...);
+    }
+
+    template<class... Args>
+    void wait_forever(Args&&... args)
+    {
+        (void)Thread::current()->block<Thread::DeprecatedWaitQueueBlocker>({}, *this, forward<Args>(args)...);
+    }
+
+protected:
+    virtual bool should_add_blocker(Thread::Blocker& b, void*) override;
+
+private:
+    bool m_wake_requested { false };
+};
+
+}