[Concurrency] Use SchedulerPrivate for the "next waiting task" link.

Reduce the size of AsyncTask by using the first slot of SchedulerPrivate
for the next waiting task. Thanks, John!

Author: DougGregor

include/swift/ABI/Task.h

@@ -88,6 +88,13 @@ class AsyncFunctionPointer {
 
 /// A schedulable job.
 class alignas(2 * alignof(void*)) Job {
+protected:
+  // Indices into SchedulerPrivate, for use by the runtime.
+  enum {
+    /// The next waiting task link, an AsyncTask that is waiting on a future.
+    NextWaitingTaskIndex = 0,
+  };
+
 public:
   // Reserved for the use of the scheduler.
   void *SchedulerPrivate[2];
@@ -190,16 +197,12 @@ class AsyncTask : public HeapObject, public Job {
   /// Reserved for the use of the task-local stack allocator.
   void *AllocatorPrivate[4];
 
-  /// The next task in the linked list of waiting tasks.
-  AsyncTask *NextWaitingTask;
-
   AsyncTask(const HeapMetadata *metadata, JobFlags flags,
             TaskContinuationFunction *run,
             AsyncContext *initialContext)
     : HeapObject(metadata), Job(flags, run),
       ResumeContext(initialContext),
-      Status(ActiveTaskStatus()),
-      NextWaitingTask(nullptr) {
+      Status(ActiveTaskStatus()) {
     assert(flags.isAsyncTask());
   }
 
@@ -265,9 +268,9 @@ class AsyncTask : public HeapObject, public Job {
     /// head of the list of tasks.
     struct WaitQueueItem {
       /// Mask used for the low status bits in a wait queue item.
-      const uintptr_t statusMask = 0x03;
+      static const uintptr_t statusMask = 0x03;
 
-      uintptr_t storage;
+      const uintptr_t storage;
 
       Status getStatus() const {
         return static_cast<Status>(storage & statusMask);
@@ -359,13 +362,20 @@ class AsyncTask : public HeapObject, public Job {
   /// executor.
   void completeFuture(AsyncContext *context, ExecutorRef executor);
 
+  /// Access the next waiting task, which establishes a singly linked list of
+  /// tasks that are waiting on a future.
+  AsyncTask *&getNextWaitingTask() {
+    return reinterpret_cast<AsyncTask *&>(
+        SchedulerPrivate[NextWaitingTaskIndex]);
+  }
+
   static bool classof(const Job *job) {
     return job->isAsyncTask();
   }
 };
 
 // The compiler will eventually assume these.
-static_assert(sizeof(AsyncTask) == 14 * sizeof(void*),
+static_assert(sizeof(AsyncTask) == 12 * sizeof(void*),
               "AsyncTask size is wrong");
 static_assert(alignof(AsyncTask) == 2 * alignof(void*),
               "AsyncTask alignment is wrong");

stdlib/public/Concurrency/Task.cpp

@@ -64,7 +64,7 @@ FutureFragment::Status AsyncTask::waitFuture(AsyncTask *waitingTask) {
     }
 
     // Put the waiting task at the beginning of the wait queue.
-    waitingTask->NextWaitingTask = queueHead.getTask();
+    waitingTask->getNextWaitingTask() = queueHead.getTask();
     auto newQueueHead = WaitQueueItem::get(Status::Executing, waitingTask);
     if (fragment->waitQueue.compare_exchange_weak(
             queueHead, newQueueHead, std::memory_order_release,
@@ -119,10 +119,10 @@ void AsyncTask::completeFuture(AsyncContext *context, ExecutorRef executor) {
   auto waitingTask = queueHead.getTask();
   while (waitingTask) {
     // Find the next waiting task.
-    auto nextWaitingTask = waitingTask->NextWaitingTask;
+    auto nextWaitingTask = waitingTask->getNextWaitingTask();
 
     // Remove this task from the list.
-    waitingTask->NextWaitingTask = nullptr;
+    waitingTask->getNextWaitingTask() = nullptr;
 
     // TODO: schedule this task on the executor rather than running it
     // directly.

unittests/runtime/TaskFuture.cpp

@@ -196,7 +196,7 @@ TEST(TaskFutureTest, objectFuture) {
     EXPECT_EQ(object, *reinterpret_cast<TestObject **>(waitResult.storage));
 
     // Make sure the object hasn't been destroyed.
-    EXPECT_EQ(7, objectValueOnComplete);
+    EXPECT_EQ(size_t(7), objectValueOnComplete);
 
     // Okay, release the task. This should destroy the object.
     swift_release(task);