Add cacheline-alignment to compiler/threading and switch from volatile to std::atomic

common/cacheline.h

@@ -7,7 +7,11 @@
 
 #include <stdint.h>
 
+#if defined(__ARM_ARCH_7A__) || defined(__aarch64__)
+#define KDB_CACHELINE_SIZE 128
+#else
 #define KDB_CACHELINE_SIZE 64
+#endif // __ARM_ARCH_7A__ || __aarch64__
 #define KDB_CACHELINE_ALIGNED __attribute__((aligned(KDB_CACHELINE_SIZE)))
 
 #endif // KDB_COMMON_CACHELINE_H

compiler/scheduler/scheduler-base.cpp

@@ -6,22 +6,22 @@
 
 #include <cassert>
 
-volatile int tasks_before_sync_node;
+std::atomic<int> tasks_before_sync_node;
 
 static SchedulerBase *scheduler;
 
 void set_scheduler(SchedulerBase *new_scheduler) {
-  assert (scheduler == nullptr);
+  assert(scheduler == nullptr);
   scheduler = new_scheduler;
 }
 
 void unset_scheduler(SchedulerBase *old_scheduler) {
-  assert (scheduler == old_scheduler);
+  assert(scheduler == old_scheduler);
   scheduler = nullptr;
 }
 
 SchedulerBase *get_scheduler() {
-  assert (scheduler != nullptr);
+  assert(scheduler != nullptr);
   return scheduler;
 }
 

compiler/scheduler/scheduler-base.h

@@ -4,6 +4,8 @@
 
 #pragma once
 
+#include <atomic>
+
 class Node;
 
 class Task;
@@ -22,7 +24,7 @@ SchedulerBase *get_scheduler();
 void set_scheduler(SchedulerBase *new_scheduler);
 void unset_scheduler(SchedulerBase *old_scheduler);
 
-extern volatile int tasks_before_sync_node;
+extern std::atomic<int> tasks_before_sync_node;
 
 inline void register_async_task(Task *task) {
   get_scheduler()->add_task(task);

compiler/scheduler/scheduler.cpp

@@ -21,15 +21,14 @@ class ThreadContext {
   bool run_flag;
 };
 
-
 void *scheduler_thread_execute(void *arg) {
   auto *tls = (ThreadContext *)arg;
   tls->scheduler->thread_execute(tls);
   return nullptr;
 }
 
-Scheduler::Scheduler() :
-  threads_count(-1) {
+Scheduler::Scheduler()
+  : threads_count(-1) {
   task_pull = new TaskPull();
 }
 
@@ -46,7 +45,7 @@ void Scheduler::add_sync_node(Node *node) {
 }
 
 void Scheduler::add_task(Task *task) {
-  assert (task_pull != nullptr);
+  assert(task_pull != nullptr);
   task_pull->add_task(task);
 }
 
@@ -55,7 +54,7 @@ void Scheduler::execute() {
   set_thread_id(0);
   std::vector<ThreadContext> threads(threads_count + 1);
 
-  assert ((int)one_thread_nodes.size() < threads_count);
+  assert((int)one_thread_nodes.size() < threads_count);
   for (int i = 1; i <= threads_count; i++) {
     threads[i].thread_id = i;
     threads[i].scheduler = this;
@@ -67,7 +66,7 @@ void Scheduler::execute() {
   }
 
   while (true) {
-    if (tasks_before_sync_node > 0) {
+    if (tasks_before_sync_node.load(std::memory_order_acquire) > 0) {
       usleep(250);
       continue;
     }
@@ -90,7 +89,7 @@ void Scheduler::execute() {
 }
 
 void Scheduler::set_threads_count(int new_threads_count) {
-  assert (1 <= new_threads_count && new_threads_count <= MAX_THREADS_COUNT);
+  assert(1 <= new_threads_count && new_threads_count <= MAX_THREADS_COUNT);
   threads_count = new_threads_count;
 }
 
@@ -101,7 +100,7 @@ bool Scheduler::thread_process_node(Node *node) {
   }
   task->execute();
   delete task;
-  __sync_fetch_and_sub(&tasks_before_sync_node, 1);
+  tasks_before_sync_node.fetch_sub(1, std::memory_order_release);
   return true;
 }
 

compiler/stage.cpp

@@ -2,6 +2,8 @@
 // Copyright (c) 2020 LLC «V Kontakte»
 // Distributed under the GPL v3 License, see LICENSE.notice.txt
 
+#include <atomic>
+
 #include "compiler/stage.h"
 
 #include "common/termformat/termformat.h"
@@ -27,11 +29,11 @@ const char *get_assert_level_desc(AssertLevelT assert_level) {
     case FATAL_ASSERT_LEVEL:
       return "Fatal error";
     default:
-      assert (0);
+      assert(0);
   }
 }
 
-volatile int ce_locker;
+std::atomic<int> ce_locker;
 
 namespace {
 FILE *warning_file{nullptr};
@@ -41,10 +43,10 @@ void stage::set_warning_file(FILE *file) noexcept {
   warning_file = file;
 }
 
-void on_compilation_error(const char *description __attribute__((unused)), const char *file_name, int line_number,
-                          const char *full_description, AssertLevelT assert_level) {
+void on_compilation_error(const char *description __attribute__((unused)), const char *file_name, int line_number, const char *full_description,
+                          AssertLevelT assert_level) {
 
-  AutoLocker<volatile int *> locker(&ce_locker);
+  AutoLocker<std::atomic<int> *> locker(&ce_locker);
   FILE *file = stdout;
   if (assert_level == WRN_ASSERT_LEVEL && warning_file) {
     file = warning_file;
@@ -60,7 +62,7 @@ void on_compilation_error(const char *description __attribute__((unused)), const
                       "It is probably happened due to incorrect or unsupported PHP input.\n"
                       "But it is still bug in compiler.\n");
 #ifdef __arm64__
-    __builtin_debugtrap();  // for easier debugging kphp_assert / kphp_fail
+    __builtin_debugtrap(); // for easier debugging kphp_assert / kphp_fail
 #endif
     abort();
   }
@@ -73,7 +75,6 @@ void on_compilation_error(const char *description __attribute__((unused)), const
   fflush(file);
 }
 
-
 void Location::set_file(SrcFilePtr new_file) {
   file = new_file;
   function = FunctionPtr();
@@ -92,10 +93,10 @@ void Location::set_line(int new_line) {
   line = new_line;
 }
 
-Location::Location(const SrcFilePtr &file, const FunctionPtr &function, int line) :
-  file(file),
-  function(function),
-  line(line) {}
+Location::Location(const SrcFilePtr &file, const FunctionPtr &function, int line)
+  : file(file)
+  , function(function)
+  , line(line) {}
 
 // return a location in the format: "{file}:{line}  in {function}"
 std::string Location::as_human_readable() const {
@@ -249,8 +250,9 @@ const std::string &stage::get_function_name() {
   return function->name;
 }
 
-bool stage::should_be_colored(FILE *f)  {
-  if (!G) return TermStringFormat::is_terminal(f);
+bool stage::should_be_colored(FILE *f) {
+  if (!G)
+    return TermStringFormat::is_terminal(f);
   switch (G->settings().get_color_settings()) {
     case CompilerSettings::colored:
       return true;

compiler/threading/data-stream.h

@@ -21,10 +21,8 @@ class DataStream {
   template<size_t data_id>
   using NthDataType = DataType;
 
-  explicit DataStream(bool is_sink = false) :
-    is_sink_mode_(is_sink)
-  {
-  }
+  explicit DataStream(bool is_sink = false)
+    : is_sink_mode_(is_sink) {}
 
   bool get(DataType &result) {
     std::lock_guard<std::mutex> lock{mutex_};
@@ -38,7 +36,7 @@ class DataStream {
 
   void operator<<(DataType input) {
     if (!is_sink_mode_) {
-      __sync_fetch_and_add(&tasks_before_sync_node, 1);
+      tasks_before_sync_node.fetch_add(1, std::memory_order_release);
     }
     std::lock_guard<std::mutex> lock{mutex_};
     queue_.push_front(std::move(input));
@@ -60,13 +58,12 @@ class DataStream {
   const bool is_sink_mode_;
 };
 
-
 struct EmptyStream {
   template<size_t stream_id>
   using NthDataType = EmptyStream;
 };
 
-template<class ...DataTypes>
+template<class... DataTypes>
 class MultipleDataStreams {
 private:
   std::tuple<DataStream<DataTypes> *...> streams_;

compiler/threading/hash-table.h

@@ -13,35 +13,35 @@ template<class T, int N = 1000000>
 class TSHashTable {
 public:
   struct HTNode : Lockable {
-    unsigned long long hash;
+    std::atomic<unsigned long long> hash;
     T data;
 
-    HTNode() :
-      hash(0),
-      data() {
-    }
+    HTNode()
+      : hash(0)
+      , data() {}
   };
 
 private:
   HTNode *nodes;
-  int used_size;
+  std::atomic<int> used_size;
+
 public:
-  TSHashTable() :
-    nodes(new HTNode[N]),
-    used_size(0) {
-  }
+  TSHashTable()
+    : nodes(new HTNode[N])
+    , used_size(0) {}
 
   HTNode *at(unsigned long long hash) {
     int i = (unsigned)hash % (unsigned)N;
     while (true) {
-      while (nodes[i].hash != 0 && nodes[i].hash != hash) {
+      while (nodes[i].hash.load(std::memory_order_acquire) != 0 && nodes[i].hash.load(std::memory_order_relaxed) != hash) {
         i++;
         if (i == N) {
           i = 0;
         }
       }
-      if (nodes[i].hash == 0 && !__sync_bool_compare_and_swap(&nodes[i].hash, 0, hash)) {
-        int id = __sync_fetch_and_add(&used_size, 1);
+      unsigned long long expected = 0;
+      if (nodes[i].hash.load(std::memory_order_acquire) == 0 && !nodes[i].hash.compare_exchange_strong(expected, hash, std::memory_order_acq_rel)) {
+        int id = used_size.fetch_add(1, std::memory_order_release);
         assert(id * 2 < N);
         continue;
       }
@@ -52,20 +52,20 @@ class TSHashTable {
 
   const T *find(unsigned long long hash) {
     int i = (unsigned)hash % (unsigned)N;
-    while (nodes[i].hash != 0 && nodes[i].hash != hash) {
+    while (nodes[i].hash.load(std::memory_order_acquire) != 0 && nodes[i].hash.load(std::memory_order_relaxed) != hash) {
       i++;
       if (i == N) {
         i = 0;
       }
     }
 
-    return nodes[i].hash == hash ? &nodes[i].data : nullptr;
+    return nodes[i].hash.load(std::memory_order_acquire) == hash ? &nodes[i].data : nullptr;
   }
 
   std::vector<T> get_all() {
     std::vector<T> res;
     for (int i = 0; i < N; i++) {
-      if (nodes[i].hash != 0) {
+      if (nodes[i].hash.load(std::memory_order_acquire) != 0) {
         res.push_back(nodes[i].data);
       }
     }
@@ -76,7 +76,7 @@ class TSHashTable {
   std::vector<T> get_all_if(const CondF &callbackF) {
     std::vector<T> res;
     for (int i = 0; i < N; i++) {
-      if (nodes[i].hash != 0 && callbackF(nodes[i].data)) {
+      if (nodes[i].hash.load(std::memory_order_acquire) != 0 && callbackF(nodes[i].data)) {
         res.push_back(nodes[i].data);
       }
     }