diff --git a/.github/workflows/ci.linux.arm.yml b/.github/workflows/ci.linux.arm.yml index c7766b4b7..98275b076 100644 --- a/.github/workflows/ci.linux.arm.yml +++ b/.github/workflows/ci.linux.arm.yml @@ -11,7 +11,7 @@ jobs: runs-on: [self-hosted, Linux, ARM64] container: - image: dokken/centos-stream-8:sha-40294ce + image: ghcr.io/coldwings/photon-ut-base:latest options: --cpus 4 steps: @@ -23,14 +23,14 @@ jobs: - uses: actions/checkout@v3 - - name: Install Dependencies - run: | - dnf install -y git gcc-c++ cmake 'dnf-command(config-manager)' - dnf install -y gcc-toolset-9-gcc-c++ - dnf install -y openssl-devel libcurl-devel libaio-devel - dnf install -y epel-release - dnf config-manager --set-enabled powertools - dnf install -y gtest-devel gmock-devel gflags-devel fuse-devel libgsasl-devel e2fsprogs-devel + # - name: Install Dependencies + # run: | + # dnf install -y git gcc-c++ cmake 'dnf-command(config-manager)' + # dnf install -y gcc-toolset-9-gcc-c++ + # dnf install -y openssl-devel libcurl-devel libaio-devel + # dnf install -y epel-release + # dnf config-manager --set-enabled powertools + # dnf install -y gtest-devel gmock-devel gflags-devel fuse-devel libgsasl-devel e2fsprogs-devel - name: Build run: | @@ -51,7 +51,7 @@ jobs: runs-on: [self-hosted, Linux, ARM64] container: - image: dokken/centos-stream-8:sha-40294ce + image: ghcr.io/coldwings/photon-ut-base:latest options: --cpus 4 steps: @@ -63,14 +63,14 @@ jobs: - uses: actions/checkout@v3 - - name: Install Dependencies - run: | - dnf install -y git gcc-c++ cmake 'dnf-command(config-manager)' - dnf install -y gcc-toolset-9-gcc-c++ - dnf install -y openssl-devel libcurl-devel libaio-devel - dnf install -y epel-release - dnf config-manager --set-enabled powertools - dnf install -y gtest-devel gmock-devel gflags-devel fuse-devel libgsasl-devel e2fsprogs-devel + # - name: Install Dependencies + # run: | + # dnf install -y git gcc-c++ cmake 'dnf-command(config-manager)' + # dnf install -y gcc-toolset-9-gcc-c++ + # dnf install -y openssl-devel libcurl-devel libaio-devel + # dnf install -y epel-release + # dnf config-manager --set-enabled powertools + # dnf install -y gtest-devel gmock-devel gflags-devel fuse-devel libgsasl-devel e2fsprogs-devel - name: Build run: | diff --git a/.github/workflows/ci.linux.x86.yml b/.github/workflows/ci.linux.x86.yml index 940ac7ca5..ba225852b 100644 --- a/.github/workflows/ci.linux.x86.yml +++ b/.github/workflows/ci.linux.x86.yml @@ -11,7 +11,7 @@ jobs: runs-on: [self-hosted, Linux, X64] container: - image: dokken/centos-stream-8:sha-40294ce + image: ghcr.io/coldwings/photon-ut-base:latest options: --cpus 4 steps: @@ -23,14 +23,14 @@ jobs: - uses: actions/checkout@v3 - - name: Install Dependencies - run: | - dnf install -y git gcc-c++ cmake 'dnf-command(config-manager)' - dnf install -y gcc-toolset-9-gcc-c++ - dnf install -y openssl-devel libcurl-devel libaio-devel - dnf install -y epel-release - dnf config-manager --set-enabled powertools - dnf install -y gtest-devel gmock-devel gflags-devel fuse-devel libgsasl-devel e2fsprogs-devel + # - name: Install Dependencies + # run: | + # dnf install -y git gcc-c++ cmake 'dnf-command(config-manager)' + # dnf install -y gcc-toolset-9-gcc-c++ + # dnf install -y openssl-devel libcurl-devel libaio-devel + # dnf install -y epel-release + # dnf config-manager --set-enabled powertools + # dnf install -y gtest-devel gmock-devel gflags-devel fuse-devel libgsasl-devel e2fsprogs-devel - name: Build run: | @@ -51,7 +51,7 @@ jobs: runs-on: [self-hosted, Linux, X64] container: - image: dokken/centos-stream-8:sha-40294ce + image: ghcr.io/coldwings/photon-ut-base:latest options: --cpus 4 steps: @@ -63,14 +63,14 @@ jobs: - uses: actions/checkout@v3 - - name: Install Dependencies - run: | - dnf install -y git gcc-c++ cmake 'dnf-command(config-manager)' - dnf install -y gcc-toolset-9-gcc-c++ - dnf install -y openssl-devel libcurl-devel libaio-devel - dnf install -y epel-release - dnf config-manager --set-enabled powertools - dnf install -y gtest-devel gmock-devel gflags-devel fuse-devel libgsasl-devel e2fsprogs-devel + # - name: Install Dependencies + # run: | + # dnf install -y git gcc-c++ cmake 'dnf-command(config-manager)' + # dnf install -y gcc-toolset-9-gcc-c++ + # dnf install -y openssl-devel libcurl-devel libaio-devel + # dnf install -y epel-release + # dnf config-manager --set-enabled powertools + # dnf install -y gtest-devel gmock-devel gflags-devel fuse-devel libgsasl-devel e2fsprogs-devel - name: Build run: | @@ -91,7 +91,7 @@ jobs: runs-on: [self-hosted, Linux, X64] container: - image: dokken/centos-stream-8:sha-40294ce + image: ghcr.io/coldwings/photon-ut-base:latest # In order to run io_uring, the docker daemon should add --default-ulimit memlock=-1:-1 options: --cpus 4 @@ -104,13 +104,13 @@ jobs: - uses: actions/checkout@v3 - - name: Install Dependencies - run: | - dnf install -y git gcc-c++ cmake - dnf install -y gcc-toolset-9-gcc-c++ - dnf install -y openssl-devel libcurl-devel - dnf install -y epel-release - dnf install -y fuse-devel libgsasl-devel e2fsprogs-devel + # - name: Install Dependencies + # run: | + # dnf install -y git gcc-c++ cmake + # dnf install -y gcc-toolset-9-gcc-c++ + # dnf install -y openssl-devel libcurl-devel + # dnf install -y epel-release + # dnf install -y fuse-devel libgsasl-devel e2fsprogs-devel - name: Build run: | diff --git a/.github/workflows/ci.macos.arm.yml b/.github/workflows/ci.macos.arm.yml index c8e177180..45558cabf 100644 --- a/.github/workflows/ci.macos.arm.yml +++ b/.github/workflows/ci.macos.arm.yml @@ -8,14 +8,14 @@ on: jobs: macOS-clang-debug: - runs-on: [self-hosted, macOS, ARM64] + runs-on: macos-14 steps: -# - uses: szenius/set-timezone@v1.2 -# with: -# timezoneLinux: "Asia/Shanghai" -# timezoneMacos: "Asia/Shanghai" -# timezoneWindows: "China Standard Time" + - uses: szenius/set-timezone@v1.2 + with: + timezoneLinux: "Asia/Shanghai" + timezoneMacos: "Asia/Shanghai" + timezoneWindows: "China Standard Time" - uses: actions/checkout@v3 diff --git a/common/alog.cpp b/common/alog.cpp index e371b8015..3aa9bb5c4 100644 --- a/common/alog.cpp +++ b/common/alog.cpp @@ -204,23 +204,26 @@ void LogFormatter::put_integer_dec(ALogBuffer& buf, ALogInteger x) } __attribute__((constructor)) static void __initial_timezone() { tzset(); } -static time_t dayid = 0; +static time_t dayid = 0, minuteid = 0, tsdelta = 0; static struct tm alog_time = {0}; -struct tm* alog_update_time(time_t now) -{ - auto now0 = now; +struct tm* alog_update_time(time_t now0) { + auto now = now0 + tsdelta; int sec = now % 60; now /= 60; + if (unlikely(now != minuteid)) { // calibrate wall time every minute + now = time(0) - timezone; + tsdelta = now - now0; + sec = now % 60; now /= 60; + minuteid = now; + } int min = now % 60; now /= 60; int hor = now % 24; now /= 24; - if (now != dayid) - { + if (unlikely(now != dayid)) { dayid = now; - gmtime_r(&now0, &alog_time); + auto now_ = now0 + tsdelta; + gmtime_r(&now_, &alog_time); alog_time.tm_year+=1900; alog_time.tm_mon++; - } - else - { + } else { alog_time.tm_sec = sec; alog_time.tm_min = min; alog_time.tm_hour = hor; @@ -499,7 +502,7 @@ LogBuffer& operator << (LogBuffer& log, const Prologue& pro) auto t = &alog_time; #else auto ts = photon::alog_update_now(); - auto t = alog_update_time(ts.tv_sec - timezone); + auto t = alog_update_time(ts.tv_sec); #endif log.printf(t->tm_year, '/'); log.printf(DEC_W2P0(t->tm_mon), '/'); diff --git a/common/enumerable.h b/common/enumerable.h index 70500ac16..ab86c4e72 100644 --- a/common/enumerable.h +++ b/common/enumerable.h @@ -38,11 +38,11 @@ struct Enumerable { T* obj; explicit iterator(T* obj) : obj(obj) { - if (obj && obj->next() < 0) + if (obj && !obj->valid()) this->obj = nullptr; } using R = typename std::result_of::type; - R operator*() { return obj->get(); } + R operator*() { return obj ? obj->get() : R{}; } bool operator==(const iterator& rhs) const { return obj == rhs.obj; } bool operator!=(const iterator& rhs) const { return !(*this == rhs); } iterator& operator++() @@ -92,6 +92,7 @@ inline void __example_of_enumerable__() { struct exam { + bool valid() { return false; } int next() { return -1; } // move to next, return 0 for success, -1 for failure double* get() { return nullptr; } // get current result }; diff --git a/common/estring.h b/common/estring.h index 10464b212..4e009ba3c 100644 --- a/common/estring.h +++ b/common/estring.h @@ -159,7 +159,7 @@ class estring_view : public std::string_view } iterator& operator++() { - _part = _host->find_part(_part.end()); + _part = _host->find_part(_part.end() + 1); return *this; } iterator& operator++(int) diff --git a/common/lockfree_queue.h b/common/lockfree_queue.h index 8bdcaac37..51ce0074f 100644 --- a/common/lockfree_queue.h +++ b/common/lockfree_queue.h @@ -179,7 +179,7 @@ class LockfreeMPMCRingQueue : public LockfreeRingQueueBase { using Base::empty; using Base::full; - bool push_weak(const T& x) { + bool push(const T& x) { auto t = tail.load(std::memory_order_acquire); for (;;) { auto& slot = slots[idx(t)]; @@ -192,15 +192,16 @@ class LockfreeMPMCRingQueue : public LockfreeRingQueueBase { } } else { auto const prevTail = t; + auto h = head.load(std::memory_order_acquire); t = tail.load(std::memory_order_acquire); - if (t == prevTail) { + if (t == prevTail && Base::check_full(h, t)) { return false; } } } } - bool pop_weak(T& x) { + bool pop(T& x) { auto h = head.load(std::memory_order_acquire); for (;;) { auto& slot = slots[idx(h)]; @@ -213,28 +214,15 @@ class LockfreeMPMCRingQueue : public LockfreeRingQueueBase { } } else { auto const prevHead = h; + auto t = tail.load(std::memory_order_acquire); h = head.load(std::memory_order_acquire); - if (h == prevHead) { + if (h == prevHead && Base::check_empty(h, t)) { return false; } } } } - bool push(const T& x) { - do { - if (push_weak(x)) return true; - } while (!full()); - return false; - } - - bool pop(T& x) { - do { - if (pop_weak(x)) return true; - } while (!empty()); - return false; - } - template void send(const T& x) { static_assert(std::is_base_of::value, @@ -536,8 +524,8 @@ namespace common { * and load balancing. * Watch out that `recv` should run in photon environment (because it has to) * use photon semaphore to be notified that new item has sended. `send` could - * running in photon or std::thread environment (needs to set template `Pause` as - * `ThreadPause`). + * running in photon or std::thread environment (needs to set template `Pause` + * as `ThreadPause`). * * @tparam QueueType shoulde be one of LockfreeMPMCRingQueue, * LockfreeBatchMPMCRingQueue, or LockfreeSPSCRingQueue, with their own template diff --git a/fs/localfs.cpp b/fs/localfs.cpp index e133bfadb..0ddaf280d 100644 --- a/fs/localfs.cpp +++ b/fs/localfs.cpp @@ -316,8 +316,9 @@ namespace fs virtual dirent* get() override { if (direntp) { - memcpy(&m_dirent, direntp, sizeof(m_dirent)); - return &m_dirent; + memcpy(&m_dirent, direntp, + std::min(sizeof(m_dirent), (size_t)direntp->d_reclen)); + return &m_dirent; } return direntp; } diff --git a/fs/path.cpp b/fs/path.cpp index 7157061b0..196df6c82 100644 --- a/fs/path.cpp +++ b/fs/path.cpp @@ -252,12 +252,16 @@ namespace fs if (path.empty() || path.back() != '/') { path_push_back("/"); } + int ret = next(); + if (ret < 0) + m_path_len = 0; } int Walker::enter_dir() { auto dir = m_filesystem->opendir(m_path_buffer); if (!dir) - LOG_ERRNO_RETURN(0, -1, "failed to opendir(`)", m_path); + LOG_ERRNO_RETURN(0, -1, "failed to opendir(`)", + std::string_view(m_path_buffer, m_path_len)); m_stack.emplace(dir); return 0; } @@ -270,36 +274,36 @@ namespace fs struct stat st; auto ret = m_filesystem->lstat(m_path_buffer, &st); if (ret < 0) - LOG_ERRNO_RETURN(0, -1, "failed to lstat '`'", m_path); + LOG_ERRNO_RETURN(0, -1, "failed to lstat '`'", + std::string_view(m_path_buffer, m_path_len)); return S_ISDIR(st.st_mode); } return 0; } - void Walker::path_push_back(string_view s) - { - auto len0 = m_path.length(); + void Walker::path_push_back(string_view s) { + auto len0 = m_path_len; auto len1 = s.length(); assert(len0 + len1 < sizeof(m_path_buffer) - 1); memcpy(m_path_buffer + len0, s.data(), len1 + 1); - m_path = string_view(m_path_buffer, len0 + len1); + m_path_len = len0 + len1; } - void Walker::path_pop_back(size_t len1) - { - auto len0 = m_path.length(); + void Walker::path_pop_back(size_t len1) { + auto len0 = m_path_len; assert(len0 > len1); len0 -= len1; m_path_buffer[len0] = '\0'; - m_path = string_view(m_path_buffer, len0); + m_path_len = len0; } int Walker::next() { again: - if (m_path.empty()) return -1; - if (m_path.back() != '/') + if (m_path_len == 0) return -1; + std::string_view path(m_path_buffer, m_path_len); + if (path.back() != '/') { - auto m = m_path.rfind('/'); - if (m != m_path.npos) { - auto len0 = m_path.length(); + auto m = path.rfind('/'); + if (m != path.npos) { + auto len0 = path.length(); path_pop_back(len0 - m - 1); m_stack.top()->next(); } @@ -318,10 +322,10 @@ namespace fs m_stack.pop(); if (m_stack.empty()) { - m_path.remove_prefix(m_path.length()); + m_path_len = 0; return -1; // finished walking } - assert(m_path.back() == '/'); + assert(m_path_buffer[m_path_len - 1] == '/'); path_pop_back(1); goto again; } diff --git a/fs/path.h b/fs/path.h index fe5514e20..be01c4921 100644 --- a/fs/path.h +++ b/fs/path.h @@ -168,12 +168,13 @@ namespace fs { public: Walker(IFileSystem* fs, string_view path); - string_view path() { return m_path; } + string_view path() { return {m_path_buffer, m_path_len}; } string_view get() { return path(); } + bool valid() { return m_path_len; } int next(); protected: - string_view m_path; + size_t m_path_len = 0; IFileSystem* m_filesystem; std::stack> m_stack; char m_path_buffer[PATH_MAX]; diff --git a/fs/test/test.cpp b/fs/test/test.cpp index 8e7ab6ead..62b61098d 100644 --- a/fs/test/test.cpp +++ b/fs/test/test.cpp @@ -1357,6 +1357,7 @@ TEST(Walker, basic) { std::system(std::string("touch " + root + file2).c_str()); int count = 0; for (auto file : enumerable(Walker(srcFs, "/"))) { + LOG_INFO(VALUE(file.data())); if (file.back() == '2') { EXPECT_EQ(0, strcmp(file.data(), file2.c_str())); } else { diff --git a/net/http/client.cpp b/net/http/client.cpp index 475b579c7..79964d79f 100644 --- a/net/http/client.cpp +++ b/net/http/client.cpp @@ -224,9 +224,10 @@ class ClientImpl : public Client { LOG_ERROR_RETURN(EINVAL, ROUNDTRIP_FAILED, "Content-Length and Transfer-Encoding conflicted"); } - op->req.headers.insert("User-Agent", USERAGENT); - op->req.headers.insert("Connection", "keep-alive"); op->req.headers.merge(m_common_headers); + op->req.headers.insert("User-Agent", m_user_agent.empty() ? std::string_view(USERAGENT) + : std::string_view(m_user_agent)); + op->req.headers.insert("Connection", "keep-alive"); if (m_cookie_jar && m_cookie_jar->set_cookies_to_headers(&op->req) != 0) LOG_ERROR_RETURN(0, -1, "set_cookies_to_headers failed"); Timeout tmo(std::min(op->timeout.timeout(), m_timeout)); diff --git a/net/http/client.h b/net/http/client.h index 350766e30..e2b52ab69 100644 --- a/net/http/client.h +++ b/net/http/client.h @@ -109,6 +109,9 @@ class Client : public Object { m_proxy_url.from_string(proxy); m_proxy = true; } + void set_user_agent(std::string_view user_agent) { + m_user_agent = std::string(user_agent); + } StoredURL* get_proxy() { return &m_proxy_url; } @@ -129,6 +132,7 @@ class Client : public Object { bool secure = false, uint64_t timeout = -1UL) = 0; protected: StoredURL m_proxy_url; + std::string m_user_agent; uint64_t m_timeout = -1UL; bool m_proxy = false; }; diff --git a/net/http/test/client_function_test.cpp b/net/http/test/client_function_test.cpp index 1e81851a6..2c9f5741f 100644 --- a/net/http/test/client_function_test.cpp +++ b/net/http/test/client_function_test.cpp @@ -512,6 +512,46 @@ TEST(http_client, partial_body) { EXPECT_EQ(true, buf == "http_clien"); } +int ua_check_handler(void*, Request &req, Response &resp, std::string_view) { + auto ua = req.headers["User-Agent"]; + LOG_DEBUG(VALUE(ua)); + EXPECT_EQ(ua, "TEST_UA"); + resp.set_result(200); + std::string str = "success"; + resp.headers.content_length(7); + resp.write((void*)str.data(), str.size()); + return 0; +} + + +TEST(http_client, user_agent) { + auto tcpserver = new_tcp_socket_server(); + DEFER(delete tcpserver); + tcpserver->bind(18731); + tcpserver->listen(); + auto server = new_http_server(); + DEFER(delete server); + server->add_handler({nullptr, &ua_check_handler}); + tcpserver->set_handler(server->get_connection_handler()); + tcpserver->start_loop(); + + std::string target_get = "http://localhost:18731/file"; + auto client = new_http_client(); + client->set_user_agent("TEST_UA"); + DEFER(delete client); + auto op = client->new_operation(Verb::GET, target_get); + DEFER(delete op); + op->req.headers.content_length(0); + client->call(op); + EXPECT_EQ(op->status_code, 200); + std::string buf; + buf.resize(op->resp.headers.content_length()); + op->resp.read((void*)buf.data(), op->resp.headers.content_length()); + LOG_DEBUG(VALUE(buf)); + EXPECT_EQ(true, buf == "success"); +} + + TEST(url, url_escape_unescape) { EXPECT_EQ( url_escape("?a=x:b&b=cd&c= feg&d=2/1[+]@alibaba.com&e='!bad';"), diff --git a/net/pooled_socket.cpp b/net/pooled_socket.cpp index 667e40fdb..acd9b3161 100644 --- a/net/pooled_socket.cpp +++ b/net/pooled_socket.cpp @@ -158,7 +158,6 @@ class TCPSocketPool : public ForwardSocketClient { list.erase(node); if (list.empty()) fdmap.erase(it); rm_watch(node); - delete node; } public: @@ -246,6 +245,7 @@ class TCPSocketPool : public ForwardSocketClient { // socket shutdown drop_from_pool(nodes[i]); } + for (int i = 0; i < ret; i++) delete nodes[i]; } } }; diff --git a/rpc/rpc.cpp b/rpc/rpc.cpp index 18ac63aac..3fbc77cb8 100644 --- a/rpc/rpc.cpp +++ b/rpc/rpc.cpp @@ -236,6 +236,8 @@ namespace rpc { COPY(func); COPY(stream); COPY(sk); + COPY(stream_serv_count); + COPY(stream_cv); COPY(w_lock); #undef COPY } @@ -378,14 +380,14 @@ namespace rpc { } static void* async_serve(void* args_) { - auto ctx = (Context*)args_; - Context context(std::move(*ctx)); - ctx->got_it = true; + bool &got_it = ((Context*)args_)->got_it; + Context context(std::move(*(Context*)args_)); + got_it = true; thread_yield_to(nullptr); context.serve_request(); // serve done, here reduce refcount - (*ctx->stream_serv_count) --; - ctx->stream_cv->notify_all(); + (*context.stream_serv_count) --; + context.stream_cv->notify_all(); return nullptr; } virtual int shutdown_no_wait() override { diff --git a/thread/thread.cpp b/thread/thread.cpp index 473feed47..fa634b9cd 100644 --- a/thread/thread.cpp +++ b/thread/thread.cpp @@ -1044,13 +1044,17 @@ R"( if (likely(__mimic_vdso_time_x86)) return photon::now = __mimic_vdso_time_x86.get_now(); #endif - struct timeval tv; - gettimeofday(&tv, NULL); - uint64_t nnow = tv.tv_sec; - nnow *= 1000 * 1000; - nnow += tv.tv_usec; + struct timespec tv; +#ifdef CLOCK_BOOTTIME + clock_gettime(CLOCK_BOOTTIME, &tv); +#else + clock_gettime(CLOCK_MONOTONIC, &tv); +#endif + int usec = tv.tv_nsec / 1000ul; + uint64_t nnow = tv.tv_sec * 1000ul * 1000ul + usec; + assert(tv.tv_sec <= UINT32_MAX && usec < 1000000); now = nnow; - return tv; + return {tv.tv_sec, usec}; } __attribute__((always_inline)) static inline uint32_t _rdtsc()