poc fix

Author: stuqdog

src/viam/sdk/robot/client.cpp

@@ -2,6 +2,7 @@
 
 #include <chrono>
 #include <cstddef>
+#include <cstdlib>
 #include <memory>
 #include <mutex>
 #include <string>
@@ -144,11 +145,16 @@ RobotClient::~RobotClient() {
 
 void RobotClient::close() {
     should_refresh_.store(false);
+    should_check_connection_ = false;
 
     if (refresh_thread_.joinable()) {
         refresh_thread_.join();
     }
 
+    if (check_connection_thread_.joinable()) {
+        check_connection_thread_.join();
+    }
+
     stop_all();
 
     viam_channel_.close();
@@ -231,6 +237,53 @@ void RobotClient::refresh_every() {
     }
 };
 
+void RobotClient::check_connection() {
+    std::exception connection_error;
+    bool connected(true);
+    while (should_check_connection_) {
+        for (int i = 0; i < 3; ++i) {
+            try {
+                std::this_thread::sleep_for(std::chrono::seconds{10});
+                impl::client_helper(impl_, &RobotService::Stub::ResourceNames).invoke([](auto&) {
+                    return;
+                });
+                connected = true;
+                break;
+            } catch (const std::exception& e) {
+                connected = false;
+                connection_error = e;
+                std::this_thread::sleep_for(std::chrono::milliseconds{100});
+            }
+        }
+        if (connected) {
+            continue;
+        }
+        const auto* uri = viam_channel_.uri_;
+        VIAM_SDK_LOG(error)
+            << "Lost connection to machine. Attempting to reconnect to every second";
+        viam_channel_.close();
+
+        for (int i = 0; i < 3; ++i) {
+            try {
+                auto channel = ViamChannel::dial(uri, {});
+                auto impl =
+                    std::make_unique<RobotClient::impl>(RobotService::NewStub(channel.channel()));
+                impl_.reset();
+                impl_.swap(impl);
+                refresh();
+                connected = true;
+            } catch (const std::exception& e) {
+                viam_channel_.close();
+                std::this_thread::sleep_for(std::chrono::seconds{1});
+            }
+        }
+        if (!connected) {
+            // NOLINTNEXTLINE
+            exit(0);
+        }
+    }
+}
+
 RobotClient::RobotClient(ViamChannel channel)
     : viam_channel_(std::move(channel)),
       impl_(std::make_unique<impl>(RobotService::NewStub(viam_channel_.channel()))) {}
@@ -262,8 +315,8 @@ void RobotClient::log(const std::string& name,
     ClientContext ctx;
     const auto response = impl_->stub->Log(ctx, req, &resp);
     if (is_error_response(response)) {
-        // Manually override to force this to get logged to console so we don't set off an infinite
-        // loop
+        // Manually override to force this to get logged to console so we don't set off an
+        // infinite loop
         VIAM_SDK_LOG(error) << boost::log::add_value(sdk::impl::attr_console_force_type{}, true)
                             << "Error sending log message over grpc: " << response.error_message()
                             << response.error_details();
@@ -279,6 +332,10 @@ std::shared_ptr<RobotClient> RobotClient::with_channel(ViamChannel channel,
         robot->refresh_thread_ = std::thread{&RobotClient::refresh_every, robot.get()};
     }
 
+    robot->should_check_connection_ = true;
+
+    robot->check_connection_thread_ = std::thread{&RobotClient::check_connection, robot.get()};
+
     robot->refresh();
     return robot;
 };

src/viam/sdk/robot/client.hpp

@@ -184,9 +184,14 @@ class RobotClient {
     void connect_logging();
 
     void refresh_every();
+    void check_connection();
 
     std::thread refresh_thread_;
+    // CR erodkin: lots of checks in this thread, it takes 10 seconds to run, shutdown is now slow
+    // on client code
+    std::thread check_connection_thread_;
     std::atomic<bool> should_refresh_;
+    bool should_check_connection_;
     std::chrono::seconds refresh_interval_;
 
     ViamChannel viam_channel_;

src/viam/sdk/rpc/dial.cpp

@@ -187,9 +187,12 @@ ViamChannel ViamChannel::dial(const char* uri, const boost::optional<DialOptions
     }
     address += proxy_path;
 
-    return ViamChannel(sdk::impl::create_viam_channel(address, grpc::InsecureChannelCredentials()),
-                       proxy_path,
-                       ptr);
+    auto chan =
+        ViamChannel(sdk::impl::create_viam_channel(address, grpc::InsecureChannelCredentials()),
+                    proxy_path,
+                    ptr);
+    chan.uri_ = uri;
+    return chan;
 }
 
 const std::shared_ptr<grpc::Channel>& ViamChannel::channel() const {

src/viam/sdk/rpc/dial.hpp

@@ -45,6 +45,9 @@ class ViamChannel {
 
     void close();
 
+    // NOLINTNEXTLINE
+    const char* uri_;
+
    private:
     struct impl;