Serialize the model instance registration in resource manager (#194)

* Serialize the model instance registration in resource manager

* Fix the comment

* Update copyright

Author: tanmayv25

src/rate_limiter.cc

@@ -72,8 +72,17 @@ RateLimiter::RegisterModelInstance(
     model_context.AddSpecificRequestQueue();
 
     if (!ignore_resources_and_priority_) {
+      // As there can be mulitple models being loaded concurrently, need
+      // to hold a lock to protect the resource counts.
+      // Without this serialization instances of other models might fail
+      // to load because of the resource constraints in this instance.
+      std::lock_guard<std::mutex> lk(resource_manager_mtx_);
       resource_manager_->AddModelInstance(model_instances.back().get());
-      RETURN_IF_ERROR(resource_manager_->UpdateResourceLimits());
+      const auto& status = resource_manager_->UpdateResourceLimits();
+      if (!status.IsOk()) {
+        resource_manager_->RemoveModelInstance(model_instances.back().get());
+        return status;
+      }
     }
   }
 

src/rate_limiter.h

@@ -1,4 +1,4 @@
-// Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions
@@ -314,6 +314,7 @@ class RateLimiter {
 
   // Manager to keep track of the resource allocations
   std::unique_ptr<ResourceManager> resource_manager_;
+  std::mutex resource_manager_mtx_;
 
   // Mutex to serialize Payload [de]allocation
   std::mutex payload_mu_;