fix: switch to a RWMutex for synchronizing the router rebuild

Global locks in ServeHTTP methods were preventing concurrent request processing. Individual handlers should manage their own synchronization.

Signed-off-by: Dorin Geman <dorin.geman@docker.com>

Author: doringeman

pkg/inference/models/manager.go

@@ -41,7 +41,7 @@ type Manager struct {
 	// registryClient is the client for model registry.
 	registryClient *registry.Client
 	// lock is used to synchronize access to the models manager's router.
-	lock sync.Mutex
+	lock sync.RWMutex
 }
 
 type ClientConfig struct {
@@ -542,8 +542,8 @@ func (m *Manager) GetDiskUsage() (int64, error, int) {
 
 // ServeHTTP implement net/http.Handler.ServeHTTP.
 func (m *Manager) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	m.lock.Lock()
-	defer m.lock.Unlock()
+	m.lock.RLock()
+	defer m.lock.RUnlock()
 	m.router.ServeHTTP(w, r)
 }
 

pkg/inference/scheduling/scheduler.go

@@ -43,7 +43,7 @@ type Scheduler struct {
 	// openAIRecorder is used to record OpenAI API inference requests and responses.
 	openAIRecorder *metrics.OpenAIRecorder
 	// lock is used to synchronize access to the scheduler's router.
-	lock sync.Mutex
+	lock sync.RWMutex
 }
 
 // NewScheduler creates a new inference scheduler.
@@ -510,7 +510,7 @@ func parseBackendMode(mode string) inference.BackendMode {
 
 // ServeHTTP implements net/http.Handler.ServeHTTP.
 func (s *Scheduler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	s.lock.Lock()
-	defer s.lock.Unlock()
+	s.lock.RLock()
+	defer s.lock.RUnlock()
 	s.router.ServeHTTP(w, r)
 }