[AutoPGLE] Prevent an AutoPGLE to run if user launched an external profiler.

PiperOrigin-RevId: 739109278

Author: Google-ML-Automation

third_party/tsl/tsl/profiler/lib/profiler_session.cc

@@ -121,7 +121,13 @@ ProfilerSession::ProfilerSession(const ProfileOptions& options)
   DCHECK(profiler_lock_.Active());
   profilers_ = std::make_unique<tsl::profiler::ProfilerCollection>(
       profiler::CreateProfilers(options_));
-  profilers_->Start().IgnoreError();
+
+  absl::Status status = profilers_->Start();
+  if (options_.ignore_start_error()) {
+    status.IgnoreError();
+  } else {
+    status_ = status;
+  }
 #endif
 }
 

third_party/tsl/tsl/profiler/protobuf/profiler_options.proto

@@ -2,7 +2,7 @@ syntax = "proto3";
 
 package tensorflow;
 
-// Next ID: 13
+// Next ID: 14
 message ProfileOptions {
   // Some default value of option are not proto3 default value. Use this version
   // to determine if we should use default option value instead of proto3
@@ -94,6 +94,8 @@ message ProfileOptions {
   //   }
   // }
   map<string, AdvancedConfigValue> advanced_configuration = 12;
+
+  bool ignore_start_error = 13;
 }
 
 // Options for remote profiler session manager.

xla/backends/gpu/codegen/triton/kernel_name_tracer_cuda.cc

@@ -54,7 +54,7 @@ void KernelNameTracerCuda::start() {
       collector_options, start_walltime_ns, start_gputime_ns);
   profiler::CuptiTracerOptions options;
   options.activities_selected = {CUPTI_ACTIVITY_KIND_KERNEL};
-  cupti_tracer_->Enable(options, cupti_collector_.get());
+  cupti_tracer_->Enable(options, cupti_collector_.get()).IgnoreError();
 }
 
 std::vector<std::string> KernelNameTracerCuda::stop() {

xla/backends/profiler/gpu/cupti_error_manager_test.cc

@@ -80,7 +80,7 @@ class CuptiErrorManagerTest : public ::testing::Test {
   }
 
   void EnableProfiling(const CuptiTracerOptions& option) {
-    cupti_tracer_->Enable(option, cupti_collector_.get());
+    cupti_tracer_->Enable(option, cupti_collector_.get()).IgnoreError();
   }
 
   void DisableProfiling() { cupti_tracer_->Disable(); }

xla/backends/profiler/gpu/cupti_tracer.cc

@@ -1035,8 +1035,8 @@ int CuptiTracer::NumGpus() {
   return num_gpus;
 }
 
-void CuptiTracer::Enable(const CuptiTracerOptions &option,
-                         CuptiTraceCollector *collector) {
+absl::Status CuptiTracer::Enable(const CuptiTracerOptions &option,
+                                 CuptiTraceCollector *collector) {
   option_ = option;
   collector_ = collector;
 
@@ -1058,10 +1058,13 @@ void CuptiTracer::Enable(const CuptiTracerOptions &option,
 
   absl::Status status = EnableApiTracing();
   need_root_access_ |= status.code() == tsl::error::PERMISSION_DENIED;
-  if (!status.ok()) return;
+  if (!status.ok()) {
+    return status;
+  }
 
   EnableActivityTracing().IgnoreError();
   tsl::profiler::AnnotationStack::Enable(true);
+  return status;
 }
 
 void CuptiTracer::Disable() {

xla/backends/profiler/gpu/cupti_tracer.h

@@ -81,7 +81,8 @@ class CuptiTracer {
   bool IsAvailable() const;
   bool NeedRootAccess() const { return need_root_access_; }
 
-  void Enable(const CuptiTracerOptions& option, CuptiTraceCollector* collector);
+  absl::Status Enable(const CuptiTracerOptions& option,
+                      CuptiTraceCollector* collector);
   void Disable();
 
   // Control threads could periodically call this function to flush the

xla/backends/profiler/gpu/device_tracer_cuda.cc

@@ -152,7 +152,7 @@ absl::Status GpuTracer::DoStart() {
   cupti_collector_ = CreateCuptiCollector(collector_options, start_walltime_ns,
                                           start_gputime_ns);
 
-  cupti_tracer_->Enable(options_, cupti_collector_.get());
+  cupti_tracer_->Enable(options_, cupti_collector_.get()).IgnoreError();
   return absl::OkStatus();
 }
 

xla/service/gpu/model/hlo_op_profiler.cc

@@ -68,7 +68,7 @@ class CuptiKernelTracer : public HloOpProfiler::KernelTracer,
         // Not interested in API callbacks, but empty list enables them all.
         CUPTI_DRIVER_TRACE_CBID_cu64GLMapBufferObject);
     options.activities_selected.push_back(CUPTI_ACTIVITY_KIND_KERNEL);
-    cupti_tracer_->Enable(options, this);
+    cupti_tracer_->Enable(options, this).IgnoreError();
   }
 
   uint64_t getMedianKernelTimeNs() && override {

xla/tsl/profiler/utils/session_manager.cc

@@ -108,6 +108,10 @@ RemoteProfilerSessionManagerOptions GetRemoteSessionManagerOptionsLocked(
       int value = std::get<int>(kw.second);
       options.set_delay_ms(value);
       VLOG(1) << "delay_ms was set to " << value;
+    } else if (key == "ignore_start_error") {
+      int value = std::get<int>(kw.second);
+      options.mutable_profiler_options()->set_ignore_start_error(value);
+      VLOG(1) << "ignore_start_error was set to " << value;
     } else {
       LOG(WARNING) << "Unrecognised key: " << key;
     }