8371944: AOT configuration is corrupted when app closes System.out

Reviewed-by: phh, iklam
Backport-of: 1535d08

Author: shipilev

src/hotspot/share/cds/dynamicArchive.cpp

@@ -348,6 +348,7 @@ void DynamicArchiveBuilder::write_archive(char* serialized_data, AOTClassLocatio
   assert(dynamic_info != nullptr, "Sanity");
 
   dynamic_info->open_as_output();
+  dynamic_info->prepare_for_writing();
   ArchiveHeapInfo no_heap_for_dynamic_dump;
   ArchiveBuilder::write_archive(dynamic_info, &no_heap_for_dynamic_dump);
 

src/hotspot/share/cds/filemap.cpp

@@ -766,7 +766,9 @@ void FileMapInfo::open_as_output() {
   }
   _fd = fd;
   _file_open = true;
+}
 
+void FileMapInfo::prepare_for_writing() {
   // Seek past the header. We will write the header after all regions are written
   // and their CRCs computed.
   size_t header_bytes = header()->header_size();

src/hotspot/share/cds/filemap.hpp

@@ -359,6 +359,7 @@ class FileMapInfo : public CHeapObj<mtInternal> {
   // File manipulation.
   bool  open_as_input() NOT_CDS_RETURN_(false);
   void  open_as_output();
+  void  prepare_for_writing();
   void  write_header();
   void  write_region(int region, char* base, size_t size,
                      bool read_only, bool allow_exec);

src/hotspot/share/cds/metaspaceShared.cpp

@@ -113,6 +113,7 @@ intx MetaspaceShared::_relocation_delta;
 char* MetaspaceShared::_requested_base_address;
 Array<Method*>* MetaspaceShared::_archived_method_handle_intrinsics = nullptr;
 bool MetaspaceShared::_use_optimized_module_handling = true;
+FileMapInfo* MetaspaceShared::_output_mapinfo = nullptr;
 
 // The CDS archive is divided into the following regions:
 //     rw  - read-write metadata
@@ -321,6 +322,24 @@ void MetaspaceShared::initialize_for_static_dump() {
     MetaspaceShared::unrecoverable_writing_error();
   }
   _symbol_region.init(&_symbol_rs, &_symbol_vs);
+  if (CDSConfig::is_dumping_preimage_static_archive()) {
+    // We are in the AOT training run. User code is executed.
+    //
+    // On Windows, if the user code closes System.out and we open the AOT config file for output
+    // only at VM exit, we might get back the same file HANDLE as stdout, and the AOT config
+    // file may get corrupted by UL logs. By opening early, we ensure that the output
+    // HANDLE is different than stdout so we can avoid such corruption.
+    open_output_mapinfo();
+  } else {
+    // No need for the above as we won't execute any user code.
+  }
+}
+
+void MetaspaceShared::open_output_mapinfo() {
+  const char* static_archive = CDSConfig::output_archive_path();
+  assert(static_archive != nullptr, "sanity");
+  _output_mapinfo = new FileMapInfo(static_archive, true);
+  _output_mapinfo->open_as_output();
 }
 
 // Called by universe_post_init()
@@ -551,14 +570,13 @@ class VM_PopulateDumpSharedSpace : public VM_Operation {
 
 public:
 
-  VM_PopulateDumpSharedSpace(StaticArchiveBuilder& b) :
-    VM_Operation(), _heap_info(), _map_info(nullptr), _builder(b) {}
+  VM_PopulateDumpSharedSpace(StaticArchiveBuilder& b, FileMapInfo* map_info) :
+    VM_Operation(), _heap_info(), _map_info(map_info), _builder(b) {}
 
   bool skip_operation() const { return false; }
 
   VMOp_Type type() const { return VMOp_PopulateDumpSharedSpace; }
   ArchiveHeapInfo* heap_info()  { return &_heap_info; }
-  FileMapInfo* map_info() const { return _map_info; }
   void doit();   // outline because gdb sucks
   bool allow_nested_vm_operations() const { return true; }
 }; // class VM_PopulateDumpSharedSpace
@@ -688,12 +706,6 @@ void VM_PopulateDumpSharedSpace::doit() {
   CppVtables::zero_archived_vtables();
 
   // Write the archive file
-  if (CDSConfig::is_dumping_final_static_archive()) {
-    FileMapInfo::free_current_info(); // FIXME: should not free current info
-  }
-  const char* static_archive = CDSConfig::output_archive_path();
-  assert(static_archive != nullptr, "sanity");
-  _map_info = new FileMapInfo(static_archive, true);
   _map_info->populate_header(MetaspaceShared::core_region_alignment());
   _map_info->set_early_serialized_data(early_serialized_data);
   _map_info->set_serialized_data(serialized_data);
@@ -1012,7 +1024,14 @@ void MetaspaceShared::preload_and_dump_impl(StaticArchiveBuilder& builder, TRAPS
   }
 #endif
 
-  VM_PopulateDumpSharedSpace op(builder);
+  if (!CDSConfig::is_dumping_preimage_static_archive()) {
+    if (CDSConfig::is_dumping_final_static_archive()) {
+      FileMapInfo::free_current_info(); // FIXME: should not free current info
+    }
+    open_output_mapinfo();
+  }
+
+  VM_PopulateDumpSharedSpace op(builder, _output_mapinfo);
   VMThread::execute(&op);
 
   if (AOTCodeCache::is_on_for_dump() && CDSConfig::is_dumping_final_static_archive()) {
@@ -1026,7 +1045,9 @@ void MetaspaceShared::preload_and_dump_impl(StaticArchiveBuilder& builder, TRAPS
     CDSConfig::disable_dumping_aot_code();
   }
 
-  bool status = write_static_archive(&builder, op.map_info(), op.heap_info());
+  bool status = write_static_archive(&builder, _output_mapinfo, op.heap_info());
+  assert(!_output_mapinfo->is_open(), "Must be closed already");
+  _output_mapinfo = nullptr;
   if (status && CDSConfig::is_dumping_preimage_static_archive()) {
     tty->print_cr("%s AOTConfiguration recorded: %s",
                   CDSConfig::has_temp_aot_config_file() ? "Temporary" : "", AOTConfiguration);
@@ -1044,11 +1065,10 @@ bool MetaspaceShared::write_static_archive(ArchiveBuilder* builder, FileMapInfo*
   // relocate the data so that it can be mapped to MetaspaceShared::requested_base_address()
   // without runtime relocation.
   builder->relocate_to_requested();
-
-  map_info->open_as_output();
   if (!map_info->is_open()) {
     return false;
   }
+  map_info->prepare_for_writing();
   builder->write_archive(map_info, heap_info);
 
   if (AllowArchivingWithJavaAgent) {

src/hotspot/share/cds/metaspaceShared.hpp

@@ -59,6 +59,7 @@ class MetaspaceShared : AllStatic {
   static char* _requested_base_address;
   static bool _use_optimized_module_handling;
   static Array<Method*>* _archived_method_handle_intrinsics;
+  static FileMapInfo* _output_mapinfo;
 
  public:
   enum {
@@ -180,6 +181,7 @@ class MetaspaceShared : AllStatic {
 private:
   static void read_extra_data(JavaThread* current, const char* filename) NOT_CDS_RETURN;
   static void fork_and_dump_final_static_archive(TRAPS);
+  static void open_output_mapinfo();
   static bool write_static_archive(ArchiveBuilder* builder, FileMapInfo* map_info, ArchiveHeapInfo* heap_info);
   static FileMapInfo* open_static_archive();
   static FileMapInfo* open_dynamic_archive();

test/hotspot/jtreg/runtime/cds/appcds/aotCache/CloseSystemOut.java

@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+/*
+ * @test
+ * @summary AOT configuration should not be corrupted even if the app closes System.out in the training run
+ * @bug 8371944
+ * @library /test/jdk/lib/testlibrary /test/lib
+ * @build CloseSystemOut
+ * @run driver jdk.test.lib.helpers.ClassFileInstaller -jar app.jar CloseSystemOutApp
+ * @run driver CloseSystemOut
+ */
+
+import java.io.PrintWriter;
+import jdk.test.lib.cds.CDSAppTester;
+import jdk.test.lib.helpers.ClassFileInstaller;
+import jdk.test.lib.process.OutputAnalyzer;
+
+public class CloseSystemOut {
+    static final String appJar = ClassFileInstaller.getJarPath("app.jar");
+    static final String mainClass = "CloseSystemOutApp";
+
+    public static void main(String[] args) throws Exception {
+        Tester tester = new Tester();
+        tester.run(new String[] {"AOT", "--two-step-training"} );
+    }
+
+    static class Tester extends CDSAppTester {
+        public Tester() {
+            super(mainClass);
+        }
+
+        @Override
+        public String classpath(RunMode runMode) {
+            return appJar;
+        }
+
+        @Override
+        public String[] appCommandLine(RunMode runMode) {
+            return new String[] {mainClass};
+        }
+
+        @Override
+        public void checkExecution(OutputAnalyzer out, RunMode runMode) {
+            if (runMode != RunMode.ASSEMBLY) {
+                out.shouldContain("Hello Confused World");
+            }
+        }
+    }
+}
+
+class CloseSystemOutApp {
+    public static void main(String args[]) {
+        // Naive code that ends up closing System.out/err when we
+        // leave the "try" block
+        try (var err = new PrintWriter(System.err);
+             var out = new PrintWriter(System.out)) {
+            out.println("Hello Confused World");
+        }
+    }
+}