fix: Enforce UTF-8 for AppMap I/O and add encoding regression tests

This commit ensures consistent UTF-8 character encoding for AppMap data
across file system operations and HTTP responses, preventing corruption
on systems where the default charset is not UTF-8 (e.g., Windows-1252).

Key changes:
- `RecordingSession.java`: Enforce `StandardCharsets.UTF_8` when creating
  writers for AppMap files, replacing reliance on default `FileWriter`.
- `Recording.java`: Explicitly use `StandardCharsets.UTF_8` in `readFully`
  to correctly decode AppMap content during retrieval.
- `ServletRequest.java`: Set `Content-Type: application/json; charset=UTF-8`
  for remote recording responses and calculate `Content-Length` based on
  UTF-8 byte size rather than string length.
- `agent/test/encoding/`: Add a comprehensive regression test suite
  (`encoding.bats`, `UnicodeTest.java`, `ReadFullyTest.java`) to verify
  encoding handling for both reading and writing operations under
  non-UTF-8 environment settings.

Author: dividedmind

agent/src/main/java/com/appland/appmap/process/hooks/remoterecording/ServletRequest.java

@@ -2,6 +2,7 @@
 
 import java.io.IOException;
 import java.io.PrintWriter;
+import java.nio.charset.StandardCharsets;
 
 import com.appland.appmap.record.Recording;
 import com.appland.appmap.reflect.HttpServletRequest;
@@ -29,8 +30,8 @@ public void setStatus(int status) {
   }
 
   public void writeJson(String responseJson) throws IOException {
-    res.setContentType("application/json");
-    res.setContentLength(responseJson.length());
+    res.setContentType("application/json; charset=UTF-8");
+    res.setContentLength(responseJson.getBytes(StandardCharsets.UTF_8).length);
     res.setStatus(HttpServletResponse.SC_OK);
 
     PrintWriter writer = res.getWriter();
@@ -39,7 +40,7 @@ public void writeJson(String responseJson) throws IOException {
   }
 
   public void writeRecording(Recording recording) throws IOException {
-    res.setContentType("application/json");
+    res.setContentType("application/json; charset=UTF-8");
     res.setContentLength(recording.size());
     recording.readFully(true, res.getWriter());
   }

agent/src/main/java/com/appland/appmap/record/Recording.java

@@ -5,11 +5,12 @@
 
 import java.io.File;
 import java.io.FileInputStream;
-import java.io.FileReader;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.io.Reader;
 import java.io.Writer;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -100,7 +101,7 @@ public Path moveTo(Path targetPath) {
     }
 
     public void readFully(boolean delete, Writer writer) throws IOException {
-      try (final Reader reader = new FileReader(this.file)) {
+      try (final Reader reader = new InputStreamReader(new FileInputStream(this.file), StandardCharsets.UTF_8)) {
         char[] buffer = new char[2048];
         int bytesRead;
         while ((bytesRead = reader.read(buffer)) != -1) {

agent/src/main/java/com/appland/appmap/record/RecordingSession.java

@@ -1,12 +1,7 @@
 package com.appland.appmap.record;
 
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
-import java.io.RandomAccessFile;
-import java.io.Writer;
+import java.io.*;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
@@ -98,7 +93,7 @@ public synchronized Recording checkpoint() {
         public void write(int b) throws IOException {
           raf.write(b);
         }
-      });
+      }, StandardCharsets.UTF_8);
       raf.seek(targetPath.toFile().length());
 
       if (  eventReceived ) {
@@ -162,7 +157,7 @@ void start() {
     try {
       this.tmpPath = Files.createTempFile(null, ".appmap.json");
       this.tmpPath.toFile().deleteOnExit();
-      this.serializer = AppMapSerializer.open(new FileWriter(this.tmpPath.toFile()));
+      this.serializer = AppMapSerializer.open(new OutputStreamWriter(new FileOutputStream(this.tmpPath.toFile()), StandardCharsets.UTF_8));
     } catch (IOException e) {
       this.tmpPath = null;
       this.serializer = null;

agent/test/encoding/ReadFullyTest.java

@@ -0,0 +1,47 @@
+package test.pkg;
+
+import com.appland.appmap.config.AppMapConfig;
+import com.appland.appmap.record.Recording;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileSystems;
+
+public class ReadFullyTest {
+    public static void main(String[] args) {
+        try {
+            runTest();
+        } catch (Exception e) {
+            e.printStackTrace();
+            System.exit(1);
+        }
+    }
+
+    public static void runTest() throws IOException {
+        // Initialize AppMapConfig
+        AppMapConfig.initialize(FileSystems.getDefault());
+
+        // 1. Create a dummy AppMap file with known UTF-8 content
+        String content = "Check: \u26A0\uFE0F \u041F\u0440\u0438\u0432\u0435\u0442";
+        File tempFile = File.createTempFile("readfully", ".appmap.json");
+        tempFile.deleteOnExit();
+
+        try (Writer fw = new OutputStreamWriter(new FileOutputStream(tempFile), StandardCharsets.UTF_8)) {
+            fw.write(content);
+        }
+
+        // 2. Create a Recording object pointing to it
+        Recording recording = new Recording("test", tempFile);
+
+        // 3. Call readFully and write to stdout using UTF-8
+        // This validates that readFully correctly reads the UTF-8 file bytes into characters
+        // regardless of the system's default encoding (which we will set to something else in BATS).
+        Writer stdoutWriter = new OutputStreamWriter(System.out, StandardCharsets.UTF_8);
+        recording.readFully(false, stdoutWriter);
+        stdoutWriter.flush();
+    }
+}

agent/test/encoding/UnicodeTest.java

@@ -0,0 +1,42 @@
+package test.pkg;
+
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+
+public class UnicodeTest {
+    public static String echo(String input) {
+        return input;
+    }
+
+    public static byte[] echoBytes(byte[] input) {
+        return input.clone();
+    }
+
+    public static void main(String[] args) {
+        try {
+            runTest();
+        } catch (IOException e) {
+            e.printStackTrace();
+            // exit 1
+            System.exit(1);
+        }
+    }
+
+    public static void runTest() throws IOException {
+        byte[] allBytes = Files.readAllBytes(Paths.get("encoding_test.cp1252"));
+
+        String allString = new String(allBytes, "Cp1252");
+        String echoedString = echo(allString);
+
+        // print out the echoed string
+        System.out.println(echoedString);
+
+        byte[] echoedBytes = echoBytes(allBytes);
+        // print out the echoed bytes as hex
+        for (byte b : echoedBytes) {
+            System.out.printf("%02X ", b);
+        }
+    }
+}

agent/test/encoding/appmap.yml

@@ -0,0 +1,3 @@
+name: encoding
+packages:
+- path: test.pkg

agent/test/encoding/encoding.bats

@@ -0,0 +1,64 @@
+#!/usr/bin/env bats
+# shellcheck disable=SC2164
+
+load '../helper'
+
+sep="$JAVA_PATH_SEPARATOR"
+AGENT_JAR="$(find_agent_jar)"
+java_cmd="java -cp ${BATS_TEST_DIRNAME}/build -javaagent:'${AGENT_JAR}'"
+
+setup() {
+  cd "${BATS_TEST_DIRNAME}"
+
+  mkdir -p build
+  # Compile tests. Output to test/encoding so package structure 'pkg' works.
+  # We need to compile both UnicodeTest.java and pkg/Target.java.
+  javac -d ./build UnicodeTest.java
+
+  # Compile ReadFullyTest, requiring the agent jar on the classpath
+  javac -cp "${AGENT_JAR}" -d ./build ReadFullyTest.java
+
+  rm -rf "${BATS_TEST_DIRNAME}/tmp/appmap"
+  _configure_logging
+}
+
+@test "AppMap file encoding with Windows-1252" {
+  # Run with windows-1252 encoding.
+  # We assert that the generated file is valid UTF-8 and contains the correct characters,
+  # even though the JVM default encoding is Windows-1252.
+  local cmd="${java_cmd} -Dfile.encoding=windows-1252 -Dappmap.recording.auto=true test.pkg.UnicodeTest"
+  [[ $BATS_VERBOSE_RUN == 1 ]] && echo "cmd: $cmd" >&3
+
+  eval "$cmd"
+
+  # Verify the output file exists — it should be the only AppMap file generated, with random name
+  # so glob for tmp/appmap/java/*.appmap.json
+  appmap_file=$(ls tmp/appmap/java/*.appmap.json)
+  [ -f "$appmap_file" ]
+
+  # Verify it is valid JSON
+  jq . "$appmap_file" > /dev/null
+
+  # Verify it is valid UTF-8
+  iconv -f UTF-8 -t UTF-8 "$appmap_file" > /dev/null
+
+  # Verify it contains the expected Unicode characters
+  grep -q "Euro: €, Accent: é, Quote: „" "$appmap_file"
+}
+
+@test "Recording.readFully works with Windows-1252 default encoding" {
+  # Run ReadFullyTest with windows-1252 default encoding.
+  # We also need to add the agent jar to the classpath so it can find the Recording class.
+  local cmd="java -cp ${BATS_TEST_DIRNAME}/build${sep}${AGENT_JAR} -Dfile.encoding=windows-1252 test.pkg.ReadFullyTest"
+  [[ $BATS_VERBOSE_RUN == 1 ]] && echo "cmd: $cmd" >&3
+
+  run eval "$cmd"
+
+  [ "$status" -eq 0 ]
+  [[ "$output" == *"Check: ⚠️ Привет"* ]]
+}
+
+teardown() {
+  rm -rf tmp
+  rm -rf build
+}

agent/test/encoding/encoding_test.cp1252

@@ -0,0 +1,5 @@
+Euro: �, Accent: �, Quote: �
+---SEPARATOR---
+:��7��ǞI�
+---BINARY---
+ޭ��