feat: Default bulk export manifest to plain JSON format

Build native JSON first in ExportResponse, avoiding lossy
Parameters-to-JSON conversion. The manifest now always includes
output and error as arrays, with native JSON attached via userData
for direct extraction by the interceptor.

Invert default in ParametersToJsonInterceptor to return plain JSON
unless client explicitly requests application/fhir+json.

Author: johngrimes

server/src/main/java/au/csiro/pathling/interceptors/ParametersToJsonInterceptor.java

@@ -61,7 +61,7 @@ public class ParametersToJsonInterceptor {
 
   private static final String APPLICATION_JSON = "application/json";
   private static final String APPLICATION_FHIR_JSON = "application/fhir+json";
-  private static final String WILDCARD = "*/*";
+  private static final String NATIVE_JSON_KEY = "nativeJson";
 
   private final ObjectMapper mapper = new ObjectMapper();
 
@@ -100,8 +100,10 @@ public boolean transformResponse(
       return true;
     }
 
-    // Transform Parameters to plain JSON and write directly to the response.
-    final String json = parametersToJson(parameters);
+    // Check for native JSON first (used by ExportResponse for lossless conversion).
+    final Object nativeJson = parameters.getUserData(NATIVE_JSON_KEY);
+    final String json = (nativeJson != null) ? (String) nativeJson : parametersToJson(parameters);
+
     response.setContentType(APPLICATION_JSON);
     response.setCharacterEncoding("UTF-8");
     response.getWriter().write(json);
@@ -111,27 +113,22 @@ public boolean transformResponse(
   }
 
   /**
-   * Determines whether the client prefers plain JSON based on the Accept header. Returns true only
-   * if the Accept header explicitly requests application/json.
+   * Determines whether the client prefers plain JSON based on the Accept header. Returns true by
+   * default, only returning false if the client explicitly requests FHIR JSON.
    *
    * @param request the HTTP servlet request
    * @return true if the client prefers plain JSON
    */
   private boolean prefersPlainJson(@Nonnull final HttpServletRequest request) {
     final String acceptHeader = request.getHeader("Accept");
 
-    // No Accept header or wildcard means continue with normal FHIR processing.
-    if (acceptHeader == null || acceptHeader.isBlank() || acceptHeader.contains(WILDCARD)) {
-      return false;
-    }
-
-    // If explicitly requesting FHIR JSON, continue with normal processing.
-    if (acceptHeader.contains(APPLICATION_FHIR_JSON)) {
+    // Only use FHIR format if explicitly requested.
+    if (acceptHeader != null && acceptHeader.contains(APPLICATION_FHIR_JSON)) {
       return false;
     }
 
-    // Only transform if explicitly requesting plain JSON.
-    return acceptHeader.contains(APPLICATION_JSON);
+    // Default to plain JSON for all other cases.
+    return true;
   }
 
   /**

server/src/main/java/au/csiro/pathling/operations/bulkexport/ExportResponse.java

@@ -20,6 +20,9 @@
 import au.csiro.pathling.OperationResponse;
 import au.csiro.pathling.library.io.sink.FileInformation;
 import au.csiro.pathling.library.io.sink.WriteDetails;
+import au.csiro.pathling.shaded.com.fasterxml.jackson.databind.ObjectMapper;
+import au.csiro.pathling.shaded.com.fasterxml.jackson.databind.node.ArrayNode;
+import au.csiro.pathling.shaded.com.fasterxml.jackson.databind.node.ObjectNode;
 import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
 import jakarta.annotation.Nonnull;
 import java.net.URISyntaxException;
@@ -41,6 +44,9 @@
  */
 public class ExportResponse implements OperationResponse<Parameters> {
 
+  private static final String NATIVE_JSON_KEY = "nativeJson";
+  private static final ObjectMapper MAPPER = new ObjectMapper();
+
   @Nonnull private final String kickOffRequestUrl;
 
   @Nonnull private final String serverBaseUrl;
@@ -72,14 +78,66 @@ public ExportResponse(
   @Nonnull
   @Override
   public Parameters toOutput() {
-    final Parameters parameters = new Parameters();
-
     // Ensure the base URL ends with a slash for proper URL construction.
     final String normalizedBaseUrl =
         serverBaseUrl.endsWith("/") ? serverBaseUrl : serverBaseUrl + "/";
 
+    // Build native JSON first.
+    final ObjectNode json = buildJson(normalizedBaseUrl);
+
+    // Convert to Parameters for FHIR compatibility.
+    final Parameters parameters = jsonToParameters(json, normalizedBaseUrl);
+
+    // Attach native JSON for direct extraction by the interceptor.
+    parameters.setUserData(NATIVE_JSON_KEY, json.toString());
+
+    return parameters;
+  }
+
+  /**
+   * Builds the native JSON representation of the export manifest.
+   *
+   * @param normalizedBaseUrl the normalised base server URL
+   * @return the JSON object
+   */
+  @Nonnull
+  private ObjectNode buildJson(@Nonnull final String normalizedBaseUrl) {
+    final ObjectNode json = MAPPER.createObjectNode();
+    json.put("transactionTime", InstantType.now().getValueAsString());
+    json.put("request", kickOffRequestUrl);
+    json.put("requiresAccessToken", requiresAccessToken);
+
+    // Output is always an array.
+    final ArrayNode outputArray = json.putArray("output");
+    for (final FileInformation fileInfo : writeDetails.fileInfos()) {
+      final ObjectNode entry = outputArray.addObject();
+      entry.put("type", fileInfo.fhirResourceType());
+      entry.put("url", buildResultUrl(normalizedBaseUrl, fileInfo.absoluteUrl()));
+    }
+
+    // Error is always an array (even when empty).
+    json.putArray("error");
+
+    return json;
+  }
+
+  /**
+   * Converts the JSON manifest to FHIR Parameters for compatibility.
+   *
+   * @param json the JSON object
+   * @param normalizedBaseUrl the normalised base server URL
+   * @return the Parameters resource
+   */
+  @Nonnull
+  private Parameters jsonToParameters(
+      @Nonnull final ObjectNode json, @Nonnull final String normalizedBaseUrl) {
+    final Parameters parameters = new Parameters();
+
     // Add transactionTime parameter.
-    parameters.addParameter().setName("transactionTime").setValue(InstantType.now());
+    parameters
+        .addParameter()
+        .setName("transactionTime")
+        .setValue(new InstantType(json.get("transactionTime").asText()));
 
     // Add request parameter.
     parameters.addParameter().setName("request").setValue(new UriType(kickOffRequestUrl));
@@ -100,6 +158,9 @@ public Parameters toOutput() {
           .setValue(new UriType(buildResultUrl(normalizedBaseUrl, fileInfo.absoluteUrl())));
     }
 
+    // Add empty error parameter to match JSON structure.
+    parameters.addParameter().setName("error");
+
     return parameters;
   }
 

server/src/test/java/au/csiro/pathling/interceptors/ParametersToJsonInterceptorTest.java

@@ -62,20 +62,22 @@ void setUp() {
   // -------------------------------------------------------------------------
 
   @Test
-  void continuesNormallyWhenNoAcceptHeader() throws Exception {
-    // When no Accept header is present, continue with normal FHIR processing.
+  void transformsToJsonWhenNoAcceptHeader() throws Exception {
+    // When no Accept header is present, default to plain JSON.
     final Parameters parameters = new Parameters();
     parameters.addParameter().setName("test").setValue(new StringType("value"));
 
+    final StringWriter writer = new StringWriter();
     final HttpServletRequest request = mockRequest(null);
-    final HttpServletResponse response = mockResponse(new StringWriter());
+    final HttpServletResponse response = mockResponse(writer);
     final RequestDetails requestDetails = mock(RequestDetails.class);
 
     final boolean result =
         interceptor.transformResponse(requestDetails, parameters, request, response);
 
-    assertThat(result).isTrue();
-    verify(response, never()).setContentType("application/json");
+    assertThat(result).isFalse();
+    verify(response).setContentType("application/json");
+    verify(response).setCharacterEncoding("UTF-8");
   }
 
   @Test
@@ -96,20 +98,22 @@ void continuesNormallyWhenAcceptIsFhirJson() throws Exception {
   }
 
   @Test
-  void continuesNormallyWhenAcceptIsWildcard() throws Exception {
-    // When Accept is */*, continue with normal FHIR processing.
+  void transformsToJsonWhenAcceptIsWildcard() throws Exception {
+    // When Accept is */*, default to plain JSON.
     final Parameters parameters = new Parameters();
     parameters.addParameter().setName("test").setValue(new StringType("value"));
 
+    final StringWriter writer = new StringWriter();
     final HttpServletRequest request = mockRequest("*/*");
-    final HttpServletResponse response = mockResponse(new StringWriter());
+    final HttpServletResponse response = mockResponse(writer);
     final RequestDetails requestDetails = mock(RequestDetails.class);
 
     final boolean result =
         interceptor.transformResponse(requestDetails, parameters, request, response);
 
-    assertThat(result).isTrue();
-    verify(response, never()).setContentType("application/json");
+    assertThat(result).isFalse();
+    verify(response).setContentType("application/json");
+    verify(response).setCharacterEncoding("UTF-8");
   }
 
   @Test
@@ -301,6 +305,40 @@ void handlesEmptyParameters() throws Exception {
     assertThat(node.isEmpty()).isTrue();
   }
 
+  // -------------------------------------------------------------------------
+  // Native JSON tests
+  // -------------------------------------------------------------------------
+
+  @Test
+  void usesNativeJsonWhenAvailable() throws Exception {
+    // When Parameters has native JSON attached, it should be used directly.
+    final Parameters parameters = new Parameters();
+    parameters.addParameter().setName("test").setValue(new StringType("converted-value"));
+
+    // Attach native JSON with a different value to prove it's being used.
+    final String nativeJson = "{\"test\":\"native-value\",\"output\":[],\"error\":[]}";
+    parameters.setUserData("nativeJson", nativeJson);
+
+    final String json = transformAndCapture(parameters);
+    final JsonNode node = mapper.readTree(json);
+
+    // Should use the native JSON value, not the converted one.
+    assertThat(node.get("test").asText()).isEqualTo("native-value");
+  }
+
+  @Test
+  void fallsBackToConversionWhenNoNativeJson() throws Exception {
+    // When Parameters does not have native JSON, conversion should be used.
+    final Parameters parameters = new Parameters();
+    parameters.addParameter().setName("test").setValue(new StringType("converted-value"));
+
+    final String json = transformAndCapture(parameters);
+    final JsonNode node = mapper.readTree(json);
+
+    // Should use the converted value.
+    assertThat(node.get("test").asText()).isEqualTo("converted-value");
+  }
+
   // -------------------------------------------------------------------------
   // Helper methods
   // -------------------------------------------------------------------------

server/src/test/java/au/csiro/pathling/operations/bulkexport/ExportResponseTest.java

@@ -21,6 +21,8 @@
 
 import au.csiro.pathling.library.io.sink.FileInformation;
 import au.csiro.pathling.library.io.sink.WriteDetails;
+import au.csiro.pathling.shaded.com.fasterxml.jackson.databind.JsonNode;
+import au.csiro.pathling.shaded.com.fasterxml.jackson.databind.ObjectMapper;
 import jakarta.annotation.Nullable;
 import java.util.List;
 import org.hl7.fhir.r4.model.BooleanType;
@@ -228,6 +230,98 @@ void serverBaseUrlWithTrailingSlashHandledCorrectly() {
     assertThat(url).startsWith("http://example.org/fhir/$result").doesNotContain("fhir//$result");
   }
 
+  // -------------------------------------------------------------------------
+  // Native JSON tests
+  // -------------------------------------------------------------------------
+
+  @Test
+  void manifestIncludesEmptyErrorArray() {
+    // The manifest should always include an error parameter, even when there are no errors.
+    final WriteDetails writeDetails = new WriteDetails(List.of());
+
+    final ExportResponse response =
+        new ExportResponse(
+            "http://example.org/fhir/$export", "http://example.org/fhir", writeDetails, false);
+
+    final Parameters parameters = response.toOutput();
+
+    assertThat(hasParameter(parameters, "error")).isTrue();
+  }
+
+  @Test
+  void manifestAttachesNativeJson() {
+    // The Parameters should have native JSON attached via userData.
+    final WriteDetails writeDetails = new WriteDetails(List.of());
+
+    final ExportResponse response =
+        new ExportResponse(
+            "http://example.org/fhir/$export", "http://example.org/fhir", writeDetails, false);
+
+    final Parameters parameters = response.toOutput();
+    final Object nativeJson = parameters.getUserData("nativeJson");
+
+    assertThat(nativeJson).isNotNull();
+    assertThat(nativeJson).isInstanceOf(String.class);
+  }
+
+  @Test
+  void nativeJsonHasCorrectStructure() throws Exception {
+    // The native JSON should have output and error as arrays, with all expected fields.
+    final FileInformation patientFile =
+        new FileInformation("Patient", "file:///tmp/jobs/job-id/Patient.ndjson");
+    final FileInformation observationFile =
+        new FileInformation("Observation", "file:///tmp/jobs/job-id/Observation.ndjson");
+    final WriteDetails writeDetails = new WriteDetails(List.of(patientFile, observationFile));
+
+    final ExportResponse response =
+        new ExportResponse(
+            "http://example.org/fhir/$export", "http://example.org/fhir", writeDetails, false);
+
+    final Parameters parameters = response.toOutput();
+    final String nativeJson = (String) parameters.getUserData("nativeJson");
+    final ObjectMapper mapper = new ObjectMapper();
+    final JsonNode json = mapper.readTree(nativeJson);
+
+    // Verify all expected fields are present.
+    assertThat(json.has("transactionTime")).isTrue();
+    assertThat(json.has("request")).isTrue();
+    assertThat(json.has("requiresAccessToken")).isTrue();
+    assertThat(json.has("output")).isTrue();
+    assertThat(json.has("error")).isTrue();
+
+    // Verify output is always an array.
+    assertThat(json.get("output").isArray()).isTrue();
+    assertThat(json.get("output").size()).isEqualTo(2);
+    assertThat(json.get("output").get(0).get("type").asText()).isEqualTo("Patient");
+    assertThat(json.get("output").get(1).get("type").asText()).isEqualTo("Observation");
+
+    // Verify error is always an array (empty when no errors).
+    assertThat(json.get("error").isArray()).isTrue();
+    assertThat(json.get("error").isEmpty()).isTrue();
+
+    // Verify field values.
+    assertThat(json.get("request").asText()).isEqualTo("http://example.org/fhir/$export");
+    assertThat(json.get("requiresAccessToken").asBoolean()).isFalse();
+  }
+
+  @Test
+  void nativeJsonOutputIsEmptyArrayWhenNoFiles() throws Exception {
+    // When there are no files, output should be an empty array, not missing.
+    final WriteDetails writeDetails = new WriteDetails(List.of());
+
+    final ExportResponse response =
+        new ExportResponse(
+            "http://example.org/fhir/$export", "http://example.org/fhir", writeDetails, false);
+
+    final Parameters parameters = response.toOutput();
+    final String nativeJson = (String) parameters.getUserData("nativeJson");
+    final ObjectMapper mapper = new ObjectMapper();
+    final JsonNode json = mapper.readTree(nativeJson);
+
+    assertThat(json.get("output").isArray()).isTrue();
+    assertThat(json.get("output").isEmpty()).isTrue();
+  }
+
   // -------------------------------------------------------------------------
   // Helper methods
   // -------------------------------------------------------------------------