feat: lists serialize as JSON arrays, streams as JSONL

Author: cablehead

README.md

@@ -257,7 +257,8 @@ Content-type is determined in the following order of precedence:
 
 3. Inferred from value type:
    - Record -> `application/json`
-   - List or stream of records -> `application/x-ndjson` (JSONL)
+   - List -> `application/json` (JSON array)
+   - Stream of records -> `application/x-ndjson` (JSONL)
    - Binary or byte stream -> `application/octet-stream`
    - Empty (null) -> no Content-Type header
 
@@ -274,7 +275,8 @@ Examples:
 
 # 3. Inferred from value type
 {|req| {foo: "bar"} }                 # Record -> application/json
-{|req| [{a: 1}, {b: 2}, {c: 3}] }     # List of records -> application/x-ndjson
+{|req| [{a: 1}, {b: 2}, {c: 3}] }     # List -> application/json (array)
+{|req| 1..10 | each { {n: $in} } }    # Stream of records -> application/x-ndjson
 {|req| 0x[deadbeef] }                 # Binary -> application/octet-stream
 {|req| null }                         # Empty -> no Content-Type header
 

src/worker.rs

@@ -53,6 +53,19 @@ pub fn spawn_eval_thread(
             let _ = tx.borrow_mut().take(); // This will drop the sender if it wasn't used
         });
         let output = result?;
+
+        // Content-type inference (when pipeline metadata has no content-type):
+        //
+        // | Value type       | Content-Type           | Conversion          |
+        // |------------------|------------------------|---------------------|
+        // | Record (__html)  | text/html              | unwrap __html       |
+        // | Record           | application/json       | JSON object         |
+        // | List             | application/json       | JSON array          |
+        // | Binary           | application/octet-stream | raw bytes         |
+        // | Empty/Nothing    | None (no header)       | empty               |
+        // | ListStream       | application/x-ndjson   | JSONL (if records)  |
+        // | Other            | text/html (default)    | .to_string()        |
+        //
         let inferred_content_type = match &output {
             PipelineData::Value(Value::Record { val, .. }, meta)
                 if meta.as_ref().and_then(|m| m.content_type.clone()).is_none() =>
@@ -63,12 +76,10 @@ pub fn spawn_eval_thread(
                     Some("application/json".to_string())
                 }
             }
-            PipelineData::Value(Value::List { vals, .. }, meta)
-                if meta.as_ref().and_then(|m| m.content_type.clone()).is_none()
-                    && !vals.is_empty()
-                    && vals.iter().all(is_jsonl_record) =>
+            PipelineData::Value(Value::List { .. }, meta)
+                if meta.as_ref().and_then(|m| m.content_type.clone()).is_none() =>
             {
-                Some("application/x-ndjson".to_string())
+                Some("application/json".to_string())
             }
             PipelineData::Value(Value::Binary { .. }, meta)
                 if meta.as_ref().and_then(|m| m.content_type.clone()).is_none() =>
@@ -98,26 +109,6 @@ pub fn spawn_eval_thread(
                 let working_set = StateWorkingSet::new(&engine.state);
                 Err(format_cli_error(&working_set, error.as_ref(), None).into())
             }
-            PipelineData::Value(Value::List { vals, .. }, meta)
-                if !vals.is_empty() && vals.iter().all(is_jsonl_record) =>
-            {
-                let http_meta = extract_http_response_meta(meta.as_ref());
-                // JSONL: each record as JSON line
-                let jsonl: Vec<u8> = vals
-                    .into_iter()
-                    .flat_map(|v| {
-                        let mut line = serde_json::to_vec(&value_to_json(&v)).unwrap_or_default();
-                        line.push(b'\n');
-                        line
-                    })
-                    .collect();
-                let _ = body_tx.send((
-                    inferred_content_type,
-                    http_meta,
-                    ResponseTransport::Full(jsonl),
-                ));
-                Ok(())
-            }
             PipelineData::Value(value, meta) => {
                 let http_meta = extract_http_response_meta(meta.as_ref());
                 let _ = body_tx.send((

tests/server_test.rs

@@ -1924,7 +1924,8 @@ async fn test_record_json_content_type() {
 }
 
 #[tokio::test]
-async fn test_list_of_records_jsonl_content_type() {
+async fn test_list_of_records_json_content_type() {
+    // Lists serialize as JSON arrays (streams serialize as JSONL)
     let server = TestServer::new("127.0.0.1:0", "{|req| [{a: 1}, {b: 2}, {c: 3}]}", false).await;
     tokio::time::sleep(std::time::Duration::from_millis(500)).await;
 
@@ -1939,22 +1940,14 @@ async fn test_list_of_records_jsonl_content_type() {
     let response = String::from_utf8_lossy(&output.stdout);
 
     assert!(
-        response.contains("content-type: application/x-ndjson"),
-        "Expected application/x-ndjson content-type, got: {response}"
+        response.contains("content-type: application/json"),
+        "Expected application/json content-type, got: {response}"
     );
 
-    // Check JSONL format: each record on its own line
-    assert!(
-        response.contains(r#"{"a":1}"#),
-        "Expected JSONL line with a:1"
-    );
-    assert!(
-        response.contains(r#"{"b":2}"#),
-        "Expected JSONL line with b:2"
-    );
+    // Check JSON array format
     assert!(
-        response.contains(r#"{"c":3}"#),
-        "Expected JSONL line with c:3"
+        response.contains(r#"[{"a":1},{"b":2},{"c":3}]"#),
+        "Expected JSON array, got: {response}"
     );
 }