feat(rest)!: require virtual column outputs

docs/src/spec.yaml

@@ -4631,7 +4631,7 @@ components:
       type: object
       required:
         - input_columns
-        - data_type
+        - outputs
         - image
         - udf_version
         - udf_name
@@ -4642,9 +4642,11 @@ components:
           items:
             type: string
           description: List of input column names for the virtual column
-        data_type:
-          type: object
-          description: Data type of the virtual column using JSON representation
+        outputs:
+          type: array
+          items:
+            $ref: "#/components/schemas/AddVirtualColumnOutputEntry"
+          description: Output columns produced by the virtual column UDF
         image:
           type: string
           description: Docker image to use for the UDF
@@ -4685,6 +4687,34 @@ components:
             type: string
           description: User-supplied field metadata (string key-value pairs)
 
+    AddVirtualColumnOutputEntry:
+      type: object
+      required:
+        - column
+        - struct_field
+        - data_type
+        - nullable
+      properties:
+        column:
+          type: string
+          description: Physical output column name
+        struct_field:
+          type: string
+          description: Field name in the UDF output struct
+        data_type:
+          type: object
+          description: Data type of the output column using JSON representation
+        nullable:
+          type: boolean
+          description: Whether the output column is nullable
+        metadata:
+          type: object
+          additionalProperties:
+            type: string
+          propertyNames:
+            type: string
+          description: User-supplied output field metadata (string key-value pairs)
+
     AlterTableAddColumnsResponse:
       type: object
       required:

java/lance-namespace-apache-client/README.md

@@ -250,6 +250,7 @@ Class | Method | HTTP request | Description
 
  - [AddColumnsEntry](docs/AddColumnsEntry.md)
  - [AddVirtualColumnEntry](docs/AddVirtualColumnEntry.md)
+ - [AddVirtualColumnOutputEntry](docs/AddVirtualColumnOutputEntry.md)
  - [AlterColumnsEntry](docs/AlterColumnsEntry.md)
  - [AlterTableAddColumnsRequest](docs/AlterTableAddColumnsRequest.md)
  - [AlterTableAddColumnsResponse](docs/AlterTableAddColumnsResponse.md)

java/lance-namespace-apache-client/api/openapi.yaml

@@ -7060,6 +7060,19 @@ components:
         - expression: expression
           name: name
           virtual_column:
+            outputs:
+            - metadata:
+                key: metadata
+              nullable: true
+              column: column
+              data_type: "{}"
+              struct_field: struct_field
+            - metadata:
+                key: metadata
+              nullable: true
+              column: column
+              data_type: "{}"
+              struct_field: struct_field
             image: image
             field_metadata:
               key: field_metadata
@@ -7070,13 +7083,25 @@ components:
             input_columns:
             - input_columns
             - input_columns
-            data_type: "{}"
             udf_backend: udf_backend
             auto_backfill: true
             udf_version: udf_version
         - expression: expression
           name: name
           virtual_column:
+            outputs:
+            - metadata:
+                key: metadata
+              nullable: true
+              column: column
+              data_type: "{}"
+              struct_field: struct_field
+            - metadata:
+                key: metadata
+              nullable: true
+              column: column
+              data_type: "{}"
+              struct_field: struct_field
             image: image
             field_metadata:
               key: field_metadata
@@ -7087,7 +7112,6 @@ components:
             input_columns:
             - input_columns
             - input_columns
-            data_type: "{}"
             udf_backend: udf_backend
             auto_backfill: true
             udf_version: udf_version
@@ -7114,6 +7138,19 @@ components:
         expression: expression
         name: name
         virtual_column:
+          outputs:
+          - metadata:
+              key: metadata
+            nullable: true
+            column: column
+            data_type: "{}"
+            struct_field: struct_field
+          - metadata:
+              key: metadata
+            nullable: true
+            column: column
+            data_type: "{}"
+            struct_field: struct_field
           image: image
           field_metadata:
             key: field_metadata
@@ -7124,7 +7161,6 @@ components:
           input_columns:
           - input_columns
           - input_columns
-          data_type: "{}"
           udf_backend: udf_backend
           auto_backfill: true
           udf_version: udf_version
@@ -7143,6 +7179,19 @@ components:
       - name
     AddVirtualColumnEntry:
       example:
+        outputs:
+        - metadata:
+            key: metadata
+          nullable: true
+          column: column
+          data_type: "{}"
+          struct_field: struct_field
+        - metadata:
+            key: metadata
+          nullable: true
+          column: column
+          data_type: "{}"
+          struct_field: struct_field
         image: image
         field_metadata:
           key: field_metadata
@@ -7153,7 +7202,6 @@ components:
         input_columns:
         - input_columns
         - input_columns
-        data_type: "{}"
         udf_backend: udf_backend
         auto_backfill: true
         udf_version: udf_version
@@ -7163,9 +7211,11 @@ components:
           items:
             type: string
           type: array
-        data_type:
-          description: Data type of the virtual column using JSON representation
-          type: object
+        outputs:
+          description: Output columns produced by the virtual column UDF
+          items:
+            $ref: '#/components/schemas/AddVirtualColumnOutputEntry'
+          type: array
         image:
           description: Docker image to use for the UDF
           type: string
@@ -7199,12 +7249,42 @@ components:
             type: string
           description: User-supplied field metadata (string key-value pairs)
       required:
-      - data_type
       - image
       - input_columns
+      - outputs
       - udf
       - udf_name
       - udf_version
+    AddVirtualColumnOutputEntry:
+      example:
+        metadata:
+          key: metadata
+        nullable: true
+        column: column
+        data_type: "{}"
+        struct_field: struct_field
+      properties:
+        column:
+          description: Physical output column name
+          type: string
+        struct_field:
+          description: Field name in the UDF output struct
+          type: string
+        data_type:
+          description: Data type of the output column using JSON representation
+          type: object
+        nullable:
+          description: Whether the output column is nullable
+          type: boolean
+        metadata:
+          additionalProperties:
+            type: string
+          description: User-supplied output field metadata (string key-value pairs)
+      required:
+      - column
+      - data_type
+      - nullable
+      - struct_field
     AlterTableAddColumnsResponse:
       example:
         version: 0

java/lance-namespace-apache-client/docs/AddVirtualColumnEntry.md

@@ -8,7 +8,7 @@
 | Name | Type | Description | Notes |
 |------------ | ------------- | ------------- | -------------|
 |**inputColumns** | **List<String>** | List of input column names for the virtual column |  |
-|**dataType** | **Object** | Data type of the virtual column using JSON representation |  |
+|**outputs** | [**List<AddVirtualColumnOutputEntry>**](AddVirtualColumnOutputEntry.md) | Output columns produced by the virtual column UDF |  |
 |**image** | **String** | Docker image to use for the UDF |  |
 |**udf** | **String** | Base64 encoded pickled UDF |  |
 |**udfName** | **String** | Name of the UDF |  |

java/lance-namespace-apache-client/docs/AddVirtualColumnOutputEntry.md

@@ -0,0 +1,17 @@
+
+
+# AddVirtualColumnOutputEntry
+
+
+## Properties
+
+| Name | Type | Description | Notes |
+|------------ | ------------- | ------------- | -------------|
+|**column** | **String** | Physical output column name |  |
+|**structField** | **String** | Field name in the UDF output struct |  |
+|**dataType** | **Object** | Data type of the output column using JSON representation |  |
+|**nullable** | **Boolean** | Whether the output column is nullable |  |
+|**metadata** | **Map<String, String>** | User-supplied output field metadata (string key-value pairs) |  [optional] |
+
+
+

java/lance-namespace-apache-client/src/main/java/org/lance/namespace/model/AddVirtualColumnEntry.java

@@ -32,7 +32,7 @@
 /** AddVirtualColumnEntry */
 @JsonPropertyOrder({
   AddVirtualColumnEntry.JSON_PROPERTY_INPUT_COLUMNS,
-  AddVirtualColumnEntry.JSON_PROPERTY_DATA_TYPE,
+  AddVirtualColumnEntry.JSON_PROPERTY_OUTPUTS,
   AddVirtualColumnEntry.JSON_PROPERTY_IMAGE,
   AddVirtualColumnEntry.JSON_PROPERTY_UDF,
   AddVirtualColumnEntry.JSON_PROPERTY_UDF_NAME,
@@ -50,8 +50,8 @@ public class AddVirtualColumnEntry {
   public static final String JSON_PROPERTY_INPUT_COLUMNS = "input_columns";
   @javax.annotation.Nonnull private List<String> inputColumns = new ArrayList<>();
 
-  public static final String JSON_PROPERTY_DATA_TYPE = "data_type";
-  @javax.annotation.Nonnull private Object dataType;
+  public static final String JSON_PROPERTY_OUTPUTS = "outputs";
+  @javax.annotation.Nonnull private List<AddVirtualColumnOutputEntry> outputs = new ArrayList<>();
 
   public static final String JSON_PROPERTY_IMAGE = "image";
   @javax.annotation.Nonnull private String image;
@@ -122,28 +122,37 @@ public void setInputColumns(@javax.annotation.Nonnull List<String> inputColumns)
     this.inputColumns = inputColumns;
   }
 
-  public AddVirtualColumnEntry dataType(@javax.annotation.Nonnull Object dataType) {
+  public AddVirtualColumnEntry outputs(
+      @javax.annotation.Nonnull List<AddVirtualColumnOutputEntry> outputs) {
 
-    this.dataType = dataType;
+    this.outputs = outputs;
+    return this;
+  }
+
+  public AddVirtualColumnEntry addOutputsItem(AddVirtualColumnOutputEntry outputsItem) {
+    if (this.outputs == null) {
+      this.outputs = new ArrayList<>();
+    }
+    this.outputs.add(outputsItem);
     return this;
   }
 
   /**
-   * Data type of the virtual column using JSON representation
+   * Output columns produced by the virtual column UDF
    *
-   * @return dataType
+   * @return outputs
    */
   @javax.annotation.Nonnull
-  @JsonProperty(JSON_PROPERTY_DATA_TYPE)
+  @JsonProperty(JSON_PROPERTY_OUTPUTS)
   @JsonInclude(value = JsonInclude.Include.ALWAYS)
-  public Object getDataType() {
-    return dataType;
+  public List<AddVirtualColumnOutputEntry> getOutputs() {
+    return outputs;
   }
 
-  @JsonProperty(JSON_PROPERTY_DATA_TYPE)
+  @JsonProperty(JSON_PROPERTY_OUTPUTS)
   @JsonInclude(value = JsonInclude.Include.ALWAYS)
-  public void setDataType(@javax.annotation.Nonnull Object dataType) {
-    this.dataType = dataType;
+  public void setOutputs(@javax.annotation.Nonnull List<AddVirtualColumnOutputEntry> outputs) {
+    this.outputs = outputs;
   }
 
   public AddVirtualColumnEntry image(@javax.annotation.Nonnull String image) {
@@ -414,7 +423,7 @@ public boolean equals(Object o) {
     }
     AddVirtualColumnEntry addVirtualColumnEntry = (AddVirtualColumnEntry) o;
     return Objects.equals(this.inputColumns, addVirtualColumnEntry.inputColumns)
-        && Objects.equals(this.dataType, addVirtualColumnEntry.dataType)
+        && Objects.equals(this.outputs, addVirtualColumnEntry.outputs)
         && Objects.equals(this.image, addVirtualColumnEntry.image)
         && Objects.equals(this.udf, addVirtualColumnEntry.udf)
         && Objects.equals(this.udfName, addVirtualColumnEntry.udfName)
@@ -439,7 +448,7 @@ private static <T> boolean equalsNullable(JsonNullable<T> a, JsonNullable<T> b)
   public int hashCode() {
     return Objects.hash(
         inputColumns,
-        dataType,
+        outputs,
         image,
         udf,
         udfName,
@@ -463,7 +472,7 @@ public String toString() {
     StringBuilder sb = new StringBuilder();
     sb.append("class AddVirtualColumnEntry {\n");
     sb.append("    inputColumns: ").append(toIndentedString(inputColumns)).append("\n");
-    sb.append("    dataType: ").append(toIndentedString(dataType)).append("\n");
+    sb.append("    outputs: ").append(toIndentedString(outputs)).append("\n");
     sb.append("    image: ").append(toIndentedString(image)).append("\n");
     sb.append("    udf: ").append(toIndentedString(udf)).append("\n");
     sb.append("    udfName: ").append(toIndentedString(udfName)).append("\n");
@@ -540,19 +549,22 @@ public String toUrlQueryString(String prefix) {
       }
     }
 
-    // add `data_type` to the URL query string
-    if (getDataType() != null) {
-      try {
-        joiner.add(
-            String.format(
-                "%sdata_type%s=%s",
-                prefix,
-                suffix,
-                URLEncoder.encode(String.valueOf(getDataType()), "UTF-8")
-                    .replaceAll("\\+", "%20")));
-      } catch (UnsupportedEncodingException e) {
-        // Should never happen, UTF-8 is always supported
-        throw new RuntimeException(e);
+    // add `outputs` to the URL query string
+    if (getOutputs() != null) {
+      for (int i = 0; i < getOutputs().size(); i++) {
+        if (getOutputs().get(i) != null) {
+          joiner.add(
+              getOutputs()
+                  .get(i)
+                  .toUrlQueryString(
+                      String.format(
+                          "%soutputs%s%s",
+                          prefix,
+                          suffix,
+                          "".equals(suffix)
+                              ? ""
+                              : String.format("%s%d%s", containerPrefix, i, containerSuffix))));
+        }
       }
     }