Replace column type overrides map with list

Author: Yaytay

query-engine-design-mode/pom.xml

@@ -100,13 +100,13 @@
     <dependency>
       <groupId>org.mockito</groupId>
       <artifactId>mockito-core</artifactId>
-      <version>5.17.0</version>
+      <version>5.18.0</version>
       <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.mockito</groupId>
       <artifactId>mockito-junit-jupiter</artifactId>
-      <version>5.17.0</version>
+      <version>5.18.0</version>
       <scope>test</scope>
     </dependency>
     <dependency>

query-engine/docs/query-engine-compose.yml

@@ -50,7 +50,7 @@ services:
     - 16686:16686
 
   query-engine:
-    image: ghcr.io/yaytay/query-engine-design-mode:0.0.70-2-main
+    image: ghcr.io/yaytay/query-engine-design-mode:0.0.70-3-main
     ports:
     - 2000:8080
     volumes:

query-engine/pom.xml

@@ -462,13 +462,13 @@
     <dependency>
       <groupId>org.mockito</groupId>
       <artifactId>mockito-core</artifactId>
-      <version>5.17.0</version>
+      <version>5.18.0</version>
       <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.mockito</groupId>
       <artifactId>mockito-junit-jupiter</artifactId>
-      <version>5.17.0</version>
+      <version>5.18.0</version>
       <scope>test</scope>
     </dependency>
     <dependency>

query-engine/src/main/java/uk/co/spudsoft/query/defn/ColumnTypeOverride.java

@@ -0,0 +1,128 @@
+/*
+ * Copyright (C) 2025 jtalbut
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package uk.co.spudsoft.query.defn;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder;
+import io.swagger.v3.oas.annotations.media.Schema;
+
+/**
+ * Override of the data type for a specific column.
+ * 
+ * This facility is rarely required, but can be useful when a database does not provide adequate information for Query Engine to correctly identify the type of a field.
+ * 
+ * This is known to be useful for boolean fields with MySQL.
+ * 
+ * Setting a column to use a type that the result does not fit is going to cause problems (loss of data or errors) - so be sure you do this with care.
+ * 
+ * @author jtalbut
+ */
+@JsonDeserialize(builder = ColumnTypeOverride.Builder.class)
+@Schema(description = """
+                      Override of the data type for a specific column.
+                      
+                      This facility is rarely required, but can be useful when a database does not provide adequate information for Query Engine to correctly identify the type of a field.
+                      
+                      This is known to be useful for boolean fields with MySQL.
+                      
+                      Setting a column to use a type that the result does not fit is going to cause problems (loss of data or errors) - so be sure you do this with care.
+                      """
+)
+public class ColumnTypeOverride {
+  
+  private final String column;
+  private final DataType type;
+
+  /**
+   * Get the name of the column that is to have its data type set.
+   * @return the name of the column that is to have its data type set.
+   */
+  @Schema(description = """
+                        <P>The name of the column that is to have its data type set.</P>
+                        """
+          , maxLength = 100
+  )
+  public String getColumn() {
+    return column;
+  }
+
+  /**
+   * Get the desired type of the column.
+   * @return the desired type of the column.
+   */
+  @Schema(description = """
+                        <P>The desired type of the column.</P>
+                        """
+  )
+  public DataType getType() {
+    return type;
+  }
+  
+  /**
+   * Builder class for ColumnTypeOverride.
+   */
+  @JsonPOJOBuilder(buildMethodName = "build", withPrefix = "")
+  public static class Builder {
+
+    private String column;
+    private DataType type;
+
+    private Builder() {
+    }
+
+    /**
+     * Set the {@link ColumnTypeOverride#column} value in the builder.
+     * @param value The value for the {@link ColumnTypeOverride#column}.
+     * @return this, so that this builder may be used in a fluent manner.
+     */
+    public Builder column(final String value) {
+      this.column = value;
+      return this;
+    }
+
+    /**
+     * Set the {@link ColumnTypeOverride#type} value in the builder.
+     * @param value The value for the {@link ColumnTypeOverride#type}.
+     * @return this, so that this builder may be used in a fluent manner.
+     */
+    public Builder type(final DataType value) {
+      this.type = value;
+      return this;
+    }
+
+    /**
+     * Create a new ColumnTypeOverride object.
+     * @return a new ColumnTypeOverride object. 
+     */
+    public ColumnTypeOverride build() {
+      return new uk.co.spudsoft.query.defn.ColumnTypeOverride(column, type);
+    }
+  }
+  
+  /**
+   * Create a new ColumnTypeOverride builder.
+   * @return a new ColumnTypeOverride builder. 
+   */
+  public static ColumnTypeOverride.Builder builder() {
+    return new ColumnTypeOverride.Builder();
+  }
+
+  private ColumnTypeOverride(final String column, final DataType type) {
+    this.column = column;
+    this.type = type;
+  }
+}

query-engine/src/main/java/uk/co/spudsoft/query/defn/SourceSql.java

@@ -18,14 +18,17 @@
 
 import com.fasterxml.jackson.annotation.JsonFormat;
 import com.fasterxml.jackson.annotation.JsonFormat.Shape;
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder;
 import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import io.swagger.v3.oas.annotations.media.Schema;
 import io.vertx.core.Context;
 import io.vertx.core.Vertx;
 import java.time.Duration;
+import java.util.List;
 import java.util.Map;
 import uk.co.spudsoft.query.exec.SharedMap;
 import uk.co.spudsoft.query.exec.SourceInstance;
@@ -72,7 +75,8 @@ public SourceInstance createInstance(Vertx vertx, Context context, SharedMap sha
   private final Duration idleTimeout;
   private final Duration connectionTimeout;
   private final Boolean replaceDoubleQuotes;
-  private final ImmutableMap<String, DataType> columnTypeOverrides;
+  private final ImmutableList<ColumnTypeOverride> columnTypeOverrides;
+  private final ImmutableMap<String, DataType> columnTypeOverrideMap;
 
 
   @Override
@@ -427,10 +431,19 @@ This is a map of column names (from the results for this query) to the Query Eng
                         Setting a column to use a type that the result does not fit is going to cause problems (loss of data or errors) - so be sure you do this with care.
                         """
   )
-  public Map<String, DataType> getColumnTypeOverrides() {
+  public List<ColumnTypeOverride> getColumnTypeOverrides() {
     return columnTypeOverrides;
   }
 
+  /**
+   * Get the defined {@link #columnTypeOverrides} as a map.
+   * @return the defined {@link #columnTypeOverrides} as a map.
+   */
+  @JsonIgnore
+  public Map<String, DataType> getColumnTypeOverrideMap() {
+    return columnTypeOverrideMap;
+  }
+  
   /**
    * Builder class for SourceSql.
    */
@@ -449,7 +462,7 @@ public static class Builder {
     private Duration idleTimeout;
     private Duration connectionTimeout;
     private Boolean replaceDoubleQuotes;
-    private ImmutableMap<String, DataType> columnTypeOverrides;
+    private ImmutableList<ColumnTypeOverride> columnTypeOverrides;
 
     private Builder() {
     }
@@ -579,7 +592,7 @@ public Builder replaceDoubleQuotes(final Boolean value) {
      * @param value The value for the {@link SourceSql#replaceDoubleQuotes}.
      * @return this, so that this builder may be used in a fluent manner.
      */
-    public Builder columnTypeOverrides(final Map<String, DataType> value) {
+    public Builder columnTypeOverrides(final List<ColumnTypeOverride> value) {
       this.columnTypeOverrides = ImmutableCollectionTools.copy(value);
       return this;
     }
@@ -635,7 +648,7 @@ public SourceSql(final SourceType type
           , final Duration idleTimeout
           , final Duration connectionTimeout
           , final Boolean replaceDoubleQuotes
-          , final Map<String, DataType> columnTypeOverrides
+          , final List<ColumnTypeOverride> columnTypeOverrides
   ) {
     validateType(SourceType.SQL, type);
     this.type = type;
@@ -650,8 +663,16 @@ public SourceSql(final SourceType type
     this.idleTimeout = idleTimeout;
     this.connectionTimeout = connectionTimeout;
     this.replaceDoubleQuotes = replaceDoubleQuotes;
-    this.columnTypeOverrides = ImmutableCollectionTools.copy(columnTypeOverrides);
+    if (columnTypeOverrides == null || columnTypeOverrides.isEmpty()) {
+      this.columnTypeOverrides = null;
+      this.columnTypeOverrideMap = null;
+    } else {
+      ImmutableMap.Builder<String, DataType> builder = ImmutableMap.<String, DataType>builder();
+      columnTypeOverrides.forEach(cto -> {
+        builder.put(cto.getColumn(), cto.getType());
+      });
+      this.columnTypeOverrides = ImmutableCollectionTools.copy(columnTypeOverrides);
+      this.columnTypeOverrideMap = builder.build();
+    }
   }
-
-  
 }

query-engine/src/main/java/uk/co/spudsoft/query/exec/sources/sql/RowStreamWrapper.java

@@ -80,11 +80,7 @@ public RowStreamWrapper(SourceNameTracker sourceNameTracker, SqlConnection conne
         if (columnTypeOverrides != null && columnTypeOverrides.containsKey(cd.name())) {
           types.putIfAbsent(cd.name(), columnTypeOverrides.get(cd.name()));
         } else {
-          try {
-            types.putIfAbsent(cd.name(), DataType.fromJdbcType(cd.jdbcType()));
-          } catch (Throwable ex) {
-            logger.warn("Failed to process column {}: ", cd.name(), ex);
-          }
+          types.putIfAbsent(cd.name(), DataType.fromJdbcType(cd.jdbcType()));
         }
       }
       logger.debug("Got types: {}", types);

query-engine/src/main/java/uk/co/spudsoft/query/exec/sources/sql/SourceSqlStreamingInstance.java

@@ -203,7 +203,7 @@ public Future<ReadStreamWithTypes> initialize(PipelineExecutor executor, Pipelin
               transaction = tran;
               logger.debug("Executing SQL stream on {} with {}", connection, args);
               MetadataRowStreamImpl rowStream = new MetadataRowStreamImpl(preparedStatement, context, definition.getStreamingFetchSize(), args);
-              rowStreamWrapper = new RowStreamWrapper(this, connection, transaction, rowStream, definition.getColumnTypeOverrides());
+              rowStreamWrapper = new RowStreamWrapper(this, connection, transaction, rowStream, definition.getColumnTypeOverrideMap());
               return rowStreamWrapper.ready();
             })
             .recover(ex -> {

query-engine/src/main/java/uk/co/spudsoft/query/main/Version.java

@@ -34,7 +34,7 @@ public final class Version {
   /**
    * The project version, as set in the Maven pom.xml.
    */
-  public static final String MAVEN_PROJECT_VERSION = "0.0.70-2-main";
+  public static final String MAVEN_PROJECT_VERSION = "0.0.70-3-main";
 
   private Version() {
   }

query-engine/src/main/resources/samples/sub1/sub2/AllDynamicIT.yaml

@@ -63,6 +63,10 @@ source:
         d."dataId" <= :maxId
     order by 
         d."dataId"
+  columnTypeOverrides:
+    - column: LongField
+      type: Double
+    
 processors:
 - type: GROUP_CONCAT
   delimiter: ","

query-engine/src/test/java/uk/co/spudsoft/query/defn/ColumnTypeOverrideTest.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2025 njt
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package uk.co.spudsoft.query.defn;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import org.junit.jupiter.api.Test;
+
+/**
+ *
+ * @author njt
+ */
+public class ColumnTypeOverrideTest {
+  
+  @Test
+  public void testGetColumn() {
+    ColumnTypeOverride cto = ColumnTypeOverride.builder().build();
+    assertNotNull(cto);
+    assertNull(cto.getColumn());
+    cto = ColumnTypeOverride.builder().column("col").build();
+    assertEquals("col", cto.getColumn());
+  }
+
+  @Test
+  public void testGetType() {
+    ColumnTypeOverride cto = ColumnTypeOverride.builder().build();
+    assertNotNull(cto);
+    assertNull(cto.getType());
+    cto = ColumnTypeOverride.builder().type(DataType.Double).build();
+    assertEquals(DataType.Double, cto.getType());
+  }
+
+}