Make DuckDB and Snowflake query client configs part of server-config.json (#1452)

Author: ferenc-csaky

documentation/docs/configuration.md

@@ -275,7 +275,10 @@ For dependent services, DataSQRL creates and uses the following environment vari
   * `POSTGRES_JDBC_URL`
   * `POSTGRES_USERNAME`
   * `POSTGRES_PASSWORD`
-
+* **Snowflake**
+  * `SNOWFLAKE_JDBC_URL`
+  * `SNOWFLAKE_EXTERNAL_VOLUME`
+  * `SNOWFLAKE_CATALOG_NAME`
 ---
 
 ## Default Configuration
@@ -303,7 +306,7 @@ The built-in fallback (excerpt - full version [here](https://raw.githubuserconte
       "schema-type": "aws-glue",
       "catalog-name": "${SNOWFLAKE_CATALOG_NAME}",
       "external-volume": "${SNOWFLAKE_EXTERNAL_VOLUME}",
-      "url": "jdbc:snowflake://${SNOWFLAKE_ID}.snowflakecomputing.com/?..."
+      "url": "${SNOWFLAKE_JDBC_URL}"
     }
   },
   "compiler": {

pom.xml

@@ -955,6 +955,13 @@
             <flinkrunner.image.repo>${flinkrunner.image.repo}</flinkrunner.image.repo>
             <flinkrunner.version>${flinkrunner.version}</flinkrunner.version>
           </systemPropertyVariables>
+          <argLine>${argLine}
+            --add-opens=java.base/java.util=ALL-UNNAMED
+            --add-opens=java.base/java.lang=ALL-UNNAMED
+            --add-opens=java.base/java.io=ALL-UNNAMED
+            --add-opens=java.base/java.nio=ALL-UNNAMED
+            --add-opens=java.base/java.net=ALL-UNNAMED
+          </argLine>
         </configuration>
         <executions>
           <execution>

sqrl-cli/src/main/resources/templates/server-config.json

@@ -109,14 +109,14 @@
     "http2RstFloodWindowDurationTimeUnit" : "SECONDS",
     "strictThreadMode" : false,
     "enabledSecureTransportProtocols" : [ "TLSv1", "TLSv1.1", "TLSv1.2" ],
+    "useAlpn" : false,
+    "sni" : false,
     "keyCertOptions" : null,
     "trustOptions" : null,
     "clientAuth" : "NONE",
     "sslHandshakeTimeout" : 10,
     "sslHandshakeTimeoutUnit" : "SECONDS",
-    "fileRegionEnabled" : true,
-    "useAlpn" : false,
-    "sni" : false
+    "fileRegionEnabled" : true
   },
   "pgConnectOptions" : {
     "host" : "${POSTGRES_HOST}",
@@ -195,5 +195,13 @@
     "value.deserializer" : "com.datasqrl.graphql.kafka.JsonDeserializer",
     "bootstrap.servers" : "${KAFKA_BOOTSTRAP_SERVERS}",
     "auto.offset.reset" : "latest"
+  },
+  "duckDbConfig" : {
+    "url" : "jdbc:duckdb:",
+    "driver" : null
+  },
+  "snowflakeConfig" : {
+    "url" : "${SNOWFLAKE_URL}",
+    "driver" : null
   }
 }

sqrl-cli/src/test/java/com/datasqrl/engine/server/GenericJavaServerEngineTest.java

@@ -18,7 +18,7 @@
 import static com.datasqrl.graphql.SqrlObjectMapper.MAPPER;
 import static org.assertj.core.api.Assertions.assertThat;
 
-import com.datasqrl.config.PackageJson.EmptyEngineConfig;
+import com.datasqrl.config.PackageJsonImpl;
 import com.datasqrl.graphql.config.ServerConfigUtil;
 import com.fasterxml.jackson.databind.JsonNode;
 import java.util.List;
@@ -28,7 +28,7 @@
 
 class GenericJavaServerEngineTest {
 
-  GenericJavaServerEngine underTest = new GenericJavaServerEngine("", new EmptyEngineConfig("")) {};
+  GenericJavaServerEngine underTest = new GenericJavaServerEngine("", new PackageJsonImpl()) {};
 
   @Test
   void test() {

sqrl-planner/src/main/java/com/datasqrl/engine/server/GenericJavaServerEngine.java

@@ -19,10 +19,13 @@
 import static com.datasqrl.graphql.SqrlObjectMapper.MAPPER;
 
 import com.datasqrl.config.EngineType;
+import com.datasqrl.config.PackageJson;
 import com.datasqrl.config.PackageJson.EngineConfig;
 import com.datasqrl.engine.EnginePhysicalPlan;
 import com.datasqrl.engine.EnginePhysicalPlan.DeploymentArtifact;
 import com.datasqrl.engine.ExecutionEngine;
+import com.datasqrl.engine.database.relational.DuckDBEngineFactory;
+import com.datasqrl.engine.database.relational.SnowflakeEngineFactory;
 import com.datasqrl.graphql.config.ServerConfig;
 import com.datasqrl.graphql.config.ServerConfigUtil;
 import io.vertx.core.json.JsonObject;
@@ -35,10 +38,12 @@
 public abstract class GenericJavaServerEngine extends ExecutionEngine.Base implements ServerEngine {
 
   private final EngineConfig engineConfig;
+  private final List<String> enabledEngines;
 
-  public GenericJavaServerEngine(String engineName, EngineConfig engineConfig) {
+  public GenericJavaServerEngine(String engineName, PackageJson packageJson) {
     super(engineName, EngineType.SERVER, NO_CAPABILITIES);
-    this.engineConfig = engineConfig;
+    engineConfig = packageJson.getEngines().getEngineConfigOrEmpty(engineName);
+    enabledEngines = packageJson.getEnabledEngines();
   }
 
   @Override
@@ -68,8 +73,17 @@ ServerConfig readDefaultConfig() {
     ServerConfig serverConfig;
     try (var input = getClass().getResourceAsStream("/templates/server-config.json")) {
       var json = MAPPER.readValue(input, JsonObject.class);
+      disableConfigIfNeeded(json, DuckDBEngineFactory.ENGINE_NAME, "duckDbConfig");
+      disableConfigIfNeeded(json, SnowflakeEngineFactory.ENGINE_NAME, "snowflakeConfig");
+
       serverConfig = new ServerConfig(json);
     }
     return serverConfig;
   }
+
+  private void disableConfigIfNeeded(JsonObject json, String engineName, String configName) {
+    if (!enabledEngines.contains(engineName)) {
+      json.putNull(configName);
+    }
+  }
 }

sqrl-planner/src/main/java/com/datasqrl/engine/server/VertxEngineFactory.java

@@ -40,7 +40,7 @@ public static class VertxEngine extends GenericJavaServerEngine {
 
     @Inject
     public VertxEngine(PackageJson packageJson) {
-      super(ENGINE_NAME, packageJson.getEngines().getEngineConfigOrEmpty(ENGINE_NAME));
+      super(ENGINE_NAME, packageJson);
     }
   }
 }

sqrl-planner/src/main/resources/default-package.json

@@ -38,7 +38,7 @@
       "schema-type": "aws-glue",
       "catalog-name": "${SNOWFLAKE_CATALOG_NAME}",
       "external-volume": "${SNOWFLAKE_EXTERNAL_VOLUME}",
-      "url": "jdbc:snowflake://${SNOWFLAKE_ID}.snowflakecomputing.com/?user=${SNOWFLAKE_USER}&password=${SNOWFLAKE_PASSWORD}&warehouse=COMPUTE_WH&db=MYSNOWFLAKEDB&schema=public&disableSslHostnameVerification=true"
+      "url": "${SNOWFLAKE_JDBC_URL}"
     }
   },
   "connectors": {

sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/graphql/jdbc/JdbcExecutionContext.java

@@ -79,14 +79,12 @@ public CompletableFuture runQuery(ResolvedSqlQuery resolvedQuery, boolean isList
     if (preparedQueryContainer != null) {
       return CompletableFuture.supplyAsync(
           () -> {
-            try (PreparedStatement statement =
-                preparedQueryContainer
-                    .getConnection()
-                    .prepareStatement(preparedQueryContainer.getPreparedQuery())) {
+            try (final var conn = preparedQueryContainer.connection();
+                final var stmt = conn.prepareStatement(preparedQueryContainer.preparedQuery())) {
               for (int i = 0; i < paramObj.size(); i++) {
-                statement.setObject(i + 1, paramObj.get(i));
+                stmt.setObject(i + 1, paramObj.get(i));
               }
-              ResultSet resultSet = statement.executeQuery();
+              var resultSet = stmt.executeQuery();
 
               return unboxList(resultSetToList(resultSet), isList);
             } catch (SQLException e) {

sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/graphql/jdbc/PreparedSqrlQueryImpl.java

@@ -17,11 +17,6 @@
 
 import com.datasqrl.graphql.server.RootGraphqlModel.PreparedSqrlQuery;
 import java.sql.Connection;
-import lombok.Value;
 
-@Value
-public class PreparedSqrlQueryImpl implements PreparedSqrlQuery<String> {
-
-  Connection connection;
-  String preparedQuery;
-}
+public record PreparedSqrlQueryImpl(Connection connection, String preparedQuery)
+    implements PreparedSqrlQuery<String> {}

sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/graphql/server/RootGraphqlModel.java

@@ -511,6 +511,6 @@ public <R, C> R accept(ResolvedQueryVisitor<R, C> visitor, C context) {
 
   public interface PreparedSqrlQuery<T> {
 
-    T getPreparedQuery();
+    T preparedQuery();
   }
 }