Skip to content

Commit 990fe21

Browse files
msprucmspruc
authored
Merge pull request #572 from juripetersen/fs-based-calcite-schema
Use file based schema and named args for SqlContext main
2 parents 7312ade + e01ed91 commit 990fe21

File tree

7 files changed

+343
-49
lines changed

7 files changed

+343
-49
lines changed

‎pom.xml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@
106106
<scala.mayor.version>2.12</scala.mayor.version>
107107
<spark.version>3.4.4</spark.version>
108108
<flink.version>1.20.0</flink.version>
109-
<calcite.version>1.39.0</calcite.version>
109+
<calcite.version>1.39.0</calcite.version>
110110

111111
<java.version>17</java.version>
112112
<source.level>17</source.level>
@@ -785,6 +785,12 @@
785785
<artifactId>guava</artifactId>
786786
<version>${guava.version}</version>
787787
</dependency>
788+
<!-- https://mvnrepository.com/artifact/commons-cli/commons-cli -->
789+
<dependency>
790+
<groupId>commons-cli</groupId>
791+
<artifactId>commons-cli</artifactId>
792+
<version>1.3.1</version>
793+
</dependency>
788794
</dependencies>
789795
</dependencyManagement>
790796

‎wayang-api/pom.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
<module>wayang-api-python</module>
4141
<module>wayang-api-sql</module>
4242
<module>wayang-api-json</module>
43+
<module>wayang-api-utils</module>
4344
</modules>
4445

4546
</project>

‎wayang-api/wayang-api-scala-java/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,11 @@
105105
<artifactId>wayang-spark</artifactId>
106106
<version>1.0.1-SNAPSHOT</version>
107107
</dependency>
108+
<dependency>
109+
<groupId>org.apache.wayang</groupId>
110+
<artifactId>wayang-tensorflow</artifactId>
111+
<version>1.0.1-SNAPSHOT</version>
112+
</dependency>
108113
<dependency>
109114
<groupId>org.apache.spark</groupId>
110115
<artifactId>spark-core_2.12</artifactId>

‎wayang-api/wayang-api-sql/pom.xml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
<version>1.0.1-SNAPSHOT</version>
2626
</parent>
2727
<modelVersion>4.0.0</modelVersion>
28-
28+
2929
<artifactId>wayang-api-sql</artifactId>
3030
<dependencies>
3131
<dependency>
@@ -49,6 +49,12 @@
4949
<version>1.0.1-SNAPSHOT</version>
5050
<scope>compile</scope>
5151
</dependency>
52+
<dependency>
53+
<groupId>org.apache.wayang</groupId>
54+
<artifactId>wayang-api-utils</artifactId>
55+
<version>1.0.1-SNAPSHOT</version>
56+
<scope>compile</scope>
57+
</dependency>
5258
<dependency>
5359
<groupId>org.apache.wayang</groupId>
5460
<artifactId>wayang-postgres</artifactId>

‎wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/context/SqlContext.java‎

Lines changed: 42 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919

2020
import org.apache.commons.lang3.StringUtils;
2121

22-
2322
import org.apache.calcite.jdbc.CalciteSchema;
2423
import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
2524
import org.apache.calcite.rel.RelNode;
@@ -38,22 +37,30 @@
3837
import org.apache.wayang.basic.data.Record;
3938
import org.apache.wayang.core.api.Configuration;
4039
import org.apache.wayang.core.plugin.Plugin;
40+
import org.apache.wayang.api.utils.Parameters;
4141
import org.apache.wayang.core.api.WayangContext;
4242
import org.apache.wayang.core.util.ReflectionUtils;
4343
import org.apache.wayang.core.plan.wayangplan.WayangPlan;
4444
import org.apache.wayang.java.Java;
4545
import org.apache.wayang.postgres.Postgres;
4646
import org.apache.wayang.spark.Spark;
47+
import org.apache.commons.cli.*;
48+
49+
import com.google.common.io.Resources;
4750

4851
import org.json.simple.JSONObject;
4952
import org.json.simple.parser.JSONParser;
5053

54+
import scala.collection.JavaConversions;
5155
import java.io.BufferedWriter;
5256
import java.io.IOException;
5357
import java.nio.file.Files;
58+
import java.nio.charset.Charset;
5459
import java.nio.file.Paths;
60+
import java.net.URL;
5561
import java.sql.SQLException;
5662
import java.util.ArrayList;
63+
import java.util.Arrays;
5764
import java.util.Collection;
5865
import java.util.Properties;
5966
import java.util.concurrent.atomic.AtomicInteger;
@@ -103,43 +110,46 @@ public static void main(final String[] args) throws Exception {
103110
throw new IllegalArgumentException(
104111
"Usage: ./bin/wayang-submit org.apache.wayang.api.sql.SqlContext <SQL statement path> <JDBC driver> <JDBC URL> <JDBC user> <JDBC password> <Result output path> [platforms...]");
105112

106-
final String queryPath = args[0];
107-
final String jdbcDriver = args[1];
108-
final String jdbcUrl = args[2];
109-
final String jdbcUser = args[3];
110-
final String jdbcPassword = args[4];
111-
final String outputPath = args[5];
113+
//Specify the named arguments
114+
Options options = new Options();
115+
options.addOption("p", "platforms", true, "[platforms...]");
116+
options.addOption("s", "schema", true, "Schema path");
117+
options.addOption("q", "query", true, "SQL statement path");
118+
options.addOption("o", "outputPath", true, "Output path");
119+
options.addOption("d", "data", true, "Data path for file-based schema");
120+
options.addOption("c", "config", true, "File path for config file");
121+
options.addOption("jdbcDriver", true, "JDBC driver");
122+
options.addOption("jdbcUrl", true, "JDBC URL");
123+
options.addOption("jdbcPassword", true, "JDBC URL");
124+
125+
CommandLineParser parser = new DefaultParser();
126+
CommandLine cmd = parser.parse(options, args);
127+
128+
final String queryPath = cmd.getOptionValue("q");
129+
final String jdbcDriver = cmd.getOptionValue("jdbcDriver");
130+
final String jdbcUrl = cmd.getOptionValue("jdbcUrl");
131+
final String jdbcUser = cmd.getOptionValue("jdbcUser");
132+
final String jdbcPassword = cmd.getOptionValue("jdbcPassword");
133+
final String outputPath = cmd.getOptionValue("o");
134+
final String dataPath = cmd.getOptionValue("d");
135+
final String schemaPath = cmd.getOptionValue("s");
112136

113137
final String query = StringUtils.chop(
114138
Files.readString(Paths.get(queryPath))
115139
.stripTrailing());
116140

117141
final String driverPlatform = jdbcDriver.split("\\.")[0];
118142

119-
final String calciteModel = String.format(
120-
"{\r\n" +
121-
"\"calcite\": {\r\n" +
122-
" \"version\": \"1.0\",\n" +
123-
" \"defaultSchema\": \"wayang\",\n" +
124-
" \"schemas\": [\n" +
125-
" {\n" +
126-
" \"name\": \"postgres\",\n" +
127-
" \"type\": \"custom\",\n" +
128-
" \"factory\": \"org.apache.wayang.api.sql.calcite.jdbc.JdbcSchema$Factory\",\n" +
129-
" \"operand\": {\n" +
130-
" \"jdbcDriver\": \"%s\",\n" +
131-
" \"jdbcUrl\": \"%s\",\n" +
132-
" \"jdbcUser\": \"%s\",\n" +
133-
" \"jdbcPassword\": \"%s\"\n" +
134-
" }\n" +
135-
" }\n" +
136-
" ]\n" +
137-
"}\r\n" +
138-
"}",
139-
jdbcDriver, jdbcUrl, jdbcUser, jdbcPassword);
140-
141143
final Configuration configuration = new Configuration();
142-
configuration.load(ReflectionUtils.loadResource("wayang-defaults.properties"));
144+
145+
if (cmd.hasOption("c")) {
146+
configuration.load(cmd.getOptionValue("c"));
147+
}
148+
149+
final String calciteModel = Resources.toString(
150+
new URL(schemaPath),
151+
Charset.defaultCharset()
152+
);
143153

144154
configuration.setProperty("wayang.calcite.model", calciteModel);
145155
configuration.setProperty(String.format("wayang.%s.jdbc.url", driverPlatform), jdbcUrl);
@@ -153,23 +163,8 @@ public static void main(final String[] args) throws Exception {
153163
final SqlContext context = new SqlContext(parseModel,
154164
List.of(Java.channelConversionPlugin(), Postgres.conversionPlugin()));
155165

156-
for (int i = 6; i < args.length; i++) {
157-
final String platform = args[i];
158-
159-
switch (platform.toLowerCase()) {
160-
case "spark":
161-
context.withPlugin(Spark.basicPlugin());
162-
break;
163-
case "java":
164-
context.withPlugin(Java.basicPlugin());
165-
break;
166-
case "postgres":
167-
context.withPlugin(Postgres.plugin());
168-
break;
169-
default:
170-
throw new IllegalArgumentException("platform not supported " + platform);
171-
}
172-
}
166+
List<Plugin> plugins = JavaConversions.seqAsJavaList(Parameters.loadPlugins(cmd.getOptionValue("p")));
167+
plugins.stream().forEach(context::register);
173168

174169
final Collection<Record> result = context.executeSql(query);
175170

‎wayang-api/wayang-api-utils/pom.xml‎

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed to the Apache Software Foundation (ASF) under one
4+
or more contributor license agreements. See the NOTICE file
5+
distributed with this work for additional information
6+
regarding copyright ownership. The ASF licenses this file
7+
to you under the Apache License, Version 2.0 (the
8+
"License"); you may not use this file except in compliance
9+
with the License. You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing,
14+
software distributed under the License is distributed on an
15+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
KIND, either express or implied. See the License for the
17+
specific language governing permissions and limitations
18+
under the License.
19+
-->
20+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
21+
<modelVersion>4.0.0</modelVersion>
22+
23+
<parent>
24+
<artifactId>wayang-api</artifactId>
25+
<groupId>org.apache.wayang</groupId>
26+
<version>1.0.1-SNAPSHOT</version>
27+
</parent>
28+
29+
<artifactId>wayang-api-utils</artifactId>
30+
<version>1.0.1-SNAPSHOT</version>
31+
32+
<name>Wayang API Utils</name>
33+
<description>Wayang implementation of an API of Scala-Java to be enable to work in functional style</description>
34+
35+
<properties>
36+
<java-module-name>org.apache.wayang.api</java-module-name>
37+
<tensorflow.version>1.0.0</tensorflow.version>
38+
</properties>
39+
40+
<dependencyManagement>
41+
<dependencies>
42+
<dependency>
43+
<groupId>org.apache.wayang</groupId>
44+
<artifactId>wayang-commons</artifactId>
45+
<version>1.0.1-SNAPSHOT</version>
46+
<type>pom</type>
47+
<scope>import</scope>
48+
</dependency>
49+
</dependencies>
50+
</dependencyManagement>
51+
52+
<dependencies>
53+
<dependency>
54+
<groupId>org.apache.wayang</groupId>
55+
<artifactId>wayang-core</artifactId>
56+
<version>1.0.1-SNAPSHOT</version>
57+
</dependency>
58+
<dependency>
59+
<groupId>org.apache.wayang</groupId>
60+
<artifactId>wayang-basic</artifactId>
61+
<version>1.0.1-SNAPSHOT</version>
62+
</dependency>
63+
<dependency>
64+
<groupId>org.apache.wayang</groupId>
65+
<artifactId>wayang-utils-profile-db</artifactId>
66+
<version>1.0.1-SNAPSHOT</version>
67+
</dependency>
68+
<dependency>
69+
<groupId>org.apache.wayang</groupId>
70+
<artifactId>wayang-api-python</artifactId>
71+
<version>1.0.1-SNAPSHOT</version>
72+
</dependency>
73+
<dependency>
74+
<groupId>org.apache.commons</groupId>
75+
<artifactId>commons-lang3</artifactId>
76+
<version>3.12.0</version>
77+
</dependency>
78+
<dependency>
79+
<groupId>org.apache.wayang</groupId>
80+
<artifactId>wayang-java</artifactId>
81+
<version>1.0.1-SNAPSHOT</version>
82+
</dependency>
83+
<dependency>
84+
<groupId>org.apache.wayang</groupId>
85+
<artifactId>wayang-sqlite3</artifactId>
86+
<version>1.0.1-SNAPSHOT</version>
87+
</dependency>
88+
<dependency>
89+
<groupId>org.apache.wayang</groupId>
90+
<artifactId>wayang-postgres</artifactId>
91+
<version>1.0.1-SNAPSHOT</version>
92+
</dependency>
93+
<dependency>
94+
<groupId>org.apache.hadoop</groupId>
95+
<artifactId>hadoop-common</artifactId>
96+
<scope>test</scope>
97+
</dependency>
98+
<dependency>
99+
<groupId>org.apache.hadoop</groupId>
100+
<artifactId>hadoop-hdfs</artifactId>
101+
<scope>test</scope>
102+
</dependency>
103+
<dependency>
104+
<groupId>org.apache.wayang</groupId>
105+
<artifactId>wayang-spark</artifactId>
106+
<version>1.0.1-SNAPSHOT</version>
107+
</dependency>
108+
<dependency>
109+
<groupId>org.apache.wayang</groupId>
110+
<artifactId>wayang-tensorflow</artifactId>
111+
<version>1.0.1-SNAPSHOT</version>
112+
</dependency>
113+
<dependency>
114+
<groupId>org.apache.spark</groupId>
115+
<artifactId>spark-core_2.12</artifactId>
116+
<version>${spark.version}</version>
117+
<scope>test</scope>
118+
</dependency>
119+
<dependency>
120+
<groupId>org.apache.spark</groupId>
121+
<artifactId>spark-graphx_2.12</artifactId>
122+
<version>${spark.version}</version>
123+
<scope>test</scope>
124+
</dependency>
125+
<dependency>
126+
<groupId>org.apache.spark</groupId>
127+
<artifactId>spark-mllib_2.12</artifactId>
128+
<version>${spark.version}</version>
129+
</dependency>
130+
<dependency>
131+
<groupId>org.scala-lang</groupId>
132+
<artifactId>scala-library</artifactId>
133+
<version>${scala.version}</version>
134+
<scope>provided</scope>
135+
</dependency>
136+
<dependency>
137+
<groupId>io.netty</groupId>
138+
<artifactId>netty-all</artifactId>
139+
<version>4.1.89.Final</version>
140+
</dependency>
141+
<dependency>
142+
<groupId>com.google.protobuf</groupId>
143+
<artifactId>protobuf-java</artifactId>
144+
<version>3.25.5</version>
145+
</dependency>
146+
<dependency>
147+
<groupId>com.fasterxml.jackson.core</groupId>
148+
<artifactId>jackson-databind</artifactId>
149+
<version>2.15.4</version>
150+
</dependency>
151+
<dependency>
152+
<groupId>com.fasterxml.jackson.module</groupId>
153+
<artifactId>jackson-module-scala_2.12</artifactId>
154+
</dependency>
155+
<dependency>
156+
<groupId>org.tensorflow</groupId>
157+
<artifactId>tensorflow-core-platform</artifactId>
158+
<version>${tensorflow.version}</version>
159+
</dependency>
160+
<dependency>
161+
<groupId>org.tensorflow</groupId>
162+
<artifactId>tensorflow-framework</artifactId>
163+
<version>${tensorflow.version}</version>
164+
</dependency>
165+
</dependencies>
166+
</project>

0 commit comments

Comments
 (0)