Skip to content

Commit 64b7b73

Browse files
ryannedolanryannedolan
authored
Support flattening and unflattening structured types (#79)
1 parent 08729e9 commit 64b7b73

File tree

19 files changed

+372
-76
lines changed

19 files changed

+372
-76
lines changed

‎Makefile‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22
install:
33
./gradlew compileJava installDist
44

5+
test:
6+
./gradlew test -x spotbugsMain -x spotbugsTest -x spotbugsTestFixtures
7+
58
build:
69
./gradlew build
710
docker build . -t hoptimator
@@ -77,4 +80,4 @@ release:
7780
test -n "$(VERSION)" # MISSING ARG: $$VERSION
7881
./gradlew publish
7982

80-
.PHONY: build clean quickstart deploy-dev-environment deploy deploy-samples deploy-demo deploy-config integration-tests bounce generate-models release
83+
.PHONY: build test install clean quickstart deploy-dev-environment deploy deploy-samples deploy-demo deploy-config integration-tests bounce generate-models release

‎hoptimator-avro/build.gradle‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,55 @@
11
plugins {
22
id 'java'
3+
id 'maven-publish'
34
}
45

56
dependencies {
67
implementation project(':hoptimator-api')
78
implementation libs.avro
89
implementation libs.calcite.core
910
}
11+
12+
publishing {
13+
repositories {
14+
maven {
15+
name 'GitHubPackages'
16+
url = 'https://maven.pkg.github.com/linkedin/Hoptimator'
17+
credentials {
18+
username = System.getenv('GITHUB_ACTOR')
19+
password = System.getenv('GITHUB_TOKEN')
20+
}
21+
}
22+
maven {
23+
name 'LinkedInJFrog'
24+
url 'https://linkedin.jfrog.io/artifactory/hoptimator'
25+
credentials {
26+
username = System.getenv('JFROG_USERNAME')
27+
password = System.getenv('JFROG_API_KEY')
28+
}
29+
}
30+
}
31+
publications {
32+
maven(MavenPublication) {
33+
groupId = 'com.linkedin.hoptimator'
34+
artifactId = 'hoptimator-avro'
35+
version = System.getenv('VERSION')
36+
from components.java
37+
pom {
38+
name = 'hoptimator-avro'
39+
description = 'Hoptimator plugin for Apache Avro'
40+
url = 'https://github.com/linkedin/Hoptimator'
41+
licenses {
42+
license {
43+
name = 'BSD 2-Clause'
44+
url = 'https://raw.githubusercontent.com/linkedin/Hoptimator/main/LICENSE'
45+
}
46+
}
47+
scm {
48+
connection = 'scm:git:git://github.com:linkedin/Hoptimator.git'
49+
developerConnection = 'scm:git:ssh://github.com:linkedin/Hoptimator.git'
50+
url = 'https://github.com/linkedin/Hoptimator'
51+
}
52+
}
53+
}
54+
}
55+
}

‎hoptimator-demodb/src/main/java/com/linkedin/hoptimator/demodb/AdsSchema.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ public class AdsSchema extends AbstractSchema {
1414
public AdsSchema() {
1515
tableMap.put("PAGE_VIEWS", new PageViewTable());
1616
tableMap.put("AD_CLICKS", new AdClickTable());
17+
tableMap.put("CAMPAIGNS", new CampaignTable());
1718
}
1819

1920
@Override

‎hoptimator-jdbc-driver/build.gradle‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ plugins {
55
}
66

77
dependencies {
8-
implementation project(':hoptimator-avro')
98
implementation project(':hoptimator-demodb')
109
implementation project(':hoptimator-jdbc')
1110
implementation project(':hoptimator-util')

‎hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/schema/CatalogTable.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
import com.linkedin.hoptimator.util.RemoteTable;
88

99

10-
/** A table populated with all available Catlaogs. */
10+
/** A table populated with all available Catalogs. */
1111
public class CatalogTable extends RemoteTable<Catalog, CatalogTable.Row> {
1212

1313
// This and other Row classes are used by generated code, so it is important

‎hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/schema/UtilityCatalog.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
import java.sql.SQLException;
44
import java.sql.Wrapper;
5-
import java.util.HashMap;
5+
import java.util.LinkedHashMap;
66
import java.util.Map;
77

88
import org.apache.calcite.schema.SchemaPlus;
@@ -15,7 +15,7 @@
1515
/** Built-in utility tables. */
1616
public class UtilityCatalog extends AbstractSchema implements Catalog {
1717

18-
private final Map<String, Table> tableMap = new HashMap<>();
18+
private final Map<String, Table> tableMap = new LinkedHashMap<>();
1919

2020
public UtilityCatalog() {
2121
tableMap.put("PRINT", new PrintTable());

‎hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sConnector.java‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
import java.io.IOException;
44
import java.io.StringReader;
55
import java.sql.SQLException;
6-
import java.util.HashMap;
6+
import java.util.LinkedHashMap;
77
import java.util.Locale;
88
import java.util.Map;
99
import java.util.Properties;
@@ -47,10 +47,9 @@ public Map<String, String> configure(Source source) throws SQLException {
4747
} catch (IOException e) {
4848
throw new SQLException(e);
4949
}
50-
Map<String, String> map = new HashMap<>();
51-
for (String key : props.stringPropertyNames()) {
52-
map.put(key, props.getProperty(key));
53-
}
50+
Map<String, String> map = new LinkedHashMap<>();
51+
props.stringPropertyNames().stream().sorted().forEach(k ->
52+
map.put(k, props.getProperty(k)));
5453
return map;
5554
}
5655
}

‎hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/K8sDatabaseTable.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ public K8sDatabaseTable(K8sContext context) {
4747
public void addDatabases(SchemaPlus parentSchema) {
4848
for (Row row : rows()) {
4949
parentSchema.add(schemaName(row),
50-
HoptimatorJdbcSchema.create(row.NAME, null, row.SCHEMA, dataSource(row), parentSchema, dialect(row)));
50+
HoptimatorJdbcSchema.create(row.NAME, row.SCHEMA, dataSource(row), parentSchema, dialect(row)));
5151
}
5252
}
5353

‎hoptimator-kafka/src/test/resources/kafka-ddl.id‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,8 @@ spec:
2424
job:
2525
entryClass: com.linkedin.hoptimator.flink.runner.FlinkRunner
2626
args:
27-
- CREATE TABLE IF NOT EXISTS `existing-topic-2` (`KEY` VARCHAR, `VALUE` BINARY) WITH ('properties.bootstrap.servers'='localhost:9092', 'topic'='existing-topic-2', 'connector'='kafka')
28-
- CREATE TABLE IF NOT EXISTS `existing-topic-1` (`KEY` VARCHAR, `VALUE` BINARY) WITH ('properties.bootstrap.servers'='localhost:9092', 'topic'='existing-topic-1', 'connector'='kafka')
27+
- CREATE TABLE IF NOT EXISTS `existing-topic-2` (`KEY` VARCHAR, `VALUE` BINARY) WITH ('connector'='kafka', 'properties.bootstrap.servers'='localhost:9092', 'topic'='existing-topic-2')
28+
- CREATE TABLE IF NOT EXISTS `existing-topic-1` (`KEY` VARCHAR, `VALUE` BINARY) WITH ('connector'='kafka', 'properties.bootstrap.servers'='localhost:9092', 'topic'='existing-topic-1')
2929
- INSERT INTO `existing-topic-1` (`KEY`, `VALUE`) SELECT * FROM `KAFKA`.`existing-topic-2`
3030
jarURI: local:///opt/hoptimator-flink-runner.jar
3131
parallelism: 1

‎hoptimator-util/build.gradle‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,69 @@
11
plugins {
22
id 'java'
3+
id 'maven-publish'
34
}
45

56
dependencies {
67
implementation project(':hoptimator-api')
78
implementation libs.calcite.core
9+
10+
testImplementation(testFixtures(project(':hoptimator-jdbc')))
11+
testImplementation(platform('org.junit:junit-bom:5.11.3'))
12+
testImplementation 'org.junit.jupiter:junit-jupiter'
13+
testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
14+
testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine'
15+
}
16+
17+
test {
18+
useJUnitPlatform {
19+
excludeTags 'integration'
20+
}
21+
testLogging {
22+
events "passed", "skipped", "failed"
23+
}
24+
}
25+
26+
publishing {
27+
repositories {
28+
maven {
29+
name 'GitHubPackages'
30+
url = 'https://maven.pkg.github.com/linkedin/Hoptimator'
31+
credentials {
32+
username = System.getenv('GITHUB_ACTOR')
33+
password = System.getenv('GITHUB_TOKEN')
34+
}
35+
}
36+
maven {
37+
name 'LinkedInJFrog'
38+
url 'https://linkedin.jfrog.io/artifactory/hoptimator'
39+
credentials {
40+
username = System.getenv('JFROG_USERNAME')
41+
password = System.getenv('JFROG_API_KEY')
42+
}
43+
}
44+
}
45+
publications {
46+
maven(MavenPublication) {
47+
groupId = 'com.linkedin.hoptimator'
48+
artifactId = 'hoptimator-util'
49+
version = System.getenv('VERSION')
50+
from components.java
51+
pom {
52+
name = 'hoptimator-util'
53+
description = 'Utilities to help with extending Hoptimator'
54+
url = 'https://github.com/linkedin/Hoptimator'
55+
licenses {
56+
license {
57+
name = 'BSD 2-Clause'
58+
url = 'https://raw.githubusercontent.com/linkedin/Hoptimator/main/LICENSE'
59+
}
60+
}
61+
scm {
62+
connection = 'scm:git:git://github.com:linkedin/Hoptimator.git'
63+
developerConnection = 'scm:git:ssh://github.com:linkedin/Hoptimator.git'
64+
url = 'https://github.com/linkedin/Hoptimator'
65+
}
66+
}
67+
}
68+
}
869
}

0 commit comments

Comments
 (0)