Publish hoptimator-cli and hoptimator-operator (#13)

* Publish to LinkedInJFrog

* Publish hoptimator-cli and hoptimator-operator

* Fix javadoc

* Disable Javadoc lint, for now

Author: ryannedolan

.github/workflows/release.yml

@@ -20,3 +20,6 @@ jobs:
           arguments: publish
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          JFROG_USERNAME:
+          JROG_API_KEY: 
+          VERSION: ${{ github.event.release.tag_name }}

Dockerfile

@@ -1,10 +1,10 @@
 FROM eclipse-temurin:18
 WORKDIR /home/
-COPY ./hoptimator-cli/run.sh ./hoptimator
-COPY ./hoptimator-operator/run.sh ./hoptimator-operator
-COPY ./hoptimator-cli/build/libs/hoptimator-cli-all.jar ./hoptimator-cli-all.jar
-COPY ./hoptimator-operator/build/libs/hoptimator-operator-all.jar ./hoptimator-operator-all.jar
-COPY ./etc/* ./
+ADD ./hoptimator-cli/run.sh ./hoptimator
+ADD ./hoptimator-operator/run.sh ./hoptimator-operator
+ADD ./hoptimator-cli/build/libs/hoptimator-cli-all.jar ./hoptimator-cli-all.jar
+ADD ./hoptimator-operator/build/libs/hoptimator-operator-all.jar ./hoptimator-operator-all.jar
+ADD ./etc/* ./
 ENTRYPOINT ["/bin/sh", "-c"]
 CMD ["./hoptimator -n '' -p '' -u jdbc:calcite:model=model.yaml"]
 

Makefile

@@ -47,6 +47,7 @@ generate-models:
 	./models/generate-models.sh
 
 release:
+	test -n "$(VERSION)"  # MISSING ARG: $$VERSION
 	./gradlew publish
 
 .PHONY: build clean quickstart deploy-dev-environment deploy deploy-samples deploy-config integration-tests bounce generate-models release

build.gradle

@@ -1 +1,12 @@
 
+
+subprojects {
+  tasks.withType(JavaCompile) {
+    options.release = 8
+    options.compilerArgs << '-Xlint:deprecation'
+    options.compilerArgs << '-Xlint:unchecked'
+  }
+  tasks.withType(Javadoc) {
+    options.addStringOption('Xdoclint:none', '-quiet')
+  }
+}

gradle/libs.versions.toml

@@ -13,11 +13,13 @@ flinkTableApiJavaBridge = "org.apache.flink:flink-table-api-java-bridge:1.17.0"
 flinkTableCommon = "org.apache.flink:flink-table-common:1.17.0"
 flinkTablePlanner = "org.apache.flink:flink-table-planner_2.12:1.17.0"
 flinkTableRuntime = "org.apache.flink:flink-table-runtime:1.17.0"
+flinkShadedForceShading = "org.apache.flink:flink-shaded-force-shading:17.0"
+flinkMetricsDropwizard = "org.apache.flink:flink-metrics-dropwizard:1.17.0"
 flinkConnectorKafka = "org.apache.flink:flink-sql-connector-kafka:1.17.0"
 flinkConnectorMySqlCdc = "com.ververica:flink-sql-connector-mysql-cdc:2.3.0"
 gson = "com.google.code.gson:gson:2.9.0"
 jackson = "com.fasterxml.jackson.core:jackson-core:2.14.1"
-jacksonYaml = "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.2"
+jacksonYaml = "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.14.1"
 javaxAnnotationApi = "javax.annotation:javax.annotation-api:1.3.2"
 junit = "junit:junit:4.12"
 kafkaClients = "org.apache.kafka:kafka-clients:2.7.1"

hoptimator-catalog/build.gradle

@@ -15,26 +15,35 @@ dependencies {
 publishing {
   repositories {
     maven {
-      url = "https://maven.pkg.github.com/linkedin/Hoptimator"
+      name 'GitHubPackages'
+      url = 'https://maven.pkg.github.com/linkedin/Hoptimator'
       credentials {
         username = System.getenv('GITHUB_ACTOR')
         password = System.getenv('GITHUB_TOKEN')
       }
     }
+    maven {
+      name 'LinkedInJFrog'
+      url 'https://linkedin.jfrog.io/artifactory/hoptimator'
+      credentials {
+        username = System.getenv('JFROG_USERNAME')
+        password = System.getenv('JFROG_API_KEY')
+      }
+    }
   }
   publications {
     maven(MavenPublication) {
       groupId = 'com.linkedin.hoptimator'
       artifactId = 'hoptimator-catalog'
-      version = System.getenv('GITHUB_REF_NAME')
+      version = System.getenv('VERSION')
       from components.java
     }
   }
 }
 
-tasks.withType(JavaCompile) {
-  options.compilerArgs << '-Xlint:deprecation'
-  options.compilerArgs << '-Xlint:unchecked'
+java {
+  withJavadocJar()
+  withSourcesJar()
 }
 
 idea {

hoptimator-catalog/src/main/java/com/linkedin/hoptimator/catalog/ConfigProvider.java

@@ -39,6 +39,24 @@ default ConfigProvider with(String key, Function<String, String> valueFunction)
     };
   }
 
+  default ConfigProvider with(Map<String, ?> configs) {
+    if (configs == null) {
+      return this;
+    }
+    return x -> {
+      Map<String, String> base = config(x);
+      Map<String, String> combined = new HashMap<>();
+      combined.putAll(base);
+      configs.forEach((k, v) -> {
+        if (base.containsKey(k)) {
+          throw new IllegalStateException("Key '" + k + "' previously defined.");
+        }
+        combined.put(k, v.toString());
+      });
+      return combined;
+    };
+  }
+
   default ConfigProvider with(String key, String value) {
     return with(key, x -> value);
   }

hoptimator-catalog/src/main/java/com/linkedin/hoptimator/catalog/ScriptImplementor.java

@@ -1,7 +1,7 @@
 package com.linkedin.hoptimator.catalog;
 
 import org.apache.calcite.sql.SqlWriter;
-import org.apache.calcite.sql.SqlWriterConfig;
+//import org.apache.calcite.sql.SqlWriterConfig;
 import org.apache.calcite.sql.SqlDataTypeSpec;
 import org.apache.calcite.sql.SqlRowTypeNameSpec;
 import org.apache.calcite.sql.SqlBasicTypeNameSpec;
@@ -93,7 +93,9 @@ default String sql() {
 
   /** Render the script as DDL/SQL in the given dialect */
   default String sql(SqlDialect dialect) {
-    SqlWriter w = new SqlPrettyWriter(SqlWriterConfig.of().withDialect(dialect));
+    SqlWriter w = new SqlPrettyWriter(dialect);
+//  above is deprecated; replace with:
+//    SqlWriter w = new SqlPrettyWriter(SqlWriterConfig.of().withDialect(dialect));
     implement(w);
     return w.toSqlString().getSql()
       .replaceAll("\\n", " ").replaceAll(";", ";\n").trim();

hoptimator-cli/build.gradle

@@ -3,36 +3,72 @@ plugins {
   id 'java'
   id 'application'
   id 'idea'
+  id 'maven-publish'
+}
+
+configurations {
+  integration.extendsFrom implementation
 }
 
 dependencies {
   implementation project(':hoptimator-planner')
   implementation project(':hoptimator-catalog')
-  implementation project(':hoptimator-flink-iterator')
-  implementation project(':hoptimator-kafka-adapter')
-  implementation project(':hoptimator-mysql-adapter')
+
+  // include adapters for integration tests
+  integration project(':hoptimator-kafka-adapter')
+  integration project(':hoptimator-mysql-adapter')
+  integration libs.flinkCsv
 
   implementation libs.avro 
   implementation libs.sqlline
   implementation libs.slf4jLog4j
   implementation libs.flinkClients
   implementation libs.flinkTableRuntime
   implementation libs.flinkTablePlanner
-  implementation libs.flinkTableApiJavaBridge
-  implementation libs.flinkCsv
   implementation libs.calciteCore
   implementation libs.calciteAvatica
 
   testImplementation libs.junit
   testImplementation libs.assertj
 }
 
-tasks.withType(JavaCompile) {
-  options.compilerArgs << '-Xlint:deprecation'
-  options.compilerArgs << '-Xlint:unchecked'
+publishing {
+  repositories {
+    maven {
+      name 'GitHubPackages'
+      url = 'https://maven.pkg.github.com/linkedin/Hoptimator'
+      credentials {
+        username = System.getenv('GITHUB_ACTOR')
+        password = System.getenv('GITHUB_TOKEN')
+      }
+    }
+    maven {
+      name 'LinkedInJFrog'
+      url 'https://linkedin.jfrog.io/artifactory/hoptimator'
+      credentials {
+        username = System.getenv('JFROG_USERNAME')
+        password = System.getenv('JFROG_API_KEY')
+      }
+    }
+  }
+  publications {
+    maven(MavenPublication) {
+      groupId = 'com.linkedin.hoptimator'
+      artifactId = 'hoptimator-cli'
+      version = System.getenv('VERSION')
+      from components.java
+    }
+  }
+}
+
+java {
+  withJavadocJar()
+  withSourcesJar()
 }
 
 shadowJar {
+  configurations = [project.configurations.integration]
+
   // This is required for Flink and Avatica to play nicely
   relocate 'com.google', 'org.apache.flink.calcite.shaded.com.google'
 

hoptimator-cli/run.sh

@@ -1,13 +1,9 @@
 #!/bin/bash
 
-JAR="./hoptimator-cli-all.jar"
-
-if [[ -f "$JAR" ]]; then
-  java \
-    --add-opens java.base/java.lang=ALL-UNNAMED \
-    --add-opens java.base/java.util=ALL-UNNAMED \
-    --add-opens java.base/java.time=ALL-UNNAMED \
-    -jar $JAR --verbose=true "$@"
-else
-  echo "jar file not found; maybe forgot to build?"
-fi
+java \
+  --add-opens java.base/java.lang=ALL-UNNAMED \
+  --add-opens java.base/java.util=ALL-UNNAMED \
+  --add-opens java.base/java.time=ALL-UNNAMED \
+  -classpath "/opt/plugins/*/lib/*:./hoptimator-cli-all.jar" \
+  $JAVA_OPTS \
+  com.linkedin.hoptimator.HoptimatorCliApp --verbose=true -nn hoptimator "$@"