diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f19880f0..15f4bb911 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## 6.4.2 (09/19/2022) + +### Bug Fixes: + +- [#870](https://github.com/greenplum-db/pxf/pull/870) Relax the requirement that C string length matches Java string length + +## 6.4.1 (09/16/2022) + +### Bug Fixes: + +- [#858](https://github.com/greenplum-db/pxf/pull/858) Add JsonProtocolHandler to use HdfsFileFragmenter for multi-line JSON + ## 6.4.0 (08/15/2022) ### Enhancements: diff --git a/automation/pom.xml b/automation/pom.xml index a87397484..2b4fba4f2 100644 --- a/automation/pom.xml +++ b/automation/pom.xml @@ -231,12 +231,6 @@ 1.10.11 - - org.jsoup - jsoup - 1.14.2 - - com.amazonaws diff --git a/automation/src/main/java/org/greenplum/pxf/automation/components/cluster/PhdCluster.java b/automation/src/main/java/org/greenplum/pxf/automation/components/cluster/PhdCluster.java index c5e4b3be8..c67e4347b 100755 --- a/automation/src/main/java/org/greenplum/pxf/automation/components/cluster/PhdCluster.java +++ b/automation/src/main/java/org/greenplum/pxf/automation/components/cluster/PhdCluster.java @@ -79,7 +79,8 @@ public void init() throws Exception { public void addPathToPxfClassPath(String path) throws Exception { String content = "export PXF_LOADER_PATH=file:" + path; // path to local fetch pxf class file - File pathToLocalClassPathFile = new File(getPxfBase() + "/conf", getPxfConfigurationFile()); + + File pathToLocalClassPathFile = new File(getPathToLocalPxfConfDirectory(), getPxfConfigurationFile()); ReportUtils.report(report, getClass(), "Add " + content + " to PXF class path (" + pathToLocalClassPathFile.getAbsolutePath() + ")"); // read file content String pxfClasspathContent = new String(Files.readAllBytes(Paths.get(pathToLocalClassPathFile.getAbsolutePath()))); diff --git a/automation/src/main/java/org/greenplum/pxf/automation/components/cluster/installer/nodes/InstallationNode.java b/automation/src/main/java/org/greenplum/pxf/automation/components/cluster/installer/nodes/InstallationNode.java deleted file mode 100755 index 6b22ab132..000000000 --- a/automation/src/main/java/org/greenplum/pxf/automation/components/cluster/installer/nodes/InstallationNode.java +++ /dev/null @@ -1,93 +0,0 @@ -package org.greenplum.pxf.automation.components.cluster.installer.nodes; - -import java.io.IOException; -import java.util.Collections; -import java.util.Comparator; - -import org.apache.commons.configuration.ConfigurationException; -import org.apache.commons.lang.StringUtils; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - -import org.greenplum.pxf.automation.components.common.cli.ShellCommandErrorException; - -/** - * Represents the node that running the installation of the cluster - */ -public class InstallationNode extends Node { - // server for downloading the required builds - private String downloadServer = "http://dist.dh.greenplum.com/dist/PHD/testing/"; - // path to resources in the installation node - private String resourceDirectory = "src/test/resources/templates/"; - - /** - * Get the latest build for given buildName - * - * @param buildName required build type - * @return latest build from given buildName - * @throws IOException - * @throws ShellCommandErrorException - * @throws ConfigurationException - */ - private String getLatestBuilds(final String buildName) throws IOException, ShellCommandErrorException, ConfigurationException { - // connect to server and get list of all files - Document doc = Jsoup.connect(downloadServer).get(); - // select only the match files according to buildName - Elements el = doc.select("a[href]:matches(" + buildName + "-\\d+)"); - - // sort files elements - Collections.sort(el, new Comparator() { - @Override - public int compare(Element e1, Element e2) { - // leave only the build number X (1.2.0.1-X) and compare it as integer - return Integer.valueOf(e1.text().replaceAll(buildName + "-", "").replaceAll(".tar.gz", "")).compareTo(Integer.valueOf(e2.text() - .replaceAll(buildName + "-", "") - .replaceAll(".tar.gz", ""))); - } - }); - - // return latest build name - return el.last().text().replaceAll(".tar.gz", ""); - } - - /** - * Get the required version for given buildPattern: if a fixed version is given than return it, - * "--" mean that the latest from the given version will be returned. - * - * @param buildPattern required build and version - * @return required build according to given buildPattern - * @throws ConfigurationException - * @throws IOException - * @throws ShellCommandErrorException - */ - public String getRequiredVersion(String buildPattern) throws ConfigurationException, IOException, ShellCommandErrorException { - // if buildPattern is empty return null - if (!StringUtils.isEmpty(buildPattern)) { - // if ends with "-"return the latest from required version, else return fixed build - if (buildPattern.endsWith("-")) { - return getLatestBuilds(buildPattern.substring(0, buildPattern.length() - 1)); - } else { - return buildPattern; - } - } - return null; - } - - public String getDownloadServer() { - return downloadServer; - } - - public void setDownloadServer(String downloadServer) { - this.downloadServer = downloadServer; - } - - public String getResourceDirectory() { - return resourceDirectory; - } - - public void setResourceDirectory(String resourceDirectory) { - this.resourceDirectory = resourceDirectory; - } -} \ No newline at end of file diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/orc/OrcReadTest.java b/automation/src/test/java/org/greenplum/pxf/automation/features/orc/OrcReadTest.java index 1e0feed3f..de41d36bf 100644 --- a/automation/src/test/java/org/greenplum/pxf/automation/features/orc/OrcReadTest.java +++ b/automation/src/test/java/org/greenplum/pxf/automation/features/orc/OrcReadTest.java @@ -13,6 +13,7 @@ public class OrcReadTest extends BaseFeature { private static final String ORC_PRIMITIVE_TYPES_UNORDERED_SUBSET = "orc_types_unordered_subset.orc"; private static final String ORC_LIST_TYPES = "orc_list_types.orc"; private static final String ORC_MULTIDIM_LIST_TYPES = "orc_multidim_list_types.orc"; + private static final String ORC_NULL_IN_STRING = "orc_null_in_string.orc"; private static final String[] ORC_TABLE_COLUMNS = { "id integer", @@ -73,6 +74,12 @@ public class OrcReadTest extends BaseFeature { "varchar_arr text[]" }; + private static final String[] ORC_NULL_IN_STRING_COLUMNS = new String[]{ + "id int", + "context text", + "value text" + }; + private String hdfsPath; private ProtocolEnum protocol; @@ -87,6 +94,7 @@ public void beforeClass() throws Exception { hdfs.copyFromLocal(resourcePath + ORC_PRIMITIVE_TYPES_UNORDERED_SUBSET, hdfsPath + ORC_PRIMITIVE_TYPES_UNORDERED_SUBSET); hdfs.copyFromLocal(resourcePath + ORC_LIST_TYPES, hdfsPath + ORC_LIST_TYPES); hdfs.copyFromLocal(resourcePath + ORC_MULTIDIM_LIST_TYPES, hdfsPath + ORC_MULTIDIM_LIST_TYPES); + hdfs.copyFromLocal(resourcePath + ORC_NULL_IN_STRING, hdfsPath + ORC_NULL_IN_STRING); prepareReadableExternalTable(PXF_ORC_TABLE, ORC_TABLE_COLUMNS, hdfsPath + ORC_PRIMITIVE_TYPES); } @@ -146,6 +154,12 @@ public void orcReadMultiDimensionalLists() throws Exception { runTincTest("pxf.features.orc.read.multidim_list_types.runTest"); } + @Test(groups = {"features", "gpdb", "security", "hcfs"}) + public void orcReadStringsContainingNullByte() throws Exception { + prepareReadableExternalTable("pxf_orc_null_in_string", ORC_NULL_IN_STRING_COLUMNS, hdfsPath + ORC_NULL_IN_STRING); + runTincTest("pxf.features.orc.read.null_in_string.runTest"); + } + private void prepareReadableExternalTable(String name, String[] fields, String path) throws Exception { prepareReadableExternalTable(name, fields, path, false); } diff --git a/automation/src/test/resources/data/orc/README.md b/automation/src/test/resources/data/orc/README.md index 62e5ceb60..0b143878e 100644 --- a/automation/src/test/resources/data/orc/README.md +++ b/automation/src/test/resources/data/orc/README.md @@ -103,3 +103,10 @@ If desired, you can copy down the CSV file by running the following command: mv "./orc_multidim_list_types/000000_0" "./${CSV_FILENAME}" ``` +## Generate the orc_null_in_string.orc file + +To generate this file, you will need to download a copy of the ORC Tools uber jar: + +```shell +java -jar orc-tools-1.8.0-uber.jar convert --schema 'struct' --output orc_null_in_string.orc orc_null_in_string.json +``` diff --git a/automation/src/test/resources/data/orc/orc_null_in_string.json b/automation/src/test/resources/data/orc/orc_null_in_string.json new file mode 100644 index 000000000..e918aad63 --- /dev/null +++ b/automation/src/test/resources/data/orc/orc_null_in_string.json @@ -0,0 +1,6 @@ +{"id": 1, "context": "simple string", "value": "hello"} +{"id": 2, "context": "simple string with space", "value": "hello world"} +{"id": 3, "context": "simple string with double quote", "value": "hello \"world\""} +{"id": 4, "context": "NUL-byte in middle of string", "value": "hello\u0000world"} +{"id": 5, "context": "NUL-byte at the beginning of string", "value": "\u0000hello world"} +{"id": 6, "context": "NUL-byte at the end of string", "value": "hello world\u0000"} diff --git a/automation/src/test/resources/data/orc/orc_null_in_string.orc b/automation/src/test/resources/data/orc/orc_null_in_string.orc new file mode 100644 index 000000000..3cc1e6aed Binary files /dev/null and b/automation/src/test/resources/data/orc/orc_null_in_string.orc differ diff --git a/automation/tincrepo/main/pxf/features/orc/read/null_in_string/__init__.py b/automation/tincrepo/main/pxf/features/orc/read/null_in_string/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/automation/tincrepo/main/pxf/features/orc/read/null_in_string/expected/query01.ans b/automation/tincrepo/main/pxf/features/orc/read/null_in_string/expected/query01.ans new file mode 100644 index 000000000..bbfd641ae --- /dev/null +++ b/automation/tincrepo/main/pxf/features/orc/read/null_in_string/expected/query01.ans @@ -0,0 +1,13 @@ +-- start_ignore +-- end_ignore +-- @description query01 for reading strings contain NUL-byte from ORC files +SELECT * FROM pxf_orc_null_in_string ORDER BY id; + id | context | value +----+-------------------------------------+--------------- + 1 | simple string | hello + 2 | simple string with space | hello world + 3 | simple string with double quote | hello "world" + 4 | NUL-byte in middle of string | hello + 5 | NUL-byte at the beginning of string | + 6 | NUL-byte at the end of string | hello world +(6 rows) \ No newline at end of file diff --git a/automation/tincrepo/main/pxf/features/orc/read/null_in_string/runTest.py b/automation/tincrepo/main/pxf/features/orc/read/null_in_string/runTest.py new file mode 100644 index 000000000..916deefe9 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/orc/read/null_in_string/runTest.py @@ -0,0 +1,11 @@ +from mpp.models import SQLConcurrencyTestCase + +class OrcNullInString(SQLConcurrencyTestCase): + """ + @db_name pxfautomation + @concurrency 1 + @gpdiff True + """ + sql_dir = 'sql' + ans_dir = 'expected' + out_dir = 'output' diff --git a/automation/tincrepo/main/pxf/features/orc/read/null_in_string/sql/query01.sql b/automation/tincrepo/main/pxf/features/orc/read/null_in_string/sql/query01.sql new file mode 100644 index 000000000..52e65cb34 --- /dev/null +++ b/automation/tincrepo/main/pxf/features/orc/read/null_in_string/sql/query01.sql @@ -0,0 +1,3 @@ +-- @description query01 for reading strings contain NUL-byte from ORC files + +SELECT * FROM pxf_orc_null_in_string ORDER BY id; diff --git a/concourse/docker/diagram/images.dot b/concourse/docker/diagram/images.dot index 70dc969fa..463e1e7a6 100644 --- a/concourse/docker/diagram/images.dot +++ b/concourse/docker/diagram/images.dot @@ -25,6 +25,7 @@ digraph pxf_container_image_flow { style=dashed node [shape=box3d fillcolor="#2aa198" style=filled fontcolor=white] gp6_rhel8_latest[label="gpdb6-rhel8-test:latest"] + gp7_rhel8_latest[label="gpdb7-rhel8-test:latest"] } # PXF Cloudbuild & Dockerfiles @@ -39,6 +40,7 @@ digraph pxf_container_image_flow { gp6_ubuntu18_dockerfile[label="gpdb6/ubuntu18.04"] gp6_oel7_dockerfile[label="gpdb6/oel7"] gp7_centos7_dockerfile[label="gpdb7/centos7"] + gp7_rhel8_dockerfile[label="gpdb7/rhel8"] gp7_ubuntu18_dockerfile[label="gpdb7/ubuntu18.04"] } @@ -69,6 +71,7 @@ digraph pxf_container_image_flow { gp6_ubuntu18_pxf_sha[label="gpdb6-ubuntu18.04-test-pxf:$COMMIT_SHA"] gp6_oel7_pxf_sha[label="gpdb6-oel7-test-pxf:$COMMIT_SHA"] gp7_centos7_pxf_sha[label="gpdb7-centos7-test-pxf:$COMMIT_SHA"] + gp7_rhel8_pxf_sha[label="gpdb7-rhel8-test-pxf:$COMMIT_SHA"] gp7_ubuntu18_pxf_sha[label="gpdb7-ubuntu18.04-test-pxf:$COMMIT_SHA"] gp5_centos7_pxf_latest[label="gpdb5-centos7-test-pxf:latest" style=filled fillcolor="#6c71c4" fontcolor=white] @@ -77,6 +80,7 @@ digraph pxf_container_image_flow { gp6_ubuntu18_pxf_latest[label="gpdb6-ubuntu18.04-test-pxf:latest" style=filled fillcolor="#6c71c4" fontcolor=white] gp6_oel7_pxf_latest[label="gpdb6-oel7-test-pxf:latest" style=filled fillcolor="#6c71c4" fontcolor=white] gp7_centos7_pxf_latest[label="gpdb7-centos7-test-pxf:latest" style=filled fillcolor="#6c71c4" fontcolor=white] + gp7_rhel8_pxf_latest[label="gpdb7-rhel8-test-pxf:latest" style=filled fillcolor="#6c71c4" fontcolor=white] gp7_ubuntu18_pxf_latest[label="gpdb7-ubuntu18.04-test-pxf:latest" style=filled fillcolor="#6c71c4" fontcolor=white] } @@ -159,6 +163,10 @@ digraph pxf_container_image_flow { gp7_centos7_dockerfile -> gp7_centos7_pxf_sha[label="CloudBuild"] gp7_centos7_pxf_sha -> gp7_centos7_pxf_latest[label="tag (concourse pipeline)"] + gp7_rhel8_latest -> gp7_rhel8_dockerfile + gp7_rhel8_dockerfile -> gp7_rhel8_pxf_sha[label="CloudBuild"] + gp7_rhel8_pxf_sha -> gp7_rhel8_pxf_latest[label="tag (concourse pipeline)"] + gp7_ubuntu18_latest -> gp7_ubuntu18_dockerfile gp7_ubuntu18_dockerfile -> gp7_ubuntu18_pxf_sha[label="CloudBuild"] gp7_ubuntu18_pxf_sha -> gp7_ubuntu18_pxf_latest[label="tag (concourse pipeline)"] @@ -203,6 +211,10 @@ digraph pxf_container_image_flow { gp7_centos7_pxf_latest -> build gp7_centos7_pxf_latest -> pr + gp7_rhel8_pxf_latest -> certification + gp7_rhel8_pxf_latest -> build + gp7_rhel8_pxf_latest -> pr + gp7_ubuntu18_pxf_latest -> build gp7_ubuntu18_pxf_latest -> pr diff --git a/concourse/docker/diagram/images.svg b/concourse/docker/diagram/images.svg index bceda5558..29746f802 100644 --- a/concourse/docker/diagram/images.svg +++ b/concourse/docker/diagram/images.svg @@ -1,742 +1,809 @@ - - + pxf_container_image_flow - + cluster_dockerhub - -Official DockerHub + +Official DockerHub cluster_gcr_images GP RelEng Images (gcr.io/data-gpdb-public-images) - -cluster_gcr_images_private - -GP RelEng Images (gcr.io/data-gpdb-private-images) - cluster_pxf_dev_base - -pxf-dev-base/cloudbuild.yaml + +pxf-dev-base/cloudbuild.yaml cluster_rpmrebuild - -rpmrebuild/cloudbuild.yaml + +rpmrebuild/cloudbuild.yaml cluster_gcr_data_gpdb_ud - -gcr.io/data-gpdb-ud + +gcr.io/data-gpdb-ud cluster_gpdb_pxf_dev - -gpdb-pxf-dev + +gpdb-pxf-dev cluster_pipelines - -Pipelines + +Pipelines cluster_local_use_only - -For local development use + +For local development use cluster_pxf_dev_server - -pxf-dev-server/cloudbuild.yaml + +pxf-dev-server/cloudbuild.yaml cluster_mapr - -mapr/cloudbuild.yaml + +mapr/cloudbuild.yaml cluster_gcr_data_gpdb_ud_mapr - -MapR Images (gcr.io/data-gpdb-ud) + +MapR Images (gcr.io/data-gpdb-ud) cluster_gcr_data_gpdb_ud_hdp2 - -HDP2 (gcr.io/data-gpdb-ud) + +HDP2 (gcr.io/data-gpdb-ud) + + +cluster_gcr_images_private + +GP RelEng Images (gcr.io/data-gpdb-private-images) centos7 - - - - -centos:7 + + + + +centos:7 - + rpm_docker_centos7 - - - -centos/Dockerfile + + + +centos/Dockerfile - + centos7->rpm_docker_centos7 - - + + gp5_centos7_latest - - - + + + centos-gpdb-dev:7-gcc6.2-llvm3.7 - + gp5_centos7_dockerfile - - - -gpdb5/centos7 + + + +gpdb5/centos7 gp5_centos7_latest->gp5_centos7_dockerfile - - + + gp6_centos7_latest - - - + + + gpdb6-centos7-test:latest - + gp6_centos7_dockerfile - - - -gpdb6/centos7 + + + +gpdb6/centos7 gp6_centos7_latest->gp6_centos7_dockerfile - - + + gp6_ubuntu18_latest - - - + + + gpdb6-ubuntu18.04-test:latest - + gp6_ubuntu18_dockerfile - - - -gpdb6/ubuntu18.04 + + + +gpdb6/ubuntu18.04 gp6_ubuntu18_latest->gp6_ubuntu18_dockerfile - - + + gp6_oel7_latest - - - + + + gpdb6-oel7-test:latest - + gp6_oel7_dockerfile - - - -gpdb6/oel7 + + + +gpdb6/oel7 gp6_oel7_latest->gp6_oel7_dockerfile - - + + gp7_centos7_latest - - - + + + gpdb7-centos7-test:latest - + gp7_centos7_dockerfile - - - -gpdb7/centos7 + + + +gpdb7/centos7 gp7_centos7_latest->gp7_centos7_dockerfile - - + + gp7_ubuntu18_latest - - - + + + gpdb7-ubuntu18.04-test:latest - + gp7_ubuntu18_dockerfile - - - -gpdb7/ubuntu18.04 + + + +gpdb7/ubuntu18.04 - + gp7_ubuntu18_latest->gp7_ubuntu18_dockerfile - - + + gp6_rhel8_latest - - - - -gpdb6-rhel8-test:latest + + + + +gpdb6-rhel8-test:latest - + gp6_rhel8_dockerfile - - - -gpdb6/rhel8 + + + +gpdb6/rhel8 gp6_rhel8_latest->gp6_rhel8_dockerfile - - + + - + rpm_docker_rhel8 - - - -rhel/Dockerfile + + + +rhel/Dockerfile - + gp6_rhel8_latest->rpm_docker_rhel8 - - + + + + + +gp7_rhel8_latest + + + + +gpdb7-rhel8-test:latest + + + +gp7_rhel8_dockerfile + + + +gpdb7/rhel8 + + + +gp7_rhel8_latest->gp7_rhel8_dockerfile + + - + gp5_centos7_pxf_sha - -gpdb5-centos7-test-pxf:$COMMIT_SHA + +gpdb5-centos7-test-pxf:$COMMIT_SHA gp5_centos7_dockerfile->gp5_centos7_pxf_sha - - -CloudBuild + + +CloudBuild - + gp6_centos7_pxf_sha - -gpdb6-centos7-test-pxf:$COMMIT_SHA + +gpdb6-centos7-test-pxf:$COMMIT_SHA gp6_centos7_dockerfile->gp6_centos7_pxf_sha - - -CloudBuild + + +CloudBuild - + gp6_rhel8_pxf_sha - -gpdb6-rhel8-test-pxf:$COMMIT_SHA + +gpdb6-rhel8-test-pxf:$COMMIT_SHA gp6_rhel8_dockerfile->gp6_rhel8_pxf_sha - - -CloudBuild + + +CloudBuild - + gp6_ubuntu18_pxf_sha - -gpdb6-ubuntu18.04-test-pxf:$COMMIT_SHA + +gpdb6-ubuntu18.04-test-pxf:$COMMIT_SHA gp6_ubuntu18_dockerfile->gp6_ubuntu18_pxf_sha - - -CloudBuild + + +CloudBuild - + gp6_oel7_pxf_sha - -gpdb6-oel7-test-pxf:$COMMIT_SHA + +gpdb6-oel7-test-pxf:$COMMIT_SHA gp6_oel7_dockerfile->gp6_oel7_pxf_sha - - -CloudBuild + + +CloudBuild - + gp7_centos7_pxf_sha - -gpdb7-centos7-test-pxf:$COMMIT_SHA + +gpdb7-centos7-test-pxf:$COMMIT_SHA gp7_centos7_dockerfile->gp7_centos7_pxf_sha - - -CloudBuild + + +CloudBuild + + + +gp7_rhel8_pxf_sha + +gpdb7-rhel8-test-pxf:$COMMIT_SHA + + + +gp7_rhel8_dockerfile->gp7_rhel8_pxf_sha + + +CloudBuild - + gp7_ubuntu18_pxf_sha - -gpdb7-ubuntu18.04-test-pxf:$COMMIT_SHA + +gpdb7-ubuntu18.04-test-pxf:$COMMIT_SHA - + gp7_ubuntu18_dockerfile->gp7_ubuntu18_pxf_sha - - -CloudBuild + + +CloudBuild - + rpm_centos7_latest - -rpmrebuild-centos7:latest + +rpmrebuild-centos7:latest - + rpm_docker_centos7->rpm_centos7_latest - - + + - + rpm_rhel8_latest - -rpmrebuild-rhel8:latest + +rpmrebuild-rhel8:latest - + rpm_docker_rhel8->rpm_rhel8_latest - - + + - + gp5_centos7_pxf_latest - -gpdb5-centos7-test-pxf:latest + +gpdb5-centos7-test-pxf:latest gp5_centos7_pxf_sha->gp5_centos7_pxf_latest - - -tag (concourse pipeline) + + +tag (concourse pipeline) - + gp6_centos7_pxf_latest - -gpdb6-centos7-test-pxf:latest + +gpdb6-centos7-test-pxf:latest gp6_centos7_pxf_sha->gp6_centos7_pxf_latest - - -tag (concourse pipeline) + + +tag (concourse pipeline) - + gp6_rhel8_pxf_latest - -gpdb6-rhel8-test-pxf:latest + +gpdb6-rhel8-test-pxf:latest gp6_rhel8_pxf_sha->gp6_rhel8_pxf_latest - - -tag (concourse pipeline) + + +tag (concourse pipeline) - + gp6_ubuntu18_pxf_latest - -gpdb6-ubuntu18.04-test-pxf:latest + +gpdb6-ubuntu18.04-test-pxf:latest gp6_ubuntu18_pxf_sha->gp6_ubuntu18_pxf_latest - - -tag (concourse pipeline) + + +tag (concourse pipeline) - + gp6_oel7_pxf_latest - -gpdb6-oel7-test-pxf:latest + +gpdb6-oel7-test-pxf:latest gp6_oel7_pxf_sha->gp6_oel7_pxf_latest - - -tag (concourse pipeline) + + +tag (concourse pipeline) - + gp7_centos7_pxf_latest - -gpdb7-centos7-test-pxf:latest + +gpdb7-centos7-test-pxf:latest gp7_centos7_pxf_sha->gp7_centos7_pxf_latest - - -tag (concourse pipeline) + + +tag (concourse pipeline) + + + +gp7_rhel8_pxf_latest + +gpdb7-rhel8-test-pxf:latest + + + +gp7_rhel8_pxf_sha->gp7_rhel8_pxf_latest + + +tag (concourse pipeline) - + gp7_ubuntu18_pxf_latest - -gpdb7-ubuntu18.04-test-pxf:latest + +gpdb7-ubuntu18.04-test-pxf:latest - + gp7_ubuntu18_pxf_sha->gp7_ubuntu18_pxf_latest - - -tag (concourse pipeline) + + +tag (concourse pipeline) - + certification - -certification + +certification - + gp5_centos7_pxf_latest->certification - - + + - + build - -pxf-build + +pxf-build - + gp5_centos7_pxf_latest->build - - + + - + pr - -pxf_pr_pipeline + +pxf_pr_pipeline - + gp5_centos7_pxf_latest->pr - - + + - + mapr_dockerfile - - - -Dockerfile + + + +Dockerfile - + gp5_centos7_pxf_latest->mapr_dockerfile - - + + - + gp6_centos7_pxf_latest->certification - - + + - + perf - -perf + +perf - + gp6_centos7_pxf_latest->perf - - + + - + longevity - -longevity + +longevity - + gp6_centos7_pxf_latest->longevity - - + + - + gp6_centos7_pxf_latest->build - - + + - + gp6_centos7_pxf_latest->pr - - + + - + server_dockerfile - - - -Dockerfile + + + +Dockerfile - + gp6_centos7_pxf_latest->server_dockerfile - - + + - + gp6_centos7_pxf_latest->mapr_dockerfile - - + + - + gp6_rhel8_pxf_latest->certification - - + + - + gp6_rhel8_pxf_latest->perf - - + + - + gp6_rhel8_pxf_latest->build - - + + - + gp6_rhel8_pxf_latest->pr - - + + - + gp6_ubuntu18_pxf_latest->certification - - + + - + gp6_ubuntu18_pxf_latest->build - - + + - + gp6_ubuntu18_pxf_latest->pr - - + + - + gp6_oel7_pxf_latest->build - - + + - + gp6_oel7_pxf_latest->pr - - + + - + gp7_centos7_pxf_latest->build - - + + - + gp7_centos7_pxf_latest->pr - - + + + + + +gp7_rhel8_pxf_latest->certification + + + + + +gp7_rhel8_pxf_latest->build + + + + + +gp7_rhel8_pxf_latest->pr + + - + gp7_ubuntu18_pxf_latest->build - - + + - + gp7_ubuntu18_pxf_latest->pr - - + + - + rpm_centos7_latest->build - - + + - + rpm_rhel8_latest->build - - + + - + gp6_centos7_pxf_hdp2_sha - -gpdb-pxf-dev/gpdb6-centos7-test-pxf-hdp2:$COMMIT_SHA + +gpdb-pxf-dev/gpdb6-centos7-test-pxf-hdp2:$COMMIT_SHA - + server_dockerfile->gp6_centos7_pxf_hdp2_sha - - -CloudBuild (add singlecluster, build deps, & automation deps) + + +CloudBuild (add singlecluster, build deps, & automation deps) - + gp6_centos7_pxf_mapr_sha - -gpdb-pxf-dev/gpdb6-centos7-test-pxf-mapr:$COMMIT_SHA + +gpdb-pxf-dev/gpdb6-centos7-test-pxf-mapr:$COMMIT_SHA - + mapr_dockerfile->gp6_centos7_pxf_mapr_sha - - -CloudBuild (install MapR) + + +CloudBuild (install MapR) - + gp6_centos7_pxf_mapr_latest - -gpdb-pxf-dev/gpdb6-centos7-test-pxf-mapr:latest + +gpdb-pxf-dev/gpdb6-centos7-test-pxf-mapr:latest - + gp6_centos7_pxf_mapr_sha->gp6_centos7_pxf_mapr_latest - - -tag (concourse pipeline) + + +tag (concourse pipeline) - + gp6_centos7_pxf_mapr_latest->build - - -Conditionally added based off mapr variable + + +Conditionally added based off mapr variable - + gp6_centos7_pxf_hdp2_latest - -gpdb-pxf-dev/gpdb6-centos7-test-pxf-hdp2 + +gpdb-pxf-dev/gpdb6-centos7-test-pxf-hdp2 - + gp6_centos7_pxf_hdp2_sha->gp6_centos7_pxf_hdp2_latest - - + + diff --git a/concourse/docker/pxf-dev-base/README.md b/concourse/docker/pxf-dev-base/README.md index cff03a414..5058542c0 100644 --- a/concourse/docker/pxf-dev-base/README.md +++ b/concourse/docker/pxf-dev-base/README.md @@ -116,6 +116,8 @@ following command to build the image: ### Docker gpdb7-centos7-test-pxf-image image +TODO: ===> remove this if Greenplum 7 will not be supported on Centos7 <=== + Build this image for Greenplum 7 running on CentOS 7. Run the following command to build the image: @@ -129,6 +131,20 @@ command to build the image: . popd +### Docker gpdb7-rhel8-test-pxf-image image + +Build this image for Greenplum 7 running on Rhel 8. Run the following +command to build the image: + + pushd ~/workspace/pxf/concourse/docker/pxf-dev-base/ + docker build \ + --build-arg=BASE_IMAGE=gcr.io/data-gpdb-private-images/gpdb7-rhel8-test:latest \ + --build-arg=GINKGO_VERSION=${GINKGO_VERSION} \ + --tag=gpdb7-rhel8-test-pxf \ + -f ~/workspace/pxf/concourse/docker/pxf-dev-base/gpdb7/rhel8/Dockerfile \ + . + popd + ### Docker gpdb7-ubuntu18.04-test-pxf-image image Build this image for Greenplum 7 running on Ubuntu 18.04. Run the following diff --git a/concourse/docker/pxf-dev-base/cloudbuild.yaml b/concourse/docker/pxf-dev-base/cloudbuild.yaml index 818c27b39..c13cbfa90 100644 --- a/concourse/docker/pxf-dev-base/cloudbuild.yaml +++ b/concourse/docker/pxf-dev-base/cloudbuild.yaml @@ -182,6 +182,32 @@ steps: waitFor: - gpdb7-centos7-test-pxf-image-cache +# Greenplum 7 Rhel 8 Image +- name: 'gcr.io/cloud-builders/docker' + id: gpdb7-rhel8-test-pxf-image-cache + entrypoint: 'bash' + args: + - '-c' + - | + mkdir -p /workspace/build + docker pull gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-rhel8-test-pxf:latest || exit 0 + waitFor: ['-'] + +- name: 'gcr.io/cloud-builders/docker' + id: gpdb7-rhel8-test-pxf-image + args: + - 'build' + - '--build-arg=BASE_IMAGE=${_PRIVATE_BASE_IMAGE_REPOSITORY}/gpdb7-rhel8-test:latest' + - '--build-arg=GINKGO_VERSION=${_GINKGO_VERSION}' + - '--tag=gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-rhel8-test-pxf:$COMMIT_SHA' + - '--cache-from' + - 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-rhel8-test-pxf:latest' + - '-f' + - 'concourse/docker/pxf-dev-base/gpdb7/rhel8/Dockerfile' + - '/workspace/build/' + waitFor: + - gpdb7-rhel8-test-pxf-image-cache + # Greenplum 7 Ubuntu 18.04 Image - name: 'gcr.io/cloud-builders/docker' id: gpdb7-ubuntu18.04-test-pxf-image-cache @@ -221,4 +247,5 @@ images: - 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-ubuntu18.04-test-pxf:$COMMIT_SHA' - 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-oel7-test-pxf:$COMMIT_SHA' - 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-centos7-test-pxf:$COMMIT_SHA' +- 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-rhel8-test-pxf:$COMMIT_SHA' - 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-ubuntu18.04-test-pxf:$COMMIT_SHA' diff --git a/concourse/docker/pxf-dev-base/gpdb7/centos7/Dockerfile b/concourse/docker/pxf-dev-base/gpdb7/centos7/Dockerfile index c30e463cd..007454842 100644 --- a/concourse/docker/pxf-dev-base/gpdb7/centos7/Dockerfile +++ b/concourse/docker/pxf-dev-base/gpdb7/centos7/Dockerfile @@ -20,9 +20,19 @@ RUN useradd -s /sbin/nologin -d /opt/minio minio \ && chmod +x /opt/minio/bin/minio \ && chown -R minio:minio /opt/minio -# install java 8 and 11 and dependencies that are missing on the base images -# java 8 is required to run hadoop. Do not use java 8 to build / run PXF -RUN yum install -y rpm-build sudo java-1.8.0-openjdk java-11-openjdk-devel jq +ARG MAVEN_VERSION=3.6.3 +ARG USER_HOME_DIR="/root" +ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries + +# install dependencies that are missing on the base images +RUN curl -fsSL -o /tmp/apache-maven.tar.gz ${BASE_URL}/apache-maven-${MAVEN_VERSION}-bin.tar.gz \ + && mkdir -p /usr/share/maven \ + && tar -xzf /tmp/apache-maven.tar.gz -C /usr/share/maven --strip-components=1 \ + && rm -f /tmp/apache-maven.tar.gz \ + && ln -s /usr/share/maven/bin/mvn /usr/bin/mvn \ + && yum install -y rpm-build python-devel jq sudo java-1.8.0-openjdk-devel java-11-openjdk-devel && yum clean all \ + && cd /tmp && /usr/bin/pip install --upgrade pip==20.3.3 \ + && /usr/bin/pip install paramiko --no-cache-dir # create user gpadmin since GPDB cannot run under root RUN ssh-keygen -t rsa -N "" -f /root/.ssh/id_rsa \ @@ -61,7 +71,7 @@ RUN ssh-keygen -t rsa -N "" -f /root/.ssh/id_rsa \ && echo >> ~gpadmin/.pxfrc 'export GPHD_ROOT=/singlecluster' \ && echo >> ~gpadmin/.pxfrc 'export PXF_HOME=/usr/local/pxf-gp7' \ && echo >> ~gpadmin/.pxfrc 'export PXF_CONF=/home/gpadmin/pxf' \ - && echo >> ~gpadmin/.pxfrc 'export JAVA_HOME=/usr/lib/jvm/java-11' \ + && echo >> ~gpadmin/.pxfrc 'export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk' \ && echo >> ~gpadmin/.pxfrc 'export PATH=${GPHOME}/bin:${PXF_HOME}/bin:${GOPATH}/bin:/usr/local/go/bin:$JAVA_HOME/bin:$PATH' \ && ln -s ~gpadmin/.pxfrc ~root \ && echo >> ~gpadmin/.bashrc 'source ~/.pxfrc' \ diff --git a/concourse/docker/pxf-dev-base/gpdb7/rhel8/Dockerfile b/concourse/docker/pxf-dev-base/gpdb7/rhel8/Dockerfile new file mode 100644 index 000000000..6217f14cc --- /dev/null +++ b/concourse/docker/pxf-dev-base/gpdb7/rhel8/Dockerfile @@ -0,0 +1,71 @@ +ARG BASE_IMAGE=gcr.io/data-gpdb-private-images/gpdb7-rhel8-test:latest + +FROM ${BASE_IMAGE} + +ARG GINKGO_VERSION + +# install Go utilities +RUN GOPATH=/opt/go /usr/local/go/bin/go install github.com/onsi/ginkgo/ginkgo@v${GINKGO_VERSION} + +# add minio software +RUN useradd -s /sbin/nologin -d /opt/minio minio \ + && mkdir -p /opt/minio/bin \ + && chmod a+rx /opt/minio \ + && mkdir /opt/minio/data \ + && wget -q https://dl.minio.io/server/minio/release/linux-amd64/minio -O /opt/minio/bin/minio \ + && chmod +x /opt/minio/bin/minio \ + && chown -R minio:minio /opt/minio + +ARG MAVEN_VERSION=3.6.3 +ARG USER_HOME_DIR="/root" +ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries + +# install dependencies that are missing on the base images +RUN curl -fsSL -o /tmp/apache-maven.tar.gz ${BASE_URL}/apache-maven-${MAVEN_VERSION}-bin.tar.gz \ + && mkdir -p /usr/share/maven \ + && tar -xzf /tmp/apache-maven.tar.gz -C /usr/share/maven --strip-components=1 \ + && rm -f /tmp/apache-maven.tar.gz \ + && ln -s /usr/share/maven/bin/mvn /usr/bin/mvn \ + && /usr/bin/pip2 install paramiko --no-cache-dir + +# create rsa key for the root user +RUN ssh-keygen -t rsa -N "" -f /root/.ssh/id_rsa \ + && cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys \ + && chmod 0600 /root/.ssh/authorized_keys \ + && echo -e "password\npassword" | passwd 2> /dev/null \ + # create rsa key for the server + && ssh-keygen -f /etc/ssh/ssh_host_rsa_key -N '' -t rsa \ + && sed -i -e 's|Defaults requiretty|#Defaults requiretty|' /etc/sudoers \ + && sed -ri 's/UsePAM yes/UsePAM no/g;s/PasswordAuthentication yes/PasswordAuthentication no/g' /etc/ssh/sshd_config \ + && sed -ri 's@^HostKey /etc/ssh/ssh_host_ecdsa_key$@#&@;s@^HostKey /etc/ssh/ssh_host_ed25519_key$@#&@' /etc/ssh/sshd_config \ + # create user gpadmin since GPDB cannot run under root + && groupadd -g 6000 gpadmin && useradd -u 6000 -g 6000 gpadmin \ + && echo "gpadmin ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/gpadmin \ + && groupadd supergroup && usermod -a -G supergroup gpadmin \ + && mkdir /home/gpadmin/.ssh \ + # create an rsa key for the gpadmin user + && ssh-keygen -m PEM -t rsa -f /home/gpadmin/.ssh/id_rsa \ + && cat /home/gpadmin/.ssh/id_rsa.pub >> /home/gpadmin/.ssh/authorized_keys \ + && chmod 0600 /home/gpadmin/.ssh/authorized_keys \ + && echo -e "password\npassword" | passwd gpadmin 2> /dev/null \ + && awk '{print "localhost", $1, $2; print "127.0.0.1", $1, $2}' /etc/ssh/ssh_host_rsa_key.pub >> /home/gpadmin/.ssh/known_hosts \ + && chown -R gpadmin:gpadmin /home/gpadmin/.ssh \ + # configure gpadmin limits + && echo >> /etc/security/limits.d/gpadmin-limits.conf 'gpadmin soft core unlimited' \ + && echo >> /etc/security/limits.d/gpadmin-limits.conf 'gpadmin soft nproc 131072' \ + && echo >> /etc/security/limits.d/gpadmin-limits.conf 'gpadmin soft nofile 65536' \ + # add locale for testing + && localedef -c -i ru_RU -f CP1251 ru_RU.CP1251 \ + # create .pxfrc + && echo >> ~gpadmin/.pxfrc 'export LANG=en_US.UTF-8' \ + && echo >> ~gpadmin/.pxfrc 'export PGPORT=5432' \ + && echo >> ~gpadmin/.pxfrc 'export GOPATH=/opt/go' \ + && echo >> ~gpadmin/.pxfrc 'export GPHOME=$(find /usr/local/ -name greenplum-db* -type d | head -n1)' \ + && echo >> ~gpadmin/.pxfrc 'export LD_LIBRARY_PATH=${GPHOME}/lib:${LD_LIBRARY_PATH-}' \ + && echo >> ~gpadmin/.pxfrc 'export GPHD_ROOT=/singlecluster' \ + && echo >> ~gpadmin/.pxfrc 'export PXF_HOME=/usr/local/pxf-gp7' \ + && echo >> ~gpadmin/.pxfrc 'export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk' \ + && echo >> ~gpadmin/.pxfrc 'export PATH=${GPHOME}/bin:${PXF_HOME}/bin:${GOPATH}/bin:/usr/local/go/bin:$JAVA_HOME/bin:$PATH' \ + && ln -s ~gpadmin/.pxfrc ~root \ + && echo >> ~gpadmin/.bashrc 'source ~/.pxfrc' \ + && chown -R gpadmin:gpadmin ~gpadmin diff --git a/concourse/docker/pxf-dev-base/gpdb7/ubuntu18.04/Dockerfile b/concourse/docker/pxf-dev-base/gpdb7/ubuntu18.04/Dockerfile index 275b9dae7..ec832065c 100644 --- a/concourse/docker/pxf-dev-base/gpdb7/ubuntu18.04/Dockerfile +++ b/concourse/docker/pxf-dev-base/gpdb7/ubuntu18.04/Dockerfile @@ -11,11 +11,22 @@ RUN mkdir -p /tmp/pxf_src/ && cd /tmp \ && tar -C /usr/local -xzf go.tgz && rm go.tgz \ && GOPATH=/opt/go /usr/local/go/bin/go install github.com/onsi/ginkgo/ginkgo@v${GINKGO_VERSION} -# install java 11 and dependencies that are missing on the base images +# install dependencies that are missing on the base images +# install a specific version of perl for tinc + +ARG MAVEN_VERSION=3.6.3 +ARG USER_HOME_DIR="/root" +ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries + RUN apt-get update -y \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y curl sudo openjdk-11-jdk jq \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y python-dev curl sudo jq openjdk-8-jdk openjdk-11-jdk \ && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ + && mkdir -p /usr/share/maven /usr/share/maven/ref \ + && curl -fsSL -o /tmp/apache-maven.tar.gz ${BASE_URL}/apache-maven-${MAVEN_VERSION}-bin.tar.gz \ + && tar -xzf /tmp/apache-maven.tar.gz -C /usr/share/maven --strip-components=1 \ + && rm -f /tmp/apache-maven.tar.gz \ + && ln -s /usr/share/maven/bin/mvn /usr/bin/mvn # create user gpadmin since GPDB cannot run under root RUN locale-gen en_US.UTF-8 \ @@ -38,6 +49,8 @@ RUN locale-gen en_US.UTF-8 \ && echo "gpadmin:password" | chpasswd 2> /dev/null \ && { ssh-keyscan localhost; ssh-keyscan 0.0.0.0; } >> /home/gpadmin/.ssh/known_hosts \ && chown -R gpadmin:gpadmin /home/gpadmin \ + # install dependencies as gpadmin + && su gpadmin -c "pip install paramiko --no-cache-dir" \ # configure gpadmin limits && echo >> /etc/security/limits.d/gpadmin-limits.conf 'gpadmin soft core unlimited' \ && echo >> /etc/security/limits.d/gpadmin-limits.conf 'gpadmin soft nproc 131072' \ @@ -53,7 +66,7 @@ RUN locale-gen en_US.UTF-8 \ && echo >> ~gpadmin/.pxfrc 'export GPHD_ROOT=/singlecluster' \ && echo >> ~gpadmin/.pxfrc 'export PXF_HOME=/usr/local/pxf-gp7' \ && echo >> ~gpadmin/.pxfrc 'export PXF_CONF=/home/gpadmin/pxf' \ - && echo >> ~gpadmin/.pxfrc 'export JAVA_HOME=/usr/lib/jvm/jdk-11' \ + && echo >> ~gpadmin/.pxfrc 'export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64' \ && echo >> ~gpadmin/.pxfrc 'export PATH=${GPHOME}/bin:${PXF_HOME}/bin:${GOPATH}/bin:/usr/local/go/bin:$JAVA_HOME/bin:$PATH' \ && ln -s ~gpadmin/.pxfrc ~root \ && echo >> ~gpadmin/.bashrc 'source ~/.pxfrc' \ diff --git a/concourse/pipelines/cloudbuild_pipeline.yml b/concourse/pipelines/cloudbuild_pipeline.yml index 05f001b9c..39672936d 100644 --- a/concourse/pipelines/cloudbuild_pipeline.yml +++ b/concourse/pipelines/cloudbuild_pipeline.yml @@ -61,6 +61,7 @@ resources: - images/docker/gpdb6/gpdb6-oel7-test - images/docker/gpdb6/gpdb6-rhel8-test - images/docker/gpdb7/gpdb7-centos7-test + - images/docker/gpdb7/gpdb7-rhel8-test ## ====================================================================== ## JOBS @@ -107,7 +108,7 @@ jobs: GOOGLE_CREDENTIALS: ((ud/pxf/secrets/pxf-cloudbuild-service-account-key)) GOOGLE_PROJECT_ID: ((ud/pxf/common/google-project-id)) GOOGLE_ZONE: ((ud/pxf/common/google-zone)) - IMAGE_LIST: "gpdb5-centos7-test-pxf gpdb6-centos7-test-pxf gpdb6-rhel8-test-pxf gpdb6-ubuntu18.04-test-pxf gpdb6-oel7-test-pxf gpdb7-centos7-test-pxf gpdb7-ubuntu18.04-test-pxf" + IMAGE_LIST: "gpdb5-centos7-test-pxf gpdb6-centos7-test-pxf gpdb6-rhel8-test-pxf gpdb6-ubuntu18.04-test-pxf gpdb6-oel7-test-pxf gpdb7-centos7-test-pxf gpdb7-rhel8-test-pxf gpdb7-ubuntu18.04-test-pxf" config: platform: linux inputs: diff --git a/concourse/scripts/get_product_files.bash b/concourse/scripts/get_product_files.bash index 361975f35..4c1e233ac 100755 --- a/concourse/scripts/get_product_files.bash +++ b/concourse/scripts/get_product_files.bash @@ -68,12 +68,18 @@ for ((i = 0; i < ${#product_files[@]}; i++)); do if [[ -z "${id}" ]]; then echo "Did not find '${file}' in product files for GPDB '${gpdb_version}'" - os_regex='^.*rhel8.*$' - if [[ $file =~ ${os_regex} ]]; then - echo "RHEL 8 artifact unavailable for the given GPDB version. Keeping existing rpm: $(find ${product_dirs[$i]}/ -name *rhel8*.rpm)" - continue - fi - exit 1 + + case "${file}" in + *rhel7*) existing_file="$(find ${product_dirs[$i]}/ -name *rhel7*.rpm)" ;; + *rhel8*) existing_file="$(find ${product_dirs[$i]}/ -name *rhel8*.rpm)" ;; + *ubuntu18*) existing_file="$(find ${product_dirs[$i]}/ -name *ubuntu18*.deb)" ;; + *) + echo "Unexpected file: ${file}" + exit 1;; + esac + + echo "Keeping existing file: ${existing_file}" + continue fi echo "Cleaning ${product_dirs[$i]} and downloading ${file} with id ${id} to ${product_dirs[$i]}..." rm -f "${product_dirs[$i]}"/*.{rpm,deb} diff --git a/concourse/settings/pxf-multinode-params.yml b/concourse/settings/pxf-multinode-params.yml index a3e20363d..28034c874 100644 --- a/concourse/settings/pxf-multinode-params.yml +++ b/concourse/settings/pxf-multinode-params.yml @@ -2,6 +2,6 @@ enable-impersonation-multinode: true number_of_gpdb_nodes: 3 tf-bucket-path: clusters-google/ no_of_files: 10000 -pxf-jvm-opts: "-Xmx512m -Xms512m" +pxf-jvm-opts: "-Xmx2g -Xms2g" kerberos-enabled: true dataproc-image-version: "1.4" diff --git a/docs/content/hdfs_orc.html.md.erb b/docs/content/hdfs_orc.html.md.erb index b424063cb..ca52ed232 100644 --- a/docs/content/hdfs_orc.html.md.erb +++ b/docs/content/hdfs_orc.html.md.erb @@ -124,7 +124,7 @@ PXF supports writing the list ORC compound type for one-dimensional arrays, for ## Creating the External Table -The PXF HDFS connector `hdfs:orc` profile supports reading and writing ORC-formatted HDFS files. When you insert records into a writable external table, the block(s) of data that you insert are written to one or more files in the directory that you specified. +The PXF HDFS connector `hdfs:orc` profile supports reading and writing ORC-formatted HDFS files. When you insert records into a writable external table, the block(s) of data that you insert are written to one file per segment in the directory that you specified. Use the following syntax to create a Greenplum Database external table that references an HDFS file or directory: @@ -156,6 +156,17 @@ The PXF `hdfs:orc` profile supports the following *read* options. You specify th | IGNORE_MISSING_PATH | A Boolean value that specifies the action to take when \ is missing or invalid. The default value is `false`, PXF returns an error in this situation. When the value is `true`, PXF ignores missing path errors and returns an empty fragment. | | MAP_BY_POSITION | A Boolean value that, when set to `true`, specifies that PXF should map an ORC column to a Greenplum Database column by position. The default value is `false`, PXF maps an ORC column to a Greenplum column by name. | +The PXF `hdfs:orc` profile supports a single compression-related write option; you specify this option in the `CREATE WRITABLE EXTERNAL TABLE` `LOCATION` clause: + +| Write Option | Value Description | +|-------|-------------------------------------| +| COMPRESSION_CODEC | The compression codec alias. Supported compression codecs for writing ORC data include: `lz4`, `lzo`, `zstd`, `snappy`, `zlib`, and `none` . If this option is not specified, PXF compresses the data using `zlib` compression. | + +## About Writing ORC data + +When you insert records into a writable external table, the block(s) of data that you insert are written to one or more files in the directory that you specify in the `LOCATION` clause. + +When you insert ORC data records, the `pxf.orc.write.timezone.utc` property in the `pxf-site.xml` file governs how PXF writes timestamp values to the external data store. By default, PXF writes a timestamp type using the UTC time zone. If you require PXF to write a timestamp type using the local time zone of the PXF JVM, set the `pxf.orc.write.timezone.utc` property to `false` for the server and synchronize the PXF configuration. ## Example: Reading an ORC File on HDFS @@ -184,11 +195,10 @@ Procedure: ``` shell $ echo '{"location": "Prague", "month": "Jan","num_orders": 101, "total_sales": 4875.33, "items_sold": ["boots", "hats"]} - {"location": "Rome", "month": "Mar","num_orders": 87, "total_sales": 1557.39, "items_sold": ["coats"]} - {"location": "Bangalore", "month": "May","num_orders": 317, "total_sales": 8936.99, "items_sold": ["winter socks", "long-sleeved shirts", "boots"]} - {"location": "Beijing", "month": "Jul","num_orders": 411, "total_sales": 11600.67, "items_sold": ["hoodies/sweaters", "pants"]} - {"location": "Los Angeles", "month": "Dec","num_orders": 0, "total_sales": 0.00, "items_sold": null} -' > /tmp/sampledata.json + {"location": "Rome", "month": "Mar","num_orders": 87, "total_sales": 1557.39, "items_sold": ["coats"]} + {"location": "Bangalore", "month": "May","num_orders": 317, "total_sales": 8936.99, "items_sold": ["winter socks", "long-sleeved shirts", "boots"]} + {"location": "Beijing", "month": "Jul","num_orders": 411, "total_sales": 11600.67, "items_sold": ["hoodies/sweaters", "pants"]} + {"location": "Los Angeles", "month": "Dec","num_orders": 0, "total_sales": 0.00, "items_sold": null}' > /tmp/sampledata.json ``` 1. [Download](https://repo1.maven.org/maven2/org/apache/orc/orc-tools) the most recent version of the `orc-tools` JAR to the current working directory. @@ -268,7 +278,9 @@ Procedure: ## Example: Writing to an ORC File on HDFS -1. Use the `hdfs:orc` profile to create a writable external table that references `sample_orc`: +In this example, you create a writable external table to write some data to the directory referenced by the `sample_orc` table. + +1. Create an external table that specifies the `hdfs:orc` profile and the HDFS directory `/data/pxf_examples/orc_example` in the `LOCATION` URL: ``` sql postgres=# CREATE WRITABLE EXTERNAL TABLE write_to_sample_orc (location text, month text, num_orders int, total_sales numeric(10,2), items_sold text[] ) @@ -276,15 +288,16 @@ Procedure: FORMAT 'CUSTOM' (FORMATTER='pxfwritable_export'); ``` -1. Write a few records to segment files in the `orc_example` directory by inserting to the `write_to_sample_orc` table: +1. Write a few records to segment files in the `orc_example` directory by inserting into the `write_to_sample_orc` table: ``` sql postgres=# INSERT INTO write_to_sample_orc VALUES ( 'Frankfurt', 'Mar', 777, 3956.98, '{"winter socks","pants",boots}' ); postgres=# INSERT INTO write_to_sample_orc VALUES ( 'Cleveland', 'Oct', 3218, 96645.37, '{"long-sleeved shirts",hats}' ); ``` -1. Recall that Greenplum Database does not support directly querying a writable external table. Query the `sample_orc` table to read the data you added to files in the `orc_example` directory: +1. Recall that Greenplum Database does not support directly querying a writable external table. Query the `sample_orc` table that you created in the previous example to read the new data that you added: ``` sql postgres=# SELECT * FROM sample_orc ORDER BY num_orders; ``` + diff --git a/docs/content/hdfs_parquet.html.md.erb b/docs/content/hdfs_parquet.html.md.erb index d879abbd0..df400d581 100644 --- a/docs/content/hdfs_parquet.html.md.erb +++ b/docs/content/hdfs_parquet.html.md.erb @@ -88,7 +88,7 @@ PXF uses the following data type mapping when writing Parquet data: | Text | utf8 | binary | | OTHERS | -- | UNSUPPORTED | -
1 PXF localizes a Timestamp to the current system timezone and converts it to universal time (UTC) before finally converting to int96. +
1 PXF localizes a Timestamp to the current system time zone and converts it to universal time (UTC) before finally converting to int96.
2 PXF converts a Timestamptz to a UTC timestamp and then converts to int96. PXF loses the time zone information during this conversion. diff --git a/docs/content/jdbc_cfg.html.md.erb b/docs/content/jdbc_cfg.html.md.erb index 9b0018bca..b5f23deab 100644 --- a/docs/content/jdbc_cfg.html.md.erb +++ b/docs/content/jdbc_cfg.html.md.erb @@ -19,7 +19,7 @@ In previous releases of Greenplum Database, you may have specified the JDBC driv ## JDBC Driver JAR Registration -PXF is bundled with the `postgresql-42.3.3.jar` JAR file. If you require a different JDBC driver, ensure that you install the JDBC driver JAR file for the external SQL database in the `$PXF_BASE/lib` directory on each Greenplum host. Be sure to install JDBC driver JAR files that are compatible with your JRE version. See [Registering PXF Library Dependencies](reg_jar_depend.html) for additional information. +PXF is bundled with the `postgresql-42.4.1.jar` JAR file. If you require a different JDBC driver, ensure that you install the JDBC driver JAR file for the external SQL database in the `$PXF_BASE/lib` directory on each Greenplum host. Be sure to install JDBC driver JAR files that are compatible with your JRE version. See [Registering PXF Library Dependencies](reg_jar_depend.html) for additional information. ## JDBC Server Configuration @@ -111,7 +111,9 @@ Ensure that the JDBC driver for the external SQL database supports any statement ### Prepared Statements -The PXF JDBC Connector supports reading using a `PreparedStatement` instead of a `Statement` from an external data source. This may be necessary to enable JDBC driver features with data sources that require a `PreparedStatement`. Enable `PreparedStatement` support using the property: +By default, the PXF JDBC Connector reads from an external data source using a JDBC `Statement`. + +The PXF `jdbc.read.prepared-statement` property governs the use of `PreparedStatements` by the connector. If the JDBC driver that you are using to access the external data source requires the use of a `PreparedStatement`, set the property to `true`: | Property | Description | Default Value | |----------------|--------------------------------------------|---------------| diff --git a/docs/content/objstore_orc.html.md.erb b/docs/content/objstore_orc.html.md.erb index 7a0c0c90a..03d64fa2b 100644 --- a/docs/content/objstore_orc.html.md.erb +++ b/docs/content/objstore_orc.html.md.erb @@ -56,17 +56,17 @@ If you are accessing an S3 object store, you can provide S3 credentials via cust Refer to [Example: Reading an ORC File on HDFS](hdfs_orc.html#read_example) in the PXF Hadoop ORC documentation for an example. Modifications that you must make to run the example with an object store include: -- Copying the segment files to the object store instead of HDFS. For example, to copy the files to S3: +- Copying the ORC file to the object store instead of HDFS. For example, to copy the file to S3: ``` shell - $ aws s3 cp /tmp/sample-orc-data s3://BUCKET/pxf_examples/ + $ aws s3 cp /tmp/sampledata.orc s3://BUCKET/pxf_examples/orc_example/ ``` - Using the `CREATE EXTERNAL TABLE` syntax and `LOCATION` keywords and settings described above. For example, if your server name is `s3srvcfg`: ``` sql CREATE EXTERNAL TABLE sample_orc( location TEXT, month TEXT, num_orders INTEGER, total_sales NUMERIC(10,2), items_sold TEXT[] ) - LOCATION('pxf://BUCKET/pxf_examples/sample-orc-data?PROFILE=s3:orc&SERVER=s3srvcfg') + LOCATION('pxf://BUCKET/pxf_examples/orc_example?PROFILE=s3:orc&SERVER=s3srvcfg') FORMAT 'CUSTOM' (FORMATTER='pxfwritable_import'); ``` @@ -74,7 +74,7 @@ Refer to [Example: Reading an ORC File on HDFS](hdfs_orc.html#read_example) in t ``` sql CREATE WRITABLE EXTERNAL TABLE write_to_sample_orc (location TEXT, month TEXT, num_orders INT, total_sales NUMERIC(10,2), items_sold TEXT[]) - LOCATION ('pxf://BUCKET/pxf_examples/sample-orc-data?PROFILE=s3:orc&SERVER=s3srvcfg') + LOCATION ('pxf://BUCKET/pxf_examples/orc_example?PROFILE=s3:orc&SERVER=s3srvcfg') FORMAT 'CUSTOM' (FORMATTER='pxfwritable_export'); ``` diff --git a/external-table/src/gpdbwritableformatter.c b/external-table/src/gpdbwritableformatter.c index 835c9a044..7db9f30be 100644 --- a/external-table/src/gpdbwritableformatter.c +++ b/external-table/src/gpdbwritableformatter.c @@ -960,19 +960,24 @@ gpdbwritableformatter_import(PG_FUNCTION_ARGS) * Read string value from data_buf and convert it to internal representation * * PXF service should send all strings with a terminating nul-byte, so we can - * determine the length of the string and compare it with the length that PXF - * service computed and sent. Since we've checked that outlen[i] is no larger - * than the number of bytes left for this tuple in data_buf, we use it as max - * number of bytes to scan. + * determine the number of bytes that the input function will read and compare + * it with the length that the PXF service computed and sent. Since we've + * checked that outlen[i] is no larger than the number of bytes left for this + * tuple in data_buf, we use it as max number of bytes to scan in the call to + * strnlen */ size_t actual_len = strnlen(data_buf + bufidx, myData->outlen[i]); - /* outlen[i] includes the terminating nul-byte */ - if (actual_len != myData->outlen[i] - 1) - ereport(FATAL, - (errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION), - errmsg("data for column %d of row %d has invalid length", i + 1, myData->lineno), - errdetail("expected column %d to have length %d, actual length is %ld", i + 1, myData->outlen[i] - 1, actual_len))); + /* + * Compare the length as returned by strnlen with the length as determined by + * the PXF service. If the source data includes ASCII NUL-byte in the string, + * then these two values will not match. It's possible that this will result + * in the Postgres input function truncating/mis-reading the value. Rather + * than treat this as an error here, we log a debug message. + */ + if (actual_len != myData->outlen[i]-1) + ereport(DEBUG1, + (errmsg("expected column %d of row %d to have length %d, actual length is %ld", i+1, myData->lineno, myData->outlen[i]-1, actual_len))); myData->values[i] = InputFunctionCall(iofunc, data_buf + bufidx, diff --git a/server/pxf-hdfs/src/main/java/org/greenplum/pxf/plugins/hdfs/HdfsFileFragmenter.java b/server/pxf-hdfs/src/main/java/org/greenplum/pxf/plugins/hdfs/HdfsFileFragmenter.java index 261644493..efcb746ca 100644 --- a/server/pxf-hdfs/src/main/java/org/greenplum/pxf/plugins/hdfs/HdfsFileFragmenter.java +++ b/server/pxf-hdfs/src/main/java/org/greenplum/pxf/plugins/hdfs/HdfsFileFragmenter.java @@ -47,7 +47,7 @@ public List getFragments() throws Exception { } fragments = Arrays.stream(fileStatusArray) - .map(fileStatus -> new Fragment(fileStatus.getPath().toUri().toString())) + .map(fileStatus -> new Fragment(fileStatus.getPath().toUri().toString(), new HcfsFragmentMetadata(0, fileStatus.getLen()))) .collect(Collectors.toList()); LOG.debug("Total number of fragments = {}", fragments.size()); diff --git a/server/pxf-json/src/main/java/org/greenplum/pxf/plugins/json/JsonProtocolHandler.java b/server/pxf-json/src/main/java/org/greenplum/pxf/plugins/json/JsonProtocolHandler.java new file mode 100644 index 000000000..9673212e2 --- /dev/null +++ b/server/pxf-json/src/main/java/org/greenplum/pxf/plugins/json/JsonProtocolHandler.java @@ -0,0 +1,41 @@ +package org.greenplum.pxf.plugins.json; + +import org.greenplum.pxf.api.model.ProtocolHandler; +import org.greenplum.pxf.api.model.RequestContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.commons.lang.StringUtils.isNotEmpty; + +/** + * Implementation of ProtocolHandler for json protocol. + */ +public class JsonProtocolHandler implements ProtocolHandler { + + private static final Logger LOG = LoggerFactory.getLogger(JsonProtocolHandler.class); + private static final String HCFS_FILE_FRAGMENTER = "org.greenplum.pxf.plugins.hdfs.HdfsFileFragmenter"; + + @Override + public String getFragmenterClassName(RequestContext context) { + String fragmenter = context.getFragmenter(); // default to fragmenter defined by the profile + if (useMultilineJson(context)) { + fragmenter = HCFS_FILE_FRAGMENTER; + } + LOG.debug("Determined to use {} fragmenter", fragmenter); + return fragmenter; + } + + @Override + public String getAccessorClassName(RequestContext context) { + return context.getAccessor(); + } + + @Override + public String getResolverClassName(RequestContext context) { + return context.getResolver(); + } + + public boolean useMultilineJson(RequestContext context) { + return isNotEmpty(context.getOption("identifier")); + } +} diff --git a/server/pxf-json/src/test/java/org/greenplum/pxf/plugins/json/JsonProtocolHandlerTest.java b/server/pxf-json/src/test/java/org/greenplum/pxf/plugins/json/JsonProtocolHandlerTest.java new file mode 100644 index 000000000..b90ab9c42 --- /dev/null +++ b/server/pxf-json/src/test/java/org/greenplum/pxf/plugins/json/JsonProtocolHandlerTest.java @@ -0,0 +1,57 @@ +package org.greenplum.pxf.plugins.json; + +import org.greenplum.pxf.api.model.RequestContext; +import org.greenplum.pxf.api.utilities.ColumnDescriptor; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class JsonProtocolHandlerTest { + + private static final String FILE_FRAGMENTER = "org.greenplum.pxf.plugins.hdfs.HdfsFileFragmenter"; + private static final String DEFAULT_ACCESSOR = "default-accessor"; + private static final String DEFAULT_RESOLVER = "default-resolver"; + private static final String DEFAULT_FRAGMENTER = "default-fragmenter"; + private JsonProtocolHandler handler; + private RequestContext context; + + @BeforeEach + public void before() { + handler = new JsonProtocolHandler(); + context = new RequestContext(); + context.setFragmenter("default-fragmenter"); + context.setAccessor("default-accessor"); + context.setResolver("default-resolver"); + List columns = new ArrayList<>(); + columns.add(new ColumnDescriptor("c1", 1, 0, "INT", null, true)); // actual args do not matter + columns.add(new ColumnDescriptor("c2", 2, 0, "INT", null, true)); // actual args do not matter + context.setTupleDescription(columns); + } + + @Test + public void testTextIdentifierOptionMissing() { + assertEquals(DEFAULT_FRAGMENTER, handler.getFragmenterClassName(context)); + assertEquals(DEFAULT_ACCESSOR, handler.getAccessorClassName(context)); + assertEquals(DEFAULT_RESOLVER, handler.getResolverClassName(context)); + } + + @Test + public void testWithEmptyIdentifier() { + context.addOption("IDENTIFIER", ""); + assertEquals(DEFAULT_FRAGMENTER, handler.getFragmenterClassName(context)); + assertEquals(DEFAULT_ACCESSOR, handler.getAccessorClassName(context)); + assertEquals(DEFAULT_RESOLVER, handler.getResolverClassName(context)); + } + + @Test + public void testWithIdentifier() { + context.addOption("IDENTIFIER", "c1"); + assertEquals(FILE_FRAGMENTER, handler.getFragmenterClassName(context)); + assertEquals(DEFAULT_ACCESSOR, handler.getAccessorClassName(context)); + assertEquals(DEFAULT_RESOLVER, handler.getResolverClassName(context)); + } +} diff --git a/server/pxf-s3/src/main/java/org/greenplum/pxf/plugins/s3/S3ProtocolHandler.java b/server/pxf-s3/src/main/java/org/greenplum/pxf/plugins/s3/S3ProtocolHandler.java index bd2e5ac45..145dea64c 100644 --- a/server/pxf-s3/src/main/java/org/greenplum/pxf/plugins/s3/S3ProtocolHandler.java +++ b/server/pxf-s3/src/main/java/org/greenplum/pxf/plugins/s3/S3ProtocolHandler.java @@ -14,6 +14,7 @@ import java.util.Objects; import java.util.Set; +import static org.apache.commons.lang.StringUtils.isNotEmpty; import static org.greenplum.pxf.plugins.s3.S3SelectAccessor.FILE_HEADER_INFO_IGNORE; import static org.greenplum.pxf.plugins.s3.S3SelectAccessor.FILE_HEADER_INFO_USE; @@ -54,7 +55,7 @@ public class S3ProtocolHandler implements ProtocolHandler { @Override public String getFragmenterClassName(RequestContext context) { String fragmenter = context.getFragmenter(); // default to fragmenter defined by the profile - if (useS3Select(context)) { + if (useS3Select(context) || useMultilineJson(context)) { fragmenter = HCFS_FILE_FRAGMENTER; } LOG.debug("Determined to use {} fragmenter", fragmenter); @@ -127,6 +128,10 @@ private boolean useS3Select(RequestContext context) { } } + public boolean useMultilineJson(RequestContext context) { + return isNotEmpty(context.getOption("identifier")); + } + /** * For CSV or TEXT files, it returns true if the file has headers * diff --git a/server/pxf-s3/src/test/java/org/greenplum/pxf/plugins/s3/S3ProtocolHandlerTest.java b/server/pxf-s3/src/test/java/org/greenplum/pxf/plugins/s3/S3ProtocolHandlerTest.java index 3092a26d6..4456afa39 100644 --- a/server/pxf-s3/src/test/java/org/greenplum/pxf/plugins/s3/S3ProtocolHandlerTest.java +++ b/server/pxf-s3/src/test/java/org/greenplum/pxf/plugins/s3/S3ProtocolHandlerTest.java @@ -60,6 +60,8 @@ public class S3ProtocolHandlerTest { private static final String[] EXPECTED_RESOLVER_TEXT_AUTO_NO_BENEFIT_HAS_HEADER = {DEFAULT_RESOLVER, STRING_PASS_RESOLVER, STRING_PASS_RESOLVER, STRING_PASS_RESOLVER, DEFAULT_RESOLVER}; private static final String[] EXPECTED_FRAGMENTER_TEXT_AUTO_NO_BENEFIT_HAS_HEADER = {DEFAULT_FRAGMENTER, FILE_FRAGMENTER, FILE_FRAGMENTER, FILE_FRAGMENTER, DEFAULT_FRAGMENTER}; + private static final String[] EXPECTED_FRAGMENTER_MULTILINE = {FILE_FRAGMENTER, FILE_FRAGMENTER, FILE_FRAGMENTER, FILE_FRAGMENTER, FILE_FRAGMENTER}; + private S3ProtocolHandler handler; private RequestContext context; @@ -415,6 +417,38 @@ public void testWithSelectOffAndWithUnSupportedCompressionType() { verifyResolvers(context, EXPECTED_RESOLVER_GPDB_WRITABLE_OFF); } + @Test + public void testTextIdentifierAndSelectOff() { + context.addOption("S3_SELECT", "off"); + context.addOption("IDENTIFIER", "c1"); + context.setOutputFormat(OutputFormat.TEXT); + verifyAccessors(context, EXPECTED_ACCESSOR_TEXT_OFF); + verifyResolvers(context, EXPECTED_RESOLVER_TEXT_OFF); + verifyFragmenters(context, EXPECTED_FRAGMENTER_MULTILINE); + } + + @Test + public void testTextIdentifierAndSelectOn() { + // s3 options should override multiline json fragmenter option + context.addOption("S3_SELECT", "on"); + context.addOption("IDENTIFIER", "c1"); + context.setOutputFormat(OutputFormat.TEXT); + verifyAccessors(context, EXPECTED_ACCESSOR_TEXT_ON); + verifyResolvers(context, EXPECTED_RESOLVER_TEXT_ON); + verifyFragmenters(context, EXPECTED_FRAGMENTER_TEXT_ON); + } + + @Test + public void testTextIdentifierAndSelectAuto() { + // s3 options should override multiline json fragmenter option + context.addOption("S3_SELECT", "auto"); + context.addOption("IDENTIFIER", "c1"); + context.setOutputFormat(OutputFormat.TEXT); + verifyAccessors(context, EXPECTED_ACCESSOR_TEXT_AUTO_NO_BENEFIT); + verifyResolvers(context, EXPECTED_RESOLVER_TEXT_AUTO_NO_BENEFIT); + verifyFragmenters(context, EXPECTED_FRAGMENTER_MULTILINE); + } + private void verifyFragmenters(RequestContext context, String[] expected) { IntStream.range(0, FORMATS.length).forEach(i -> { context.setFormat(FORMATS[i]); diff --git a/server/pxf-service/src/main/resources/pxf-profiles-default.xml b/server/pxf-service/src/main/resources/pxf-profiles-default.xml index 2ebadf82b..77255f33e 100644 --- a/server/pxf-service/src/main/resources/pxf-profiles-default.xml +++ b/server/pxf-service/src/main/resources/pxf-profiles-default.xml @@ -709,6 +709,7 @@ under the License. org.greenplum.pxf.plugins.json.JsonAccessor org.greenplum.pxf.plugins.json.JsonResolver + org.greenplum.pxf.plugins.json.JsonProtocolHandler hdfs:json @@ -723,6 +724,7 @@ under the License. org.greenplum.pxf.plugins.json.JsonAccessor org.greenplum.pxf.plugins.json.JsonResolver + org.greenplum.pxf.plugins.json.JsonProtocolHandler s3:json @@ -757,6 +759,7 @@ under the License. org.greenplum.pxf.plugins.json.JsonAccessor org.greenplum.pxf.plugins.json.JsonResolver + org.greenplum.pxf.plugins.json.JsonProtocolHandler adl @@ -772,6 +775,7 @@ under the License. org.greenplum.pxf.plugins.json.JsonAccessor org.greenplum.pxf.plugins.json.JsonResolver + org.greenplum.pxf.plugins.json.JsonProtocolHandler wasbs @@ -787,6 +791,7 @@ under the License. org.greenplum.pxf.plugins.json.JsonAccessor org.greenplum.pxf.plugins.json.JsonResolver + org.greenplum.pxf.plugins.json.JsonProtocolHandler gs @@ -802,6 +807,7 @@ under the License. org.greenplum.pxf.plugins.json.JsonAccessor org.greenplum.pxf.plugins.json.JsonResolver + org.greenplum.pxf.plugins.json.JsonProtocolHandler