Port Hive, Yarn, Flink, Kylin, Camel, Nifi to framework

core/Dockerfile

@@ -7,7 +7,7 @@ RUN \
   apt-get install -y git && \
 # Install python
   apt-get update && \
-  apt-get install -y python python-dev python-pip python-virtualenv && \
+  apt-get install -y python python-dev python3-pip python3-virtualenv && \
   rm -rf /var/lib/apt/lists/* && \
 # Install misc
   apt-get update && \

core/README.md

@@ -59,7 +59,7 @@ To generate ctests or run ctest, you need to first clone the target project.
 1. In `openctest/core`, run `./add_project.sh <main project>` to clone the project, switch to and build the branch `ctest-injection`. This branch will be later used by `generate_ctest` and `run_ctest`.
 2. In `openctest/core/identify_param`, run `./add_project.sh <main project>` to clone the project, switch to and build the branch `ctest-logging`. This branch will be later used by `identify_param`.
 
-`<main project>` can be `hadoop`, `hbase`, `zookeeper` or `alluxio`.
+`<main project>` can be `hadoop`, `hbase`, `zookeeper`, `hive-common`, `alluxio` or `nifi-commons`.
 
 ## Usage
 

core/add_project.sh

@@ -47,6 +47,20 @@ function setup_alluxio() {
     cd core
     mvn clean install -DskipTests -Dcheckstyle.skip -Dlicense.skip -Dfindbugs.skip -Dmaven.javadoc.skip=true
 }
+function setup_nifi(){
+    [ ! -d "app/ctest-nifi" ] && git clone https://github.com/lilacyl/nifi.git app/ctest-nifi
+    cd app/ctest-nifi
+    git fetch && git checkout ctest-injection 
+    mvn clean install -pl nifi-commons/ -DskipTest
+}
+
+function setup_hive(){
+    [ ! -d "app/ctest-hive" ] && git clone https://github.com/lilacyl/hive.git app/ctest-hive
+    cd app/ctest-hive
+    git fetch && git checkout ctest-injection
+    cd common
+    mvn clean install -DskipTests
+}
 
 function usage() {
     echo "Usage: add_project.sh <main project>"
@@ -64,7 +78,9 @@ function main() {
             hbase) setup_hbase ;;
             zookeeper) setup_zookeeper ;;
             alluxio) setup_alluxio ;;
-            *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper and alluxio." ;;
+            hive) setup_hive ;;
+            nifi) setup_nifi ;;
+            *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper, nifi, and alluxio." ;;
         esac
     fi
 }

core/ctest_const.py

@@ -12,18 +12,24 @@
 HBASE = "hbase-server"
 ZOOKEEPER = "zookeeper-server"
 ALLUXIO = "alluxio-core"
+HIVE = "hive-common"
+NIFI = "nifi-commons"
 
 CTEST_HADOOP_DIR = os.path.join(APP_DIR, "ctest-hadoop")
 CTEST_HBASE_DIR = os.path.join(APP_DIR, "ctest-hbase")
 CTEST_ZK_DIR = os.path.join(APP_DIR, "ctest-zookeeper")
 CTEST_ALLUXIO_DIR = os.path.join(APP_DIR, "ctest-alluxio")
+CTEST_HIVE_DIR = os.path.join(APP_DIR, "ctest-hive")
+CTEST_NIFI_DIR = os.path.join(APP_DIR, "ctest-nifi")
 
 PROJECT_DIR = {
     HCOMMON: CTEST_HADOOP_DIR,
     HDFS: CTEST_HADOOP_DIR,
     HBASE: CTEST_HBASE_DIR,
     ZOOKEEPER: CTEST_ZK_DIR,
     ALLUXIO: CTEST_ALLUXIO_DIR,
+    HIVE: CTEST_HIVE_DIR,
+    NIFI: CTEST_NIFI_DIR,
 }
 
 
@@ -34,12 +40,16 @@
     HBASE: "hbase-server",
     ZOOKEEPER: "zookeeper-server",
     ALLUXIO: "core",
+    HIVE: "common",
+    NIFI: "nifi-commons",
 }
 
 
 # surefire report
 SUREFIRE_SUBDIR = "target/surefire-reports/"
 SUREFIRE_XML = "TEST-{}.xml" # slot is the classname
+HIVE_SUREFIRE_XML =  "TEST-org.apache.hadoop.hive.conf.{}.xml" # slot is the classname
+SUREFIRE_XML_NIFI = "TEST-org.apache.nifi.util.{}.xml" # slot is the classname
 SUREFIRE_TXT = "{}.txt" # testclass
 SUREFIRE_OUTTXT = "{}-output.txt" #testclass 
 
@@ -58,6 +68,8 @@
         os.path.join(CTEST_ALLUXIO_DIR, MODULE_SUBDIR[ALLUXIO], "server/worker", SUREFIRE_SUBDIR),
         os.path.join(CTEST_ALLUXIO_DIR, MODULE_SUBDIR[ALLUXIO], "server/master", SUREFIRE_SUBDIR),
     ],
+    HIVE: [os.path.join(CTEST_HIVE_DIR, MODULE_SUBDIR[HIVE], SUREFIRE_SUBDIR)],
+    NIFI: [os.path.join(CTEST_NIFI_DIR, MODULE_SUBDIR[NIFI],"nifi-properties", SUREFIRE_SUBDIR)],
 }
 
 # default or deprecate conf path
@@ -74,7 +86,9 @@
     HDFS: os.path.join(DEFAULT_CONF_DIR, HDFS + "-default.tsv"),
     HBASE: os.path.join(DEFAULT_CONF_DIR, HBASE + "-default.tsv"),
     ALLUXIO: os.path.join(DEFAULT_CONF_DIR, ALLUXIO + "-default.tsv"),
-    ZOOKEEPER: os.path.join(DEFAULT_CONF_DIR, ZOOKEEPER + "-default.tsv")
+    ZOOKEEPER: os.path.join(DEFAULT_CONF_DIR, ZOOKEEPER + "-default.tsv"),
+    HIVE: os.path.join(DEFAULT_CONF_DIR, HIVE + "-default.tsv"),
+    NIFI: os.path.join(DEFAULT_CONF_DIR, NIFI + "-default.tsv"),
 }
 
 
@@ -96,6 +110,12 @@
     ],
     ALLUXIO: [
         os.path.join(CTEST_ALLUXIO_DIR, "core/alluxio-ctest.properties")
+    ],
+    HIVE: [
+        os.path.join(CTEST_HIVE_DIR, "conf/hive-ctest.xml")
+    ],
+    NIFI: [
+        os.path.join(CTEST_NIFI_DIR, "nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/ctest.properties")
     ]
 }