feat: support jetty-servlet

Author: Moonlor

core/add_project.sh

@@ -48,6 +48,15 @@ function setup_alluxio() {
     mvn clean install -DskipTests -Dcheckstyle.skip -Dlicense.skip -Dfindbugs.skip -Dmaven.javadoc.skip=true
 }
 
+function setup_jetty() {
+    [ ! -d "app/ctest-jetty" ] && git clone https://github.com/Moonlor/ctest-jetty.git app/ctest-jetty
+    cd app/ctest-jetty
+    home_dir=$PWD
+    git fetch && git checkout ctest-injection
+    cd $home_dir/jetty-servlet
+    mvn clean install -DskipTests
+}
+
 function usage() {
     echo "Usage: add_project.sh <main project>"
     exit 1
@@ -64,7 +73,8 @@ function main() {
             hbase) setup_hbase ;;
             zookeeper) setup_zookeeper ;;
             alluxio) setup_alluxio ;;
-            *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper and alluxio." ;;
+            jetty) setup_jetty;;
+            *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper, alluxio and jetty." ;;
         esac
     fi
 }

core/ctest_const.py

@@ -12,18 +12,21 @@
 HBASE = "hbase-server"
 ZOOKEEPER = "zookeeper-server"
 ALLUXIO = "alluxio-core"
+JTSERVLET = "jetty-servlet"
 
 CTEST_HADOOP_DIR = os.path.join(APP_DIR, "ctest-hadoop")
 CTEST_HBASE_DIR = os.path.join(APP_DIR, "ctest-hbase")
 CTEST_ZK_DIR = os.path.join(APP_DIR, "ctest-zookeeper")
 CTEST_ALLUXIO_DIR = os.path.join(APP_DIR, "ctest-alluxio")
+CTEST_JTSERVLET_DIR = os.path.join(APP_DIR, "ctest-jetty")
 
 PROJECT_DIR = {
     HCOMMON: CTEST_HADOOP_DIR,
     HDFS: CTEST_HADOOP_DIR,
     HBASE: CTEST_HBASE_DIR,
     ZOOKEEPER: CTEST_ZK_DIR,
     ALLUXIO: CTEST_ALLUXIO_DIR,
+    JTSERVLET: CTEST_JTSERVLET_DIR,
 }
 
 
@@ -34,6 +37,7 @@
     HBASE: "hbase-server",
     ZOOKEEPER: "zookeeper-server",
     ALLUXIO: "core",
+    JTSERVLET: "jetty-servlet",
 }
 
 
@@ -58,6 +62,9 @@
         os.path.join(CTEST_ALLUXIO_DIR, MODULE_SUBDIR[ALLUXIO], "server/worker", SUREFIRE_SUBDIR),
         os.path.join(CTEST_ALLUXIO_DIR, MODULE_SUBDIR[ALLUXIO], "server/master", SUREFIRE_SUBDIR),
     ],
+    "jetty-servlet": [
+        os.path.join(CTEST_JTSERVLET_DIR, MODULE_SUBDIR[JTSERVLET], SUREFIRE_SUBDIR)
+    ]
 }
 
 # default or deprecate conf path
@@ -74,7 +81,8 @@
     HDFS: os.path.join(DEFAULT_CONF_DIR, HDFS + "-default.tsv"),
     HBASE: os.path.join(DEFAULT_CONF_DIR, HBASE + "-default.tsv"),
     ALLUXIO: os.path.join(DEFAULT_CONF_DIR, ALLUXIO + "-default.tsv"),
-    ZOOKEEPER: os.path.join(DEFAULT_CONF_DIR, ZOOKEEPER + "-default.tsv")
+    ZOOKEEPER: os.path.join(DEFAULT_CONF_DIR, ZOOKEEPER + "-default.tsv"),
+    JTSERVLET: os.path.join(DEFAULT_CONF_DIR, JTSERVLET + "-default.tsv"),
 }
 
 
@@ -96,6 +104,9 @@
     ],
     ALLUXIO: [
         os.path.join(CTEST_ALLUXIO_DIR, "core/alluxio-ctest.properties")
+    ],
+    JTSERVLET: [
+        os.path.join(CTEST_JTSERVLET_DIR, "jetty-servlet/target/classes/ctest.xml")
     ]
 }
 

core/default_configs/jetty-servlet-default.tsv

@@ -0,0 +1,23 @@
+FORCE_WRAPPER		
+acceptRanges	true	If true, range requests and responses are supported.
+cacheControl	max-age=3600,public	If set, all static content will have this value set as the cache-control header.
+dirAllowed	true	If true, directory listings are returned if no welcome file is found. Else 403 Forbidden.
+dispatchPath		
+encodingHeaderCacheSize	0	Max entries in a cache of ACCEPT-ENCODING headers.
+etags	false	If True, weak etags will be generated and handled.
+gzip	true	If set to true, then static content will be served as gzip content encoded if a matching resource is found ending with ".gz"
+maxCacheSize	256000000	The maximum total size of the cache or 0 for no cache.
+maxCachedFileSize	200000000	The maximum size of a file to cache
+maxCachedFiles	2048	maxCachedFiles
+nonContextServlets		
+otherGzipFileExtensions		
+pathInfoOnly	false	If true, only the path info will be applied to the resourceBase
+precompressed		
+redirectWelcome	false	If true, welcome files are redirected rather than forwarded to.
+relativeResourceBase		Set with a pathname relative to the base of the servlet context root. Useful for only serving static content out of only specific subdirectories.
+resourceBase		Set to replace the context resource base
+resourceCache	resourceCache	If set, this is a context attribute name, which the servlet will use to look for a shared ResourceCache instance.
+restrictToLocalhost		
+stylesheet		Set with the location of an optional stylesheet that will be used to decorate the directory listing html.
+useFileMappedBuffer	true	If set to true, it will use mapped file buffers to serve static content when using an NIO connector. Setting this value to false means that a direct buffer will be used instead of a mapped file buffer. This file sets the value to true.
+welcomeServlets	false	If true, attempt to dispatch to welcome files that are servlets, but only after no matching static resources could be found. If false, then a welcome file must exist on disk. If "exact", then exact servlet matches are supported without an existing file. Default is true. This must be false if you want directory listings, but have index.jsp in your welcome file list.

core/generate_ctest/inject.py

@@ -21,7 +21,7 @@ def inject_config(param_value_pairs):
             for p, v in param_value_pairs.items():
                 file.write(p + "=" + v + "\n")
             file.close()
-    elif project in [HCOMMON, HDFS, HBASE]:
+    elif project in [HCOMMON, HDFS, HBASE, JTSERVLET]:
         conf = ET.Element("configuration")
         for p, v in param_value_pairs.items():
             prop = ET.SubElement(conf, "property")
@@ -46,7 +46,7 @@ def clean_conf_file(project):
             file = open(inject_path, "w")
             file.write("\n")
             file.close()
-    elif project in [HCOMMON, HDFS, HBASE]:
+    elif project in [HCOMMON, HDFS, HBASE, JTSERVLET]:
         conf = ET.Element("configuration")
         for inject_path in INJECTION_PATH[project]:
             file = open(inject_path, "wb")

core/generate_ctest/program_input.py

@@ -4,11 +4,11 @@
     # run mode
     "run_mode": "generate_ctest", # string
     # name of the project, i.e. hadoop-common, hadoop-hdfs, see constant.py
-    "project": "hadoop-common", # string
+    "project": "jetty-servlet", # string
     # path to param -> tests json mapping
-    "mapping_path": "../../data/ctest_mapping/opensource-hadoop-common.json", # string
+    "mapping_path": "../../data/ctest_mapping/opensource-jetty-servlet.json", # string
     # good values of params tests will be run against
-    "param_value_tsv": "sample-hadoop-common.tsv", # string
+    "param_value_tsv": "sample-jetty-servlet.tsv", # string
     # display the terminal output live, without saving any results
     "display_mode": False, # bool
     # whether to use mvn test or mvn surefire:test

core/generate_ctest/sample-jetty-servlet.tsv

@@ -0,0 +1,23 @@
+FORCE_WRAPPER	SKIP	SKIP
+acceptRanges	false	SKIP
+cacheControl	max-age=3600	public
+dirAllowed	false	SKIP
+dispatchPath	SKIP	SKIP
+encodingHeaderCacheSize	1	-1
+etags	true	SKIP
+gzip	false	SKIP
+maxCacheSize	128000000	512000000
+maxCachedFileSize	100000000	400000000
+maxCachedFiles	1024	4096
+nonContextServlets	SKIP	SKIP
+otherGzipFileExtensions	SKIP	SKIP
+pathInfoOnly	true	SKIP
+precompressed	SKIP	SKIP
+redirectWelcome	true	SKIP
+relativeResourceBase	SKIP	SKIP
+resourceBase	SKIP	SKIP
+resourceCache	SKIP	SKIP
+restrictToLocalhost	SKIP	SKIP
+stylesheet	SKIP	SKIP
+useFileMappedBuffer	false	SKIP
+welcomeServlets	true	SKIP

core/generate_value/value_generation.py

@@ -25,6 +25,9 @@ def read_tsv(module):
     if module == "zookeeper-server":
         assert len(params) == 32
         return 32
+    elif module == "jetty-servlet":
+        assert len(params) == 23
+        return 23
     else:
         assert len(params) == 90
         return 90
@@ -105,6 +108,8 @@ def print_params(module):
     f = open(module + output, "w")
     if module == "zookeeper-server":
         assert len(params) == 32
+    elif module == "jetty-servlet":
+        assert len(params) == 23
     else:
         assert len(params) >= 90
     for param in params:

core/identify_param/add_project.sh

@@ -47,6 +47,15 @@ function setup_alluxio() {
     mvn clean install -DskipTests -Dcheckstyle.skip -Dlicense.skip -Dfindbugs.skip -Dmaven.javadoc.skip=true
 }
 
+function setup_jetty() {
+    [ ! -d "app/ctest-jetty" ] && git clone https://github.com/Moonlor/ctest-jetty.git app/ctest-jetty
+    cd app/ctest-jetty
+    home_dir=$PWD
+    git fetch && git checkout ctest-logging
+    cd $home_dir/jetty-servlet
+    mvn clean install -DskipTests
+}
+
 function usage() {
     echo "Usage: add_project.sh <main project>"
     exit 1
@@ -63,7 +72,8 @@ function main() {
             hbase) setup_hbase ;;
             zookeeper) setup_zookeeper ;;
             alluxio) setup_alluxio ;;
-            *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper and alluxio." ;;
+            jetty) setup_jetty;;
+            *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper, alluxio and jetty." ;;
         esac
     fi
 }

core/identify_param/constant.py

@@ -7,20 +7,23 @@
 CTEST_HBASE_DIR = os.path.join(APP_DIR, "ctest-hbase")
 CTEST_ZOOKEEPER_DIR = os.path.join(APP_DIR, "ctest-zookeeper")
 CTEST_ALLUXIO_DIR = os.path.join(APP_DIR, "ctest-alluxio")
+CTEST_JETTY_DIR = os.path.join(APP_DIR, "ctest-jetty")
 
 MODULE_PATH = {
     "hadoop-common": CTEST_HADOOP_DIR,
     "hadoop-hdfs": CTEST_HADOOP_DIR,
     "hbase-server": CTEST_HBASE_DIR,
-    "alluxio-core": CTEST_ALLUXIO_DIR
+    "alluxio-core": CTEST_ALLUXIO_DIR,
+    "jetty-servlet": CTEST_JETTY_DIR,
 }
 
 SRC_SUBDIR = {
     "hadoop-common": "hadoop-common-project/hadoop-common",
     "hadoop-hdfs": "hadoop-hdfs-project/hadoop-hdfs",
     "hbase-server": "hbase-server",
     "zookeeper-server": "zookeeper-server",
-    "alluxio-core": "core"
+    "alluxio-core": "core",
+    "jetty-servlet": "jetty-servlet",
 }
 
 MVN_TEST_PATH = {
@@ -29,14 +32,16 @@
     "hbase-server": os.path.join(CTEST_HBASE_DIR, SRC_SUBDIR["hbase-server"]),
     "zookeeper-server": os.path.join(CTEST_ZOOKEEPER_DIR, SRC_SUBDIR["zookeeper-server"]),
     "alluxio-core": os.path.join(CTEST_ALLUXIO_DIR, SRC_SUBDIR["alluxio-core"]),
+    "jetty-servlet": os.path.join(CTEST_JETTY_DIR, SRC_SUBDIR["jetty-servlet"])
 }
 
 LOCAL_CONF_PATH = {
     "hadoop-common": "results/hadoop-common/conf_params.txt",
     "hadoop-hdfs": "results/hadoop-hdfs/conf_params.txt",
     "hbase-server": "results/hbase-server/conf_params.txt",
     "zookeeper-server": "results/zookeeper-server/conf_params.txt",
-    "alluxio-core": "results/alluxio-core/conf_params.txt"
+    "alluxio-core": "results/alluxio-core/conf_params.txt",
+    "jetty-servlet": "results/jetty-servlet/conf_params.txt"
 }
 
 SUREFIRE_SUBDIR = "target/surefire-reports/*"
@@ -63,7 +68,10 @@
         os.path.join(CTEST_ALLUXIO_DIR, "core/server/proxy", SUREFIRE_SUBDIR),
         os.path.join(CTEST_ALLUXIO_DIR, "core/server/worker", SUREFIRE_SUBDIR),
         os.path.join(CTEST_ALLUXIO_DIR, "core/server/master", SUREFIRE_SUBDIR)
-    ]
+    ],
+    "jetty-servlet": [
+        os.path.join(CTEST_JETTY_DIR, SRC_SUBDIR["jetty-servlet"], SUREFIRE_SUBDIR)
+    ],
 }
 
 LOCAL_SUREFIRE_SUFFIX = "surefire-reports/*"
@@ -83,5 +91,8 @@
     ],
     "alluxio-core": [
         os.path.join("surefire-reports/alluxio-core", LOCAL_SUREFIRE_SUFFIX)
-    ]
+    ],
+    "jetty-servlet": [
+        os.path.join("surefire-reports/jetty-servlet", LOCAL_SUREFIRE_SUFFIX)
+    ],
 }

core/identify_param/identify_param.sh

@@ -12,9 +12,9 @@ function main() {
       usage
     else
         case $project in
-            hadoop-common | hadoop-hdfs | hbase-server | zookeeper-server | alluxio-core) python3 runner.py $project; python3 collector.py $project ;;
+            hadoop-common | hadoop-hdfs | hbase-server | zookeeper-server | alluxio-core | jetty-servlet) python3 runner.py $project; python3 collector.py $project ;;
             -h | --help) usage ;;
-            *) echo "Unexpected project: $project - only support hadoop-common, hadoop-hdfs, hbase-server, zookeeper-server and alluxio-core." ;;
+            *) echo "Unexpected project: $project - only support hadoop-common, hadoop-hdfs, hbase-server, zookeeper-server, alluxio-core and jetty-servlet." ;;
         esac
     fi
 }