merge master

Author: nozjkoitop

.github/config/codeql-config.yml

@@ -1,3 +1,3 @@
 query-filters:
   - exclude:
-      id: java/stack-trace-exposure
+      id: java/stack-trace-exposure

.github/dependabot.yml

@@ -15,3 +15,17 @@ updates:
       # pin Testng dependencies to 7.3.0
       - dependency-name: "org.testng"
         versions: "[7.4.0,)"
+
+  # Don't run dependabot on website
+  - package-ecosystem: "npm"
+    directory: "/website/"
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-patch", "version-update:semver-minor", "version-update:semver-major"]
+
+  # Don't run dependabot on web-console
+  - package-ecosystem: "npm"
+    directory: "/web-console/"
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-patch", "version-update:semver-minor", "version-update:semver-major"]

.github/labeler.yml

@@ -24,7 +24,7 @@
   - changed-files:
     - any-glob-to-any-file:
       - 'indexing-hadoop/**'
-      - 'extensions-core/multi-stage-query/**'
+      - 'multi-stage-query/**'
 
 'Area - Dependencies':
   - changed-files:
@@ -61,13 +61,13 @@
 'Area - MSQ':
   - changed-files:
     - any-glob-to-any-file:
-      - 'extensions-core/multi-stage-query/**'
+      - 'multi-stage-query/**'
 
 'Area - Querying':
   - changed-files:
     - any-glob-to-any-file:
       - 'sql/**'
-      - 'extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/**'
+      - 'multi-stage-query/src/main/java/org/apache/druid/msq/sql/**'
 
 'Area - Segment Format and Ser/De':
   - changed-files:
@@ -94,3 +94,6 @@
   - changed-files:
       - any-glob-to-any-file:
           - '.github/**'
+
+'Backport':
+  - head-branch: ['^backport']

.github/pull_request_template.md

@@ -69,4 +69,4 @@ This PR has:
 - [ ] added comments explaining the "why" and the intent of the code wherever would not be obvious for an unfamiliar reader.
 - [ ] added unit tests or modified existing tests to cover new code paths, ensuring the threshold for [code coverage](https://github.com/apache/druid/blob/master/dev/code-review/code-coverage.md) is met.
 - [ ] added integration tests.
-- [ ] been tested in a test Druid cluster.
+- [ ] been tested in a test Druid cluster.

.github/scripts/collect_jstacks

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#--------------------------------------------------------------------
+
+echo "@ capturing threaddumps every minute for all java processes" >&2
+
+while :;do
+	sleep 60
+	t=`date +%s`
+	echo "@ capturing threaddumps $t" >&2
+	pgrep java | while read pid;do
+		jstack $pid > target/jstack.$pid.$t
+	done
+done

.github/scripts/create-jacoco-coverage-report.sh

@@ -20,11 +20,6 @@ set -x
 
 echo "GITHUB_BASE_REF: ${GITHUB_BASE_REF}"
 
-if [ "$GITHUB_BASE_REF" == "" ] ;then
-  echo "GITHUB_BASE_REF is not set; skipping this check!"
-  exit 0
-fi
-
 echo "Setting up git remote"
 git remote set-branches --add origin ${GITHUB_BASE_REF}
 git fetch
@@ -72,3 +67,6 @@ fi
 
 echo " * test balancing details"
 .github/scripts/test_balancing_calc
+
+echo " * validate junit4/junit5 usage"
+dev/validate-junit5-usage

.github/scripts/run-unit-tests.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Runs Docker tests
+# Requires environment variable DRUID_DIST_IMAGE_NAME
+
+set -e
+
+DRUID_IMAGE_NAME_ENV="DRUID_DIST_IMAGE_NAME"
+DRUID_IMAGE_NAME=$DRUID_DIST_IMAGE_NAME
+
+DRUID_IMAGE_SYS_PROPERTY="druid.testing.docker.image"
+
+if [ -z "${DRUID_IMAGE_NAME}" ]; then
+  echo "ERROR!! Environment variable [$DRUID_IMAGE_NAME_ENV] not set!"
+  echo "Run 'export $DRUID_IMAGE_NAME_ENV=<druid-image-to-test>' to specify the image to use in the Docker tests."
+  exit 1
+else
+  echo "Running Docker tests with image[$DRUID_IMAGE_NAME]"
+fi
+
+# No snapshot updates
+OPTS+=" -nsu"
+OPTS+=" -pl !integration-tests"
+mvn -B $OPTS verify -Pdocker-tests,skip-static-checks -DskipUTs -D$DRUID_IMAGE_SYS_PROPERTY=$DRUID_IMAGE_NAME "-DjfrProfilerArgLine=$JFR_PROFILER_ARG_LINE" "$@"

.github/scripts/run_docker-tests

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+set -x
+
+OPTS+="-pl !integration-tests"
+OPTS+=" -Dsurefire.failIfNoSpecifiedTests=false -P skip-static-checks -Dweb.console.skip=true"
+OPTS+=" -Djacoco.destFile=target/jacoco-${HASH}.exec"
+
+mvn -B $OPTS test "-DjfrProfilerArgLine=$JFR_PROFILER_ARG_LINE" "$@"

.github/scripts/run_unit-tests

@@ -17,10 +17,14 @@
 
 set -e
 
-sudo apt-get update && sudo apt-get install python3 -y
+if [ "$(uname)" = "Linux" ]; then
+  # github action runs on ubuntu, we might need to install python3 and pip3
+  sudo apt-get update && sudo apt-get install python3 -y
+  sudo apt install python3-pip
+fi
+
 # creating python virtual env
 python3 -m venv ~/.python3venv
 source ~/.python3venv/bin/activate
-sudo apt install python3-pip
 pip3 install wheel  # install wheel first explicitly
 pip3 install --upgrade pyyaml