Use openjdk:17-slim as base image for IT (#18251) (#18256)

Co-authored-by: Cece Mei <yingqian.mei@gmail.com>

Author: capistrant

integration-tests/docker/Dockerfile

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARG JDK_VERSION=17-slim-buster
+ARG JDK_VERSION=17-slim-bullseye
 FROM openjdk:$JDK_VERSION as druidbase
 
 # Bundle everything into one script so cleanup can reduce image size.
@@ -27,7 +27,6 @@ ARG ZK_VERSION
 ARG APACHE_ARCHIVE_MIRROR_HOST=https://downloads.apache.org
 RUN APACHE_ARCHIVE_MIRROR_HOST=${APACHE_ARCHIVE_MIRROR_HOST} /root/base-setup.sh && rm -f /root/base-setup.sh
 
-
 FROM druidbase
 ARG MYSQL_VERSION
 ARG MARIA_VERSION
@@ -37,14 +36,6 @@ ARG CONFLUENT_VERSION
 # Verify Java version
 RUN java -version
 
-RUN echo "[mysqld]\ncharacter-set-server=utf8\ncollation-server=utf8_bin\n" >> /etc/mysql/my.cnf
-
-# Setup metadata store
-# touch is needed because OverlayFS's copy-up operation breaks POSIX standards. See https://github.com/docker/for-linux/issues/72.
-RUN find /var/lib/mysql -type f -exec touch {} \; && /etc/init.d/mysql start \
-      && echo "CREATE USER 'druid'@'%' IDENTIFIED BY 'diurd'; GRANT ALL ON druid.* TO 'druid'@'%'; CREATE database druid DEFAULT CHARACTER SET utf8mb4;" | mysql -u root \
-      && /etc/init.d/mysql stop
-
 # Add Druid scripts and jars
 ADD bin/* /usr/local/druid/bin/
 ADD lib/* /usr/local/druid/lib/
@@ -65,11 +56,17 @@ RUN if [ "$MYSQL_DRIVER_CLASSNAME" = "com.mysql.jdbc.Driver" ] ; \
 RUN wget -q "https://packages.confluent.io/maven/io/confluent/kafka-protobuf-provider/$CONFLUENT_VERSION/kafka-protobuf-provider-$CONFLUENT_VERSION.jar" \
     -O /usr/local/druid/lib/kafka-protobuf-provider.jar
 
-# Add sample data
-# touch is needed because OverlayFS's copy-up operation breaks POSIX standards. See https://github.com/docker/for-linux/issues/72.
-RUN find /var/lib/mysql -type f -exec touch {} \; && service mysql start \
-      && /usr/local/druid/bin/run-java -cp "/usr/local/druid/lib/*" -Ddruid.extensions.directory=/usr/local/druid/extensions -Ddruid.extensions.loadList='["mysql-metadata-storage"]' -Ddruid.metadata.storage.type=mysql -Ddruid.metadata.mysql.driver.driverClassName=$MYSQL_DRIVER_CLASSNAME org.apache.druid.cli.Main tools metadata-init --connectURI="jdbc:mysql://localhost:3306/druid" --user=druid --password=diurd \
-      && /etc/init.d/mysql stop
+# Setup metadata store
+ADD run-mysql.sh /run-mysql.sh
+RUN echo "[mysqld]\ncharacter-set-server=utf8\ncollation-server=utf8_bin\n" >> /etc/mysql/my.cnf
+RUN mkdir -p /run/mysqld && chown mysql:mysql /run/mysqld
+RUN ( \
+    bash /run-mysql.sh; \
+    echo "CREATE USER 'druid'@'%' IDENTIFIED BY 'diurd'; GRANT ALL ON druid.* TO 'druid'@'%'; CREATE database druid DEFAULT CHARACTER SET utf8mb4;" | mysql -u root; \
+    /usr/local/druid/bin/run-java -cp "/usr/local/druid/lib/*" -Ddruid.extensions.directory=/usr/local/druid/extensions -Ddruid.extensions.loadList='["mysql-metadata-storage"]' -Ddruid.metadata.storage.type=mysql -Ddruid.metadata.mysql.driver.driverClassName=$MYSQL_DRIVER_CLASSNAME org.apache.druid.cli.Main tools metadata-init --connectURI="jdbc:mysql://localhost:3306/druid" --user=druid --password=diurd; \
+    mysqladmin shutdown \
+    )
+
 ADD test-data /test-data
 
 # Setup supervisord
@@ -78,9 +75,6 @@ ADD supervisord.conf /etc/supervisor/conf.d/supervisord.conf
 # Add druid configuration setup script
 ADD druid.sh /druid.sh
 
-# mysql
-ADD run-mysql.sh /run-mysql.sh
-
 # internal docker_ip:9092 endpoint is used to access Kafka from other Docker containers
 # external docker ip:9093 endpoint is used to access Kafka from test code
 # run this last to avoid rebuilding the image every time the ip changes

integration-tests/docker/druid.sh

@@ -78,30 +78,31 @@ setupConfig()
       var=$(echo "$evar" | sed -e 's?^\([^=]*\)=.*?\1?g' -e 's?_?.?g')
       setKey $DRUID_SERVICE "$var" "$val"
   done
+  if [ "$MYSQL_DRIVER_CLASSNAME" != "com.mysql.jdbc.Driver" ] ; then
+    setKey $DRUID_SERVICE druid.metadata.mysql.driver.driverClassName $MYSQL_DRIVER_CLASSNAME
+  fi
 }
 
 setupData()
 {
+  # note: this function exists for legacy reasons, ideally we should do data insert in IT's setup method.
+  if [ -n "$DRUID_SERVICE" ]; then
+    echo "DRUID_SERVICE is set, skipping data setup"
+    return
+  fi
+  # note: this function exists for legacy reasons, ideally we should do data insert in IT's setup method.
+
+  bash /run-mysql.sh
   # The "query" and "security" test groups require data to be setup before running the tests.
   # In particular, they requires segments to be download from a pre-existing s3 bucket.
   # This is done by using the loadSpec put into metadatastore and s3 credientials set below.
   if [ "$DRUID_INTEGRATION_TEST_GROUP" = "query" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "query-retry" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "query-error" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "high-availability" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "security" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "ldap-security" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "upgrade" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "centralized-datasource-schema" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "cds-task-schema-publish-disabled" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "cds-coordinator-metadata-query-disabled" ]; then
-    # touch is needed because OverlayFS's copy-up operation breaks POSIX standards. See https://github.com/docker/for-linux/issues/72.
-    find /var/lib/mysql -type f -exec touch {} \; && service mysql start \
-      && cat /test-data/${DRUID_INTEGRATION_TEST_GROUP}-sample-data.sql | mysql -u root druid \
-      && /etc/init.d/mysql stop
-  fi
-
-  if [ "$MYSQL_DRIVER_CLASSNAME" != "com.mysql.jdbc.Driver" ] ; then
-    setKey $DRUID_SERVICE druid.metadata.mysql.driver.driverClassName $MYSQL_DRIVER_CLASSNAME
+    cat /test-data/${DRUID_INTEGRATION_TEST_GROUP}-sample-data.sql | mysql -u root druid
   fi
 
   # The SqlInputSource tests in the "input-source" test group require data to be setup in MySQL before running the tests.
   if [ "$DRUID_INTEGRATION_TEST_GROUP" = "input-source" ] ; then
-    # touch is needed because OverlayFS's copy-up operation breaks POSIX standards. See https://github.com/docker/for-linux/issues/72.
-    find /var/lib/mysql -type f -exec touch {} \; && service mysql start \
-        && echo "GRANT ALL ON sqlinputsource.* TO 'druid'@'%'; CREATE database sqlinputsource DEFAULT CHARACTER SET utf8mb4;" | mysql -u root druid \
-        && cat /test-data/sql-input-source-sample-data.sql | mysql -u root druid \
-        && /etc/init.d/mysql stop
+    echo "GRANT ALL ON sqlinputsource.* TO 'druid'@'%'; CREATE database sqlinputsource DEFAULT CHARACTER SET utf8mb4;" | mysql -u root druid
+    cat /test-data/sql-input-source-sample-data.sql | mysql -u root druid
   fi
 }

integration-tests/docker/run-mysql.sh

@@ -15,4 +15,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-find /var/lib/mysql -type f -exec touch {} \; && /usr/bin/pidproxy /var/run/mysqld/mysqld.pid /usr/bin/mysqld_safe --bind-address=0.0.0.0
+(mysqld --bind-address=0.0.0.0 || true) & # there might already be a mysqld running, so we ignore the error
+/usr/bin/pidproxy /var/run/mysqld/mysqld.pid /usr/bin/mysqld_safe --bind-address=0.0.0.0
+echo "Waiting for MySQL to be ready...";
+for i in {30..0}; do
+  mysqladmin ping --silent && break;
+  sleep 1; \
+done; \
+if [ "$i" = 0 ]; then
+  echo "MySQL did not start"; exit 1;
+fi;

integration-tests/pom.xml

@@ -569,6 +569,7 @@
                 <start.hadoop.docker>false</start.hadoop.docker>
                 <docker.run.skip>false</docker.run.skip>
                 <docker.build.skip>false</docker.build.skip>
+                <docker.build.skipMavenBuild>false</docker.build.skipMavenBuild>
                 <docker.build.hadoop>false</docker.build.hadoop>
                 <it.indexer>middleManager</it.indexer>
                 <override.config.path />
@@ -598,6 +599,7 @@
                                         <DRUID_INTEGRATION_TEST_OVERRIDE_CONFIG_PATH>${override.config.path}</DRUID_INTEGRATION_TEST_OVERRIDE_CONFIG_PATH>
                                         <DRUID_INTEGRATION_TEST_RESOURCE_FILE_DIR_PATH>${resource.file.dir.path}</DRUID_INTEGRATION_TEST_RESOURCE_FILE_DIR_PATH>
                                         <DRUID_INTEGRATION_TEST_SKIP_BUILD_DOCKER>${docker.build.skip}</DRUID_INTEGRATION_TEST_SKIP_BUILD_DOCKER>
+                                        <DRUID_INTEGRATION_TEST_SKIP_MAVEN_BUILD_DOCKER>${docker.build.skipMavenBuild}</DRUID_INTEGRATION_TEST_SKIP_MAVEN_BUILD_DOCKER>
                                         <DRUID_INTEGRATION_TEST_SKIP_RUN_DOCKER>${docker.run.skip}</DRUID_INTEGRATION_TEST_SKIP_RUN_DOCKER>
                                         <DRUID_INTEGRATION_TEST_INDEXER>${it.indexer}</DRUID_INTEGRATION_TEST_INDEXER>
                                         <MYSQL_VERSION>${mysql.version}</MYSQL_VERSION>

integration-tests/script/copy_resources_template.sh

@@ -28,14 +28,21 @@ rm -rf $SHARED_DIR/docker
 mkdir -p $SHARED_DIR
 cp -R docker $SHARED_DIR/docker
 
-pushd ../
-rm -rf distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin
-# using parallel build here may not yield significant speedups
-mvn -B -Pskip-static-checks,skip-tests -Dweb.console.skip=true install -Pintegration-test
-mv distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin/bin $SHARED_DIR/docker/bin
-mv distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin/lib $SHARED_DIR/docker/lib
-mv distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin/extensions $SHARED_DIR/docker/extensions
-popd
+if [ -z "$DRUID_INTEGRATION_TEST_SKIP_MAVEN_BUILD_DOCKER" ] || [ "$DRUID_INTEGRATION_TEST_SKIP_MAVEN_BUILD_DOCKER" = "false" ]
+then
+  pushd ../
+  rm -rf distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin
+  # using parallel build here may not yield significant speedups
+  mvn -B -Pskip-static-checks,skip-tests -Dweb.console.skip=true install -Pintegration-test
+  mv distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin/bin $SHARED_DIR/docker/bin
+  mv distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin/lib $SHARED_DIR/docker/lib
+  mv distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin/extensions $SHARED_DIR/docker/extensions
+  popd
+else
+  cp -R ../distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin/bin $SHARED_DIR/docker/bin
+  cp -R ../distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin/lib $SHARED_DIR/docker/lib
+  cp -R ../distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin/extensions $SHARED_DIR/docker/extensions
+fi
 
 # Make directoriess if they dont exist
 mkdir -p $SHARED_DIR/hadoop_xml

integration-tests/script/docker_build_containers.sh

@@ -29,7 +29,7 @@ else
   11 | 17 | 21)
     echo "Build druid-cluster with Java $DRUID_INTEGRATION_TEST_JVM_RUNTIME"
     docker build -t druid/cluster \
-      --build-arg JDK_VERSION=$DRUID_INTEGRATION_TEST_JVM_RUNTIME-slim-buster \
+      --build-arg JDK_VERSION=$DRUID_INTEGRATION_TEST_JVM_RUNTIME-slim-bullseye \
       --build-arg ZK_VERSION \
       --build-arg KAFKA_VERSION \
       --build-arg CONFLUENT_VERSION \