Harmonize /test and /publish commands on github + Fix MySQL tests in CI (#2531)

* Harmonize /test and /publish commands on github

* Make normalization build on clean system without PyICU

see https://getdbt.slack.com/archives/C50NEBJGG/p1615803416053800?thread_ts=1615511262.026400&cid=C50NEBJGG

* Follow example from testcontainers/testcontainers-java#2689 to modify MySQL container as root

* Add docs on new syntax for /test

Author: ChristopheDuong

.github/workflows/test-command.yml

@@ -13,11 +13,11 @@ jobs:
   integration_test:
     runs-on: ubuntu-latest
     steps:
-      - name: Search for valid integration name format
+      - name: Search for valid connector name format
         id: regex
         uses: AsasInnab/regex-action@v1
         with:
-          regex_pattern: '^[a-zA-Z0-9-_]+$'
+          regex_pattern: '^((connectors|bases)/)?[a-zA-Z0-9-_]+$'
           regex_flags: 'i' # required to be set for this plugin
           search_string: ${{ github.event.inputs.connector }}
       - name: Validate input workflow format

airbyte-integrations/bases/base-normalization/setup.py

@@ -31,7 +31,7 @@
     author_email="[email protected]",
     url="https://github.com/airbytehq/airbyte",
     packages=setuptools.find_packages(),
-    install_requires=["airbyte-protocol", "dbt==0.18.1", "pyyaml"],
+    install_requires=["airbyte-protocol", "dbt==0.18.2rc1", "pyyaml"],
     package_data={"": ["*.yml"]},
     setup_requires=["pytest-runner"],
     entry_points={

airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcStandardTest.java

@@ -27,11 +27,13 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.collect.ImmutableMap;
 import io.airbyte.commons.json.Jsons;
-import io.airbyte.commons.resources.MoreResources;
 import io.airbyte.db.Database;
 import io.airbyte.db.Databases;
 import io.airbyte.integrations.source.jdbc.AbstractJdbcSource;
 import io.airbyte.integrations.source.jdbc.test.JdbcSourceStandardTest;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.jooq.SQLDialect;
 import org.junit.jupiter.api.AfterAll;
@@ -50,13 +52,15 @@ class MySqlJdbcStandardTest extends JdbcSourceStandardTest {
   private Database database;
 
   @BeforeAll
-  static void init() {
-    // test containers withInitScript only accepts scripts that are mounted as resources.
-    MoreResources.writeResource("init.sql",
-        "CREATE USER '" + TEST_USER + "'@'%' IDENTIFIED BY '" + TEST_PASSWORD + "';\n"
-            + "GRANT ALL PRIVILEGES ON *.* TO '" + TEST_USER + "'@'%';\n");
-    container = new MySQLContainer<>("mysql:8.0").withInitScript("init.sql").withUsername("root").withPassword("");
+  static void init() throws SQLException {
+    container = new MySQLContainer<>("mysql:8.0")
+        .withUsername(TEST_USER)
+        .withPassword(TEST_PASSWORD)
+        .withEnv("MYSQL_ROOT_HOST", "%")
+        .withEnv("MYSQL_ROOT_PASSWORD", TEST_PASSWORD);
     container.start();
+    Connection connection = DriverManager.getConnection(container.getJdbcUrl(), "root", TEST_PASSWORD);
+    connection.createStatement().execute("GRANT ALL PRIVILEGES ON *.* TO '" + TEST_USER + "'@'%';\n");
   }
 
   @BeforeEach
@@ -76,6 +80,7 @@ public void setup() throws Exception {
             config.get("host").asText(),
             config.get("port").asText()),
         MySqlSource.DRIVER_CLASS,
+
         SQLDialect.MYSQL);
 
     database.query(ctx -> {

airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlStressTest.java

@@ -27,11 +27,13 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.collect.ImmutableMap;
 import io.airbyte.commons.json.Jsons;
-import io.airbyte.commons.resources.MoreResources;
 import io.airbyte.db.Database;
 import io.airbyte.db.Databases;
 import io.airbyte.integrations.source.jdbc.AbstractJdbcSource;
 import io.airbyte.integrations.source.jdbc.test.JdbcStressTest;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
 import java.util.Optional;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.jooq.SQLDialect;
@@ -53,13 +55,15 @@ class MySqlStressTest extends JdbcStressTest {
   private Database database;
 
   @BeforeAll
-  static void init() {
-    // test containers withInitScript only accepts scripts that are mounted as resources.
-    MoreResources.writeResource("init.sql",
-        "CREATE USER '" + TEST_USER + "'@'%' IDENTIFIED BY '" + TEST_PASSWORD + "';\n"
-            + "GRANT ALL PRIVILEGES ON *.* TO '" + TEST_USER + "'@'%';\n");
-    container = new MySQLContainer<>("mysql:8.0").withInitScript("init.sql").withUsername("root").withPassword("");
+  static void init() throws SQLException {
+    container = new MySQLContainer<>("mysql:8.0")
+        .withUsername(TEST_USER)
+        .withPassword(TEST_PASSWORD)
+        .withEnv("MYSQL_ROOT_HOST", "%")
+        .withEnv("MYSQL_ROOT_PASSWORD", TEST_PASSWORD);
     container.start();
+    Connection connection = DriverManager.getConnection(container.getJdbcUrl(), "root", TEST_PASSWORD);
+    connection.createStatement().execute("GRANT ALL PRIVILEGES ON *.* TO '" + TEST_USER + "'@'%';\n");
   }
 
   @BeforeEach

docs/contributing-to-airbyte/building-new-connector/testing-connectors.md

@@ -70,16 +70,31 @@ Here are some example commands:
 
 1. `/test connector=all` - Runs integration tests for all connectors in a single GitHub workflow. Some of our integration tests interact with rate-limited resources, so please use this judiciously.
 2. `/test connector=source-sendgrid` - Runs integration tests for a single connector on the latest PR commit.
-3. `/test connector=source-sendgrid ref=master` - Runs integration tests for a single connector on a different branch. 
-4. `/test connector=source-sendgrid ref=d5c53102` - Runs integration tests for a single connector on a specific commit.
+3. `/test connector=connectors/source-sendgrid` - Runs integration tests for a single connector on the latest PR commit.
+4. `/test connector=source-sendgrid ref=master` - Runs integration tests for a single connector on a different branch. 
+5. `/test connector=source-sendgrid ref=d5c53102` - Runs integration tests for a single connector on a specific commit.
 
 A command dispatcher GitHub workflow will launch on comment submission. This dispatcher will add an :eyes: reaction to the comment when it starts processing. If there is an error dispatching your request, an error will be appended to your comment. If it launches the test run successfully, a :rocket: reaction will appear on your comment.
 
 Once the integration test workflow launches, it will append a link to the workflow at the end of the comment. A success or failure response will also be added upon workflow completion.
 
 Integration tests can also be manually requested by clicking "[Run workflow](https://github.com/airbytehq/airbyte/actions?query=workflow%3Aintegration-test)" and specifying the connector and GitHub ref.
 
-### 3. Automatically Run From `master`
+### 3. Requesting GitHub PR publishing Docker Images
+
+In order for users to reference the new versions of a connector, it needs to be published and available in the [dockerhub](https://hub.docker.com/r/airbyte/source-sendgrid/tags?page=1&ordering=last_updated) with the latest tag updated.
+
+As seen previously, GitHub workflow can be triggered by comment submission. 
+Publishing docker images to the dockerhub repository can also be submitted likewise:
+
+Note that integration tests can be triggered with a slightly different syntax for arguments. 
+This second set is required to distinguish between `connectors` and `bases` folders.
+Thus, it is also easier to switch between the `/test` and `/publish` commands:
+
+- `/test connector=connectors/source-sendgrid` - Runs integration tests for a single connector on the latest PR commit.
+- `/publish connector=connectors/source-sendgrid` - Publish the docker image if it doesn't exist for a single connector on the latest PR commit.
+
+### 4. Automatically Run From `master`
 
 Commits to `master` attempt to launch integration tests. Two workflows launch for each commit: one is a launcher for integration tests, the other is the core build \(the same as the default for PR and branch builds\).
 

tools/bin/ci_integration_test.sh

@@ -2,6 +2,8 @@
 
 set -e
 
+. tools/lib/lib.sh
+
 # runs integration tests for an integration name
 
 connector="$1"
@@ -11,8 +13,14 @@ if [[ "$connector" == "all" ]] ; then
   echo "Running: ./gradlew --no-daemon --scan integrationTest"
   ./gradlew --no-daemon --scan integrationTest
 else
-  selected_integration_test=$(echo "$all_integration_tests" | grep "^$connector$" || echo "")
-  integrationTestCommand=":airbyte-integrations:connectors:$connector:integrationTest"
+  if [[ "$connector" == *"connectors"* ]] || [[ "$connector" == *"bases"* ]]; then
+    connector_name=$(echo $connector | cut -d / -f 2)
+    selected_integration_test=$(echo "$all_integration_tests" | grep "^$connector_name$" || echo "")
+    integrationTestCommand="$(_to_gradle_path "airbyte-integrations/$connector" integrationTest)"
+  else
+    selected_integration_test=$(echo "$all_integration_tests" | grep "^$connector$" || echo "")
+    integrationTestCommand=":airbyte-integrations:connectors:$connector:integrationTest"
+  fi
   if [ -n "$selected_integration_test" ] ; then
     echo "Running: ./gradlew --no-daemon --scan $integrationTestCommand"
     ./gradlew --no-daemon --scan "$integrationTestCommand"