Add insta / snapshot testing to CLI & set up AWS mock (#13672)

* Do not normalize values

* Fix tests & update docs

* Prettier

* Lowercase config params

* Add snap to CLI & set up AWS mock

* Refactor tests

* Unify transform and parse

* Fix tests

* Setup CLI

* Show minio output

* Format Cargo.toml

* Do not hardcode AWS params

* Test options parsing

* Add allow http

* Fix aws build

* Fix ip

* Remove slash ☠️

* Format cargo toml

* Remove integration_setup.bash

* Update docs

* Do not hardcode test names

* Relock cargo

* Remove aws sdk and set up minio in-place

* Nit: Add missing ready local to the docs

* Fix backslash test

* Add missing backslash

* put integration scripts in a separate folder

* Move s3 tests from extended to rust flow

* Reorganise the docs

* Prettier

* Do not use rust container to get docker

* Add missing protobuf

* revert change to extended.yml

---------

Co-authored-by: Andrew Lamb <andrew@nerdnetworks.org>

Author: blaginin

.github/workflows/rust.yml

@@ -171,21 +171,41 @@ jobs:
     name: cargo test (amd64)
     needs: linux-build-lib
     runs-on: ubuntu-latest
-    container:
-      image: amd64/rust
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 1
       - name: Setup Rust toolchain
-        uses: ./.github/actions/setup-builder
-        with:
-          rust-version: stable
+        run: rustup toolchain install stable
+      - name: Install Protobuf Compiler
+        run: sudo apt-get install -y protobuf-compiler
+      - name: Setup Minio - S3-compatible storage
+        run: |
+          docker run -d --name minio-container \
+            -p 9000:9000 \
+            -e MINIO_ROOT_USER=TEST-DataFusionLogin -e MINIO_ROOT_PASSWORD=TEST-DataFusionPassword \
+            -v $(pwd)/datafusion/core/tests/data:/source quay.io/minio/minio \
+            server /data
+          docker exec minio-container /bin/sh -c "\
+            mc ready local
+            mc alias set localminio http://localhost:9000 TEST-DataFusionLogin TEST-DataFusionPassword && \
+            mc mb localminio/data && \
+            mc cp -r /source/* localminio/data"
       - name: Run tests (excluding doctests)
-        run: cargo test --profile ci --exclude datafusion-examples --exclude ffi_example_table_provider --exclude datafusion-benchmarks --workspace --lib --tests --bins --features serde,avro,json,backtrace,integration-tests
+        env:
+          RUST_BACKTRACE: 1
+          AWS_ENDPOINT: http://127.0.0.1:9000
+          AWS_ACCESS_KEY_ID: TEST-DataFusionLogin
+          AWS_SECRET_ACCESS_KEY: TEST-DataFusionPassword
+          TEST_STORAGE_INTEGRATION: 1
+          AWS_ALLOW_HTTP: true
+        run: cargo test --profile ci --exclude datafusion-examples --exclude ffi_example_table_provider --exclude datafusion-benchmarks --workspace --lib --tests --bins --features avro,json,backtrace,integration-tests
       - name: Verify Working Directory Clean
         run: git diff --exit-code
+      - name: Minio Output
+        if: ${{ !cancelled() }}
+        run: docker logs minio-container
 
   linux-test-example:
     name: cargo examples (amd64)

Cargo.lock

@@ -0,0 +1,75 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# Development instructions
+
+## Running Tests
+
+Tests can be run using `cargo`
+
+```shell
+cargo test
+```
+
+## Running Storage Integration Tests
+
+By default, storage integration tests are not run. To run them you will need to set `TEST_STORAGE_INTEGRATION=1` and
+then provide the necessary configuration for that object store.
+
+For some of the tests, [snapshots](https://datafusion.apache.org/contributor-guide/testing.html#snapshot-testing) are used.
+
+### AWS
+
+To test the S3 integration against [Minio](https://github.com/minio/minio)
+
+First start up a container with Minio and load test files.
+
+```shell
+docker run -d \
+  --name datafusion-test-minio \
+  -p 9000:9000 \
+  -e MINIO_ROOT_USER=TEST-DataFusionLogin \
+  -e MINIO_ROOT_PASSWORD=TEST-DataFusionPassword \
+  -v $(pwd)/../datafusion/core/tests/data:/source \
+  quay.io/minio/minio server /data
+
+docker exec datafusion-test-minio /bin/sh -c "\
+  mc ready local
+  mc alias set localminio http://localhost:9000 TEST-DataFusionLogin TEST-DataFusionPassword && \
+  mc mb localminio/data && \
+  mc cp -r /source/* localminio/data"
+```
+
+Setup environment
+
+```shell
+export TEST_STORAGE_INTEGRATION=1
+export AWS_ACCESS_KEY_ID=TEST-DataFusionLogin
+export AWS_SECRET_ACCESS_KEY=TEST-DataFusionPassword
+export AWS_ENDPOINT=http://127.0.0.1:9000
+export AWS_ALLOW_HTTP=true
+```
+
+Note that `AWS_ENDPOINT` is set without slash at the end.
+
+Run tests
+
+```shell
+cargo test
+```

datafusion-cli/CONTRIBUTING.md

@@ -67,5 +67,7 @@ url = { workspace = true }
 [dev-dependencies]
 assert_cmd = "2.0"
 ctor = { workspace = true }
+insta = { version = "1.41.1", features = ["glob", "filters"] }
+insta-cmd = "0.6.0"
 predicates = "3.0"
 rstest = { workspace = true }

datafusion-cli/Cargo.toml

@@ -17,46 +17,131 @@
 
 use std::process::Command;
 
-use assert_cmd::prelude::{CommandCargoExt, OutputAssertExt};
-use predicates::prelude::predicate;
 use rstest::rstest;
 
+use insta::{glob, Settings};
+use insta_cmd::{assert_cmd_snapshot, get_cargo_bin};
+use std::{env, fs};
+
+fn cli() -> Command {
+    Command::new(get_cargo_bin("datafusion-cli"))
+}
+
+fn make_settings() -> Settings {
+    let mut settings = Settings::clone_current();
+    settings.set_prepend_module_to_snapshot(false);
+    settings.add_filter(r"Elapsed .* seconds\.", "[ELAPSED]");
+    settings.add_filter(r"DataFusion CLI v.*", "[CLI_VERSION]");
+    settings
+}
+
 #[cfg(test)]
 #[ctor::ctor]
 fn init() {
     // Enable RUST_LOG logging configuration for tests
     let _ = env_logger::try_init();
 }
 
-// Disabled due to https://github.com/apache/datafusion/issues/10793
-#[cfg(not(target_family = "windows"))]
 #[rstest]
-#[case::exec_from_commands(
-    ["--command", "select 1", "--format", "json", "-q"],
-    "[{\"Int64(1)\":1}]\n"
-)]
 #[case::exec_multiple_statements(
-    ["--command", "select 1; select 2;", "--format", "json", "-q"],
-    "[{\"Int64(1)\":1}]\n[{\"Int64(2)\":2}]\n"
+    "statements",
+    ["--command", "select 1; select 2;", "-q"],
 )]
 #[case::exec_backslash(
-    ["--file", "tests/data/backslash.txt", "--format", "json", "-q"],
-    "[{\"Utf8(\\\"\\\\\\\")\":\"\\\\\",\"Utf8(\\\"\\\\\\\\\\\")\":\"\\\\\\\\\",\"Utf8(\\\"\\\\\\\\\\\\\\\\\\\\\\\")\":\"\\\\\\\\\\\\\\\\\\\\\",\"Utf8(\\\"dsdsds\\\\\\\\\\\\\\\\\\\")\":\"dsdsds\\\\\\\\\\\\\\\\\",\"Utf8(\\\"\\\\t\\\")\":\"\\\\t\",\"Utf8(\\\"\\\\0\\\")\":\"\\\\0\",\"Utf8(\\\"\\\\n\\\")\":\"\\\\n\"}]\n"
+    "backslash",
+    ["--file", "tests/sql/backslash.sql", "--format", "json", "-q"],
 )]
 #[case::exec_from_files(
-    ["--file", "tests/data/sql.txt", "--format", "json", "-q"],
-    "[{\"Int64(1)\":1}]\n"
+    "files",
+    ["--file", "tests/sql/select.sql", "-q"],
 )]
 #[case::set_batch_size(
-    ["--command", "show datafusion.execution.batch_size", "--format", "json", "-q", "-b", "1"],
-    "[{\"name\":\"datafusion.execution.batch_size\",\"value\":\"1\"}]\n"
+    "batch_size",
+    ["--command", "show datafusion.execution.batch_size", "-q", "-b", "1"],
 )]
 #[test]
 fn cli_quick_test<'a>(
+    #[case] snapshot_name: &'a str,
     #[case] args: impl IntoIterator<Item = &'a str>,
-    #[case] expected: &str,
 ) {
-    let mut cmd = Command::cargo_bin("datafusion-cli").unwrap();
+    let mut settings = make_settings();
+    settings.set_snapshot_suffix(snapshot_name);
+    let _bound = settings.bind_to_scope();
+
+    let mut cmd = cli();
     cmd.args(args);
-    cmd.assert().stdout(predicate::eq(expected));
+
+    assert_cmd_snapshot!(cmd);
+}
+
+#[rstest]
+#[case("csv")]
+#[case("tsv")]
+#[case("table")]
+#[case("json")]
+#[case("nd-json")]
+#[case("automatic")]
+#[test]
+fn test_cli_format<'a>(#[case] format: &'a str) {
+    let mut settings = make_settings();
+    settings.set_snapshot_suffix(format);
+    let _bound = settings.bind_to_scope();
+
+    let mut cmd = cli();
+    cmd.args(["--command", "select 1", "-q", "--format", format]);
+
+    assert_cmd_snapshot!(cmd);
+}
+
+#[tokio::test]
+async fn test_cli() {
+    if env::var("TEST_STORAGE_INTEGRATION").is_err() {
+        eprintln!("Skipping external storages integration tests");
+        return;
+    }
+
+    let settings = make_settings();
+    let _bound = settings.bind_to_scope();
+
+    glob!("sql/integration/*.sql", |path| {
+        let input = fs::read_to_string(path).unwrap();
+        assert_cmd_snapshot!(cli().pass_stdin(input))
+    });
+}
+
+#[tokio::test]
+async fn test_aws_options() {
+    // Separate test is needed to pass aws as options in sql and not via env
+
+    if env::var("TEST_STORAGE_INTEGRATION").is_err() {
+        eprintln!("Skipping external storages integration tests");
+        return;
+    }
+
+    let settings = make_settings();
+    let _bound = settings.bind_to_scope();
+
+    let access_key_id =
+        env::var("AWS_ACCESS_KEY_ID").expect("AWS_ACCESS_KEY_ID is not set");
+    let secret_access_key =
+        env::var("AWS_SECRET_ACCESS_KEY").expect("AWS_SECRET_ACCESS_KEY is not set");
+    let endpoint_url = env::var("AWS_ENDPOINT").expect("AWS_ENDPOINT is not set");
+
+    let input = format!(
+        r#"CREATE EXTERNAL TABLE CARS
+STORED AS CSV
+LOCATION 's3://data/cars.csv'
+OPTIONS(
+    'aws.access_key_id' '{}',
+    'aws.secret_access_key' '{}',
+    'aws.endpoint' '{}',
+    'aws.allow_http' 'true'
+);
+
+SELECT * FROM CARS limit 1;
+"#,
+        access_key_id, secret_access_key, endpoint_url
+    );
+
+    assert_cmd_snapshot!(cli().env_clear().pass_stdin(input));
 }

datafusion-cli/tests/cli_integration.rs

@@ -0,0 +1,25 @@
+---
+source: tests/cli_integration.rs
+info:
+  program: datafusion-cli
+  args: []
+  stdin: "CREATE EXTERNAL TABLE CARS\nSTORED AS CSV\nLOCATION 's3://data/cars.csv'\nOPTIONS(\n    'aws.access_key_id' 'TEST-DataFusionLogin',\n    'aws.secret_access_key' 'TEST-DataFusionPassword',\n    'aws.endpoint' 'http://127.0.0.1:9000',\n    'aws.allow_http' 'true'\n);\n\nSELECT * FROM CARS limit 1;\n"
+---
+success: true
+exit_code: 0
+----- stdout -----
+[CLI_VERSION]
+0 row(s) fetched. 
+[ELAPSED]
+
++-----+-------+---------------------+
+| car | speed | time                |
++-----+-------+---------------------+
+| red | 20.0  | 1996-04-12T12:05:03 |
++-----+-------+---------------------+
+1 row(s) fetched. 
+[ELAPSED]
+
+\q
+
+----- stderr -----