Adds support for custom s3 endpoint. (#1093)

* Adds support for custom s3 endpoint.

This removes our trickery to use localstack,
and replaces it by a way to set a specific
endpoint via an environment variable:
`QW_S3_REGION`.

Closes #49
Closes #1037


Co-authored-by: Adrien Guillo <[email protected]>

Author: fulmicoton

.github/workflows/coverage.yml

@@ -12,6 +12,7 @@ on:
       - "quickwit-*/**"
 
 env:
+  QW_S3_ENDPOINT: "http://localstack:4566"
   AWS_DEFAULT_REGION  : "localhost"
   AWS_ACCESS_KEY_ID   : "placeholder"
   AWS_SECRET_ACCESS_KEY: "placeholder"

CONTRIBUTING.md

@@ -28,7 +28,7 @@ The process is simple and fast. Upon your first pull request, you will be prompt
 1. Install Docker (https://docs.docker.com/engine/install/) and Docker Compose (https://docs.docker.com/compose/install/)
 2. Install awslocal https://github.com/localstack/awscli-local
 3. Start the external services with `make docker-compose-up`
-5. Run `QW_ENV=LOCAL cargo test --all-features`
+5. Run `QW_S3_ENDPOINT=http://localhost:4566 cargo test --all-features`
 
 ## Running services such as Amazon Kinesis or S3, Kafka, or PostgreSQL locally.
 1. Ensure Docker and Docker Compose are correctly installed on your machine (see above)
@@ -48,8 +48,8 @@ The process is simple and fast. Upon your first pull request, you will be prompt
 ## Building binaries
 
 Currently, we use [cross](https://github.com/rust-embedded/cross) to build Quickwit binaries for different architectures.
-For this to work, we've had to customize the docker images cross uses. These customizations can be found in docker files located in `./cross-images` folder. To make cross take into account any change on those 
-docker files, you will need to build and push the images on dockerhub by running `make cross-images`. 
+For this to work, we've had to customize the docker images cross uses. These customizations can be found in docker files located in `./cross-images` folder. To make cross take into account any change on those
+docker files, you will need to build and push the images on dockerhub by running `make cross-images`.
 We also have nightly builds that are pushed to dockerhub. This helps continiously check our binaries are still built even with external dependency update. Successful builds let you accessed the artifacts for the next three days. Release builds always have their artifacts attached to the release.
 
 ## Testing release (alpha, beta, rc)

Makefile

@@ -30,7 +30,7 @@ fmt:
 # `make test-all` starts the Docker services and runs all the tests.
 # `make -k test-all docker-compose-down`, tears down the Docker services after running all the tests.
 test-all: docker-compose-up
-	QW_ENV=local AWS_ACCESS_KEY_ID=ignored AWS_SECRET_ACCESS_KEY=ignored cargo test --all-features
+	QW_S3_ENDPOINT=http://localhost:4566 AWS_ACCESS_KEY_ID=ignored AWS_SECRET_ACCESS_KEY=ignored cargo test --all-features
 
 # This will build and push all custom cross images for cross-compilation.
 # You will need to login into Docker Hub with the `quickwit` account.
@@ -59,7 +59,7 @@ build:
 	esac
 
 # Usage:
-# `BINARY_FILE=path/to/quickwit/binary BINARY_VERSION=0.1.0 ARCHIVE_NAME=quickwit make archive` 
+# `BINARY_FILE=path/to/quickwit/binary BINARY_VERSION=0.1.0 ARCHIVE_NAME=quickwit make archive`
 # - BINARY_FILE: Path of the quickwit binary file.
 # - BINARY_VERSION: Version of the quickwit binary.
 # - ARCHIVE_NAME: Name of the resulting archive file (without extension).

docs/reference/storage-uri.md

@@ -7,7 +7,7 @@ In Quickwit, Storage URIs refer to different kinds of storage.
 
 Generally speaking, you can use a storage URI or a regular file path wherever you would have expected a file path.
 
- 
+
 For instance
 
 - when configuring the index storage. (Passed as the `index_uri` in the index command line.)
@@ -26,7 +26,7 @@ The following are valid local file system URIs
 - /var/quickwit
 - file:///var/quickwit
 - /home/quickwit/data
-- ~/data 
+- ~/data
 - ./quickwit
 ```
 
@@ -64,13 +64,20 @@ Quickwit will detect the S3 credentials using the first successful method in thi
 
 ### Region
 
-The region will be detected using the first successful method in this list (order matters)
-
+The region or custom endpoint will be detected using the first successful method in this list (order matters)
 - `AWS_DEFAULT_REGION` environment variable
 - `AWS_REGION` environment variable
 - Amazon’s instance metadata API [https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html)
 
-:::caution
-Custom endpoints are not supported yet.
+### S3-compatible Objective storage like Minio
 
-:::
+
+Quickwit can target other S3-compatible storage like MinIO.
+This is done by setting an endpoint url in the `QW_S3_ENDPOINT` environment variable.
+
+In this case, the region will be ignored.
+
+For instance:
+```bash
+export QW_S3_ENDPOINT=http://localhost:9000/
+```

quickwit-cli/tests/cli.rs

@@ -709,7 +709,7 @@ async fn test_all_local_index() -> Result<()> {
 #[tokio::test]
 #[serial]
 #[cfg_attr(not(feature = "ci-test"), ignore)]
-async fn test_all_with_s3_localstack_cli() -> Result<()> {
+async fn test_cmd_all_with_s3_localstack_cli() -> Result<()> {
     let index_id = append_random_suffix("test-all--cli-s3-localstack");
     let test_env = create_test_env(index_id, TestStorageType::S3)?;
     make_command(
@@ -723,11 +723,11 @@ async fn test_all_with_s3_localstack_cli() -> Result<()> {
     .assert()
     .success();
 
-    let index_metadata = test_env
+    test_env
         .metastore()
         .index_metadata(&test_env.index_id)
-        .await;
-    assert_eq!(index_metadata.is_ok(), true);
+        .await
+        .unwrap();
 
     ingest_docs(test_env.resource_files["logs"].as_path(), &test_env);
 
@@ -797,7 +797,7 @@ async fn test_all_with_s3_localstack_cli() -> Result<()> {
 #[tokio::test]
 #[serial]
 #[cfg_attr(not(feature = "ci-test"), ignore)]
-async fn test_all_with_s3_localstack_internal_api() -> Result<()> {
+async fn test_cmd_all_with_s3_localstack_internal_api() -> Result<()> {
     let index_id = append_random_suffix("test-all--cli-API");
     let test_env = create_test_env(index_id, TestStorageType::S3)?;
     let args = CreateIndexArgs {

quickwit-cli/tests/helpers.rs

@@ -29,7 +29,7 @@ use predicates::str;
 use quickwit_common::net::find_available_port;
 use quickwit_common::uri::Uri;
 use quickwit_metastore::FileBackedMetastore;
-use quickwit_storage::{LocalFileStorage, RegionProvider, S3CompatibleObjectStorage, Storage};
+use quickwit_storage::{LocalFileStorage, S3CompatibleObjectStorage, Storage};
 use tempfile::{tempdir, TempDir};
 
 const PACKAGE_BIN_NAME: &str = "quickwit";
@@ -90,16 +90,13 @@ const WIKI_JSON_DOCS: &str = r#"{"body": "foo", "title": "shimroy", "url": "http
 {"body": "biz", "title": "modern", "url": "https://wiki.com?id=13"}
 "#;
 
-const AWS_DEFAULT_REGION_ENV: &str = "AWS_DEFAULT_REGION";
-
 /// Creates a quickwit-cli command with provided list of arguments.
 pub fn make_command(arguments: &str) -> Command {
     let mut cmd = Command::cargo_bin(PACKAGE_BIN_NAME).unwrap();
     cmd.env(
         quickwit_telemetry::DISABLE_TELEMETRY_ENV_KEY,
         "disable-for-tests",
     )
-    .env(AWS_DEFAULT_REGION_ENV, "us-east-1")
     .args(arguments.split_whitespace());
     cmd
 }
@@ -110,7 +107,6 @@ pub fn make_command_with_list_of_args(arguments: &[&str]) -> Command {
         quickwit_telemetry::DISABLE_TELEMETRY_ENV_KEY,
         "disable-for-tests",
     )
-    .env(AWS_DEFAULT_REGION_ENV, "us-east-1")
     .args(arguments.iter());
     cmd
 }
@@ -123,8 +119,6 @@ pub fn spawn_command(arguments: &str) -> io::Result<Child> {
             quickwit_telemetry::DISABLE_TELEMETRY_ENV_KEY,
             "disable-for-tests",
         )
-        .env(AWS_DEFAULT_REGION_ENV, "us-east-1")
-        .stdout(Stdio::piped())
         .stdout(Stdio::piped())
         .spawn()
 }
@@ -179,15 +173,13 @@ pub fn create_test_env(index_id: String, storage_type: TestStorageType) -> anyho
             (metastore_uri, storage)
         }
         TestStorageType::S3 => {
-            let metastore_uri = "s3+localstack://quickwit-integration-tests/indexes";
-            let storage: Arc<dyn Storage> = Arc::new(S3CompatibleObjectStorage::from_uri(
-                RegionProvider::Localstack.get_region(),
-                metastore_uri,
-            )?);
+            let metastore_uri = "s3://quickwit-integration-tests/indexes";
+            let storage: Arc<dyn Storage> =
+                Arc::new(S3CompatibleObjectStorage::from_uri(metastore_uri)?);
             (metastore_uri.to_string(), storage)
         }
     };
-    let index_uri = Uri::try_new(&format!("{}/{}", metastore_uri, index_id))?;
+    let index_uri = Uri::try_new(&format!("{metastore_uri}/{index_id}"))?;
     let index_config_path = resources_dir_path.join("index_config.yaml");
     fs::write(
         &index_config_path,

quickwit-common/src/lib.rs

@@ -33,26 +33,6 @@ pub use checklist::{print_checklist, run_checklist, BLUE_COLOR, GREEN_COLOR, RED
 pub use coolid::new_coolid;
 use tracing::{error, info};
 
-#[derive(Debug, PartialEq, Eq)]
-pub enum QuickwitEnv {
-    UNSET,
-    LOCAL,
-}
-
-impl Default for QuickwitEnv {
-    fn default() -> Self {
-        Self::UNSET
-    }
-}
-
-pub fn get_quickwit_env() -> QuickwitEnv {
-    match std::env::var("QW_ENV") {
-        Ok(val) if val.to_lowercase().trim() == "local" => QuickwitEnv::LOCAL,
-        Ok(val) => panic!("QW_ENV value `{}` is not supported", val),
-        Err(_) => QuickwitEnv::UNSET,
-    }
-}
-
 pub fn chunk_range(range: Range<usize>, chunk_size: usize) -> impl Iterator<Item = Range<usize>> {
     range.clone().step_by(chunk_size).map(move |block_start| {
         let block_end = (block_start + chunk_size).min(range.end);

quickwit-metastore/src/metastore_resolver.rs

@@ -71,12 +71,12 @@ pub fn quickwit_metastore_uri_resolver() -> &'static MetastoreUriResolver {
         let mut builder = MetastoreUriResolver::builder()
             .register("ram", FileBackedMetastoreFactory::default())
             .register("file", FileBackedMetastoreFactory::default())
-            .register("s3", FileBackedMetastoreFactory::default())
-            .register("s3+localstack", FileBackedMetastoreFactory::default());
+            .register("s3", FileBackedMetastoreFactory::default());
         #[cfg(feature = "postgres")]
         {
-            builder = builder.register("postgres", PostgresqlMetastoreFactory::default());
-            builder = builder.register("postgresql", PostgresqlMetastoreFactory::default());
+            builder = builder
+                .register("postgres", PostgresqlMetastoreFactory::default())
+                .register("postgresql", PostgresqlMetastoreFactory::default());
         }
 
         builder.build()
@@ -113,13 +113,9 @@ mod tests {
     use crate::quickwit_metastore_uri_resolver;
 
     #[tokio::test]
-    async fn test_metastore_resolver_should_not_raise_errors_on_file_and_s3() -> anyhow::Result<()>
-    {
+    async fn test_metastore_resolver_should_not_raise_errors_on_file() -> anyhow::Result<()> {
         let metastore_resolver = quickwit_metastore_uri_resolver();
         metastore_resolver.resolve("file://").await?;
-        metastore_resolver
-            .resolve("s3://bucket/path/to/object")
-            .await?;
         Ok(())
     }
 

quickwit-serve/src/lib.rs

@@ -32,9 +32,7 @@ use quickwit_cluster::service::ClusterServiceImpl;
 use quickwit_config::{QuickwitConfig, SEARCHER_CONFIG_INSTANCE};
 use quickwit_metastore::Metastore;
 use quickwit_search::{ClusterClient, SearchClientPool, SearchServiceImpl};
-use quickwit_storage::{
-    LocalFileStorageFactory, RegionProvider, S3CompatibleObjectStorageFactory, StorageUriResolver,
-};
+use quickwit_storage::quickwit_storage_uri_resolver;
 use tracing::{debug, info};
 
 pub use crate::args::ServeArgs;
@@ -45,22 +43,6 @@ use crate::grpc_adapter::cluster_adapter::GrpcClusterAdapter;
 use crate::grpc_adapter::search_adapter::GrpcSearchAdapter;
 use crate::rest::start_rest_service;
 
-/// Builds a storage uri resolver that handles
-/// - s3:// uris. This storage comes with a cache that stores hotcache files.
-/// - s3+localstack://
-/// - file:// uris.
-fn storage_uri_resolver() -> StorageUriResolver {
-    let s3_storage = S3CompatibleObjectStorageFactory::default();
-    StorageUriResolver::builder()
-        .register(LocalFileStorageFactory::default())
-        .register(s3_storage)
-        .register(S3CompatibleObjectStorageFactory::new(
-            RegionProvider::Localstack,
-            "s3+localstack",
-        ))
-        .build()
-}
-
 /// Starts a search node, aka a `searcher`.
 pub async fn run_searcher(
     quickwit_config: QuickwitConfig,
@@ -79,7 +61,7 @@ pub async fn run_searcher(
         debug!(peer_seed_addr = %seed_socket_addr, "Add peer seed node.");
         cluster.add_peer_node(seed_socket_addr).await;
     }
-    let storage_uri_resolver = storage_uri_resolver();
+    let storage_uri_resolver = quickwit_storage_uri_resolver().clone();
     let client_pool = Arc::new(SearchClientPool::new(cluster.clone()).await?);
     let cluster_client = ClusterClient::new(client_pool.clone());
     let search_service = Arc::new(SearchServiceImpl::new(

quickwit-storage/src/error.rs

@@ -30,7 +30,7 @@ pub enum StorageErrorKind {
     DoesNotExist,
     /// The request credentials do not allow for this operation.
     Unauthorized,
-    /// A third-party service forbids this operation.
+    /// A third-party service forbids this operation, or is misconfigured.
     Service,
     /// Any generic internal error.
     InternalError,