diff --git a/CONTROLLERS.md b/CONTROLLERS.md
index 587470fdde..952064b848 100644
--- a/CONTROLLERS.md
+++ b/CONTROLLERS.md
@@ -27,6 +27,7 @@ limitations under the License.
- [NetworkPrioritizerService](#NetworkPrioritizerService)
- [ODBCService](#ODBCService)
- [PersistentMapStateStorage](#PersistentMapStateStorage)
+- [ProxyConfigurationService](#ProxyConfigurationService)
- [RocksDbStateStorage](#RocksDbStateStorage)
- [SmbConnectionControllerService](#SmbConnectionControllerService)
- [SSLContextService](#SSLContextService)
@@ -244,6 +245,24 @@ In the list below, the names of required properties appear in bold. Any other pr
| **File** | | | Path to a file to store state |
+## ProxyConfigurationService
+
+### Description
+
+Provides a set of configurations for different MiNiFi C++ components to use a proxy server. Currently these properties can only be used for HTTP proxy configuration, not other protocols are supported at this time.
+
+### Properties
+
+In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language.
+
+| Name | Default Value | Allowable Values | Description |
+|-----------------------|---------------|------------------|--------------------------------------------------------------------------------------------|
+| **Proxy Server Host** | | | Proxy server hostname or ip-address. |
+| Proxy Server Port | | | Proxy server port number. |
+| Proxy User Name | | | The name of the proxy client for user authentication. |
+| Proxy User Password | | | The password of the proxy client for user authentication.
**Sensitive Property: true** |
+
+
## RocksDbStateStorage
### Description
diff --git a/PROCESSORS.md b/PROCESSORS.md
index 1fc8af2383..daa5007bf8 100644
--- a/PROCESSORS.md
+++ b/PROCESSORS.md
@@ -553,6 +553,7 @@ In the list below, the names of required properties appear in bold. Any other pr
| Name | Default Value | Allowable Values | Description |
|----------------------------------------|-----------------|-----------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. |
+| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. |
| **Container Name** | | | Name of the Azure Storage container. In case of PutAzureBlobStorage processor, container can be created if it does not exist.
**Supports Expression Language: true** |
| Storage Account Name | | | The storage account name.
**Supports Expression Language: true** |
| Storage Account Key | | | The storage account key. This is an admin-like password providing access to every container in this account. It is recommended one uses Shared Access Signature (SAS) token instead for fine-grained control with policies if Credential Configuration Strategy is set to From Properties. If set, SAS Token must be empty.
**Sensitive Property: true**
**Supports Expression Language: true** |
@@ -585,6 +586,7 @@ In the list below, the names of required properties appear in bold. Any other pr
| Name | Default Value | Allowable Values | Description |
|-----------------------------------|---------------|------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. |
+| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. |
| **Filesystem Name** | | | Name of the Azure Storage File System. It is assumed to be already existing.
**Supports Expression Language: true** |
| Directory Name | | | Name of the Azure Storage Directory. The Directory Name cannot contain a leading '/'. If left empty it designates the root directory. The directory will be created if not already existing.
**Supports Expression Language: true** |
| File Name | | | The filename in Azure Storage. If left empty the filename attribute will be used by default.
**Supports Expression Language: true** |
@@ -645,7 +647,6 @@ In the list below, the names of required properties appear in bold. Any other pr
| Name | Default Value | Allowable Values | Description |
|----------------------------------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| **Bucket** | | | The S3 bucket
**Supports Expression Language: true** |
| Access Key | | | AWS account access key
**Supports Expression Language: true** |
| Secret Key | | | AWS account secret key
**Sensitive Property: true**
**Supports Expression Language: true** |
| Credentials File | | | Path to a file containing AWS access key and secret key in properties file format. Properties used: accessKey and secretKey |
@@ -657,7 +658,9 @@ In the list below, the names of required properties appear in bold. Any other pr
| Proxy Port | | | The port number of the proxy host
**Supports Expression Language: true** |
| Proxy Username | | | Username to set when authenticating against proxy
**Supports Expression Language: true** |
| Proxy Password | | | Password to set when authenticating against proxy
**Sensitive Property: true**
**Supports Expression Language: true** |
+| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. When used, this will override any values specified for Proxy Host, Proxy Port, Proxy Username, and Proxy Password properties. |
| **Use Default Credentials** | false | true
false | If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc. |
+| **Bucket** | | | The S3 bucket
**Supports Expression Language: true** |
| Object Key | | | The key of the S3 object. If none is given the filename attribute will be used by default.
**Supports Expression Language: true** |
| Version | | | The Version of the Object to delete
**Supports Expression Language: true** |
@@ -817,6 +820,7 @@ In the list below, the names of required properties appear in bold. Any other pr
| Name | Default Value | Allowable Values | Description |
|----------------------------------------|-----------------|-----------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. |
+| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. |
| **Container Name** | | | Name of the Azure Storage container. In case of PutAzureBlobStorage processor, container can be created if it does not exist.
**Supports Expression Language: true** |
| Storage Account Name | | | The storage account name.
**Supports Expression Language: true** |
| Storage Account Key | | | The storage account key. This is an admin-like password providing access to every container in this account. It is recommended one uses Shared Access Signature (SAS) token instead for fine-grained control with policies if Credential Configuration Strategy is set to From Properties. If set, SAS Token must be empty.
**Sensitive Property: true**
**Supports Expression Language: true** |
@@ -850,6 +854,7 @@ In the list below, the names of required properties appear in bold. Any other pr
| Name | Default Value | Allowable Values | Description |
|-----------------------------------|---------------|------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. |
+| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. |
| **Filesystem Name** | | | Name of the Azure Storage File System. It is assumed to be already existing.
**Supports Expression Language: true** |
| Directory Name | | | Name of the Azure Storage Directory. The Directory Name cannot contain a leading '/'. If left empty it designates the root directory. The directory will be created if not already existing.
**Supports Expression Language: true** |
| File Name | | | The filename in Azure Storage. If left empty the filename attribute will be used by default.
**Supports Expression Language: true** |
@@ -1017,7 +1022,6 @@ In the list below, the names of required properties appear in bold. Any other pr
| Name | Default Value | Allowable Values | Description |
|----------------------------------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| **Bucket** | | | The S3 bucket
**Supports Expression Language: true** |
| Access Key | | | AWS account access key
**Supports Expression Language: true** |
| Secret Key | | | AWS account secret key
**Sensitive Property: true**
**Supports Expression Language: true** |
| Credentials File | | | Path to a file containing AWS access key and secret key in properties file format. Properties used: accessKey and secretKey |
@@ -1029,7 +1033,9 @@ In the list below, the names of required properties appear in bold. Any other pr
| Proxy Port | | | The port number of the proxy host
**Supports Expression Language: true** |
| Proxy Username | | | Username to set when authenticating against proxy
**Supports Expression Language: true** |
| Proxy Password | | | Password to set when authenticating against proxy
**Sensitive Property: true**
**Supports Expression Language: true** |
+| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. When used, this will override any values specified for Proxy Host, Proxy Port, Proxy Username, and Proxy Password properties. |
| **Use Default Credentials** | false | true
false | If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc. |
+| **Bucket** | | | The S3 bucket
**Supports Expression Language: true** |
| Object Key | | | The key of the S3 object. If none is given the filename attribute will be used by default.
**Supports Expression Language: true** |
| Version | | | The Version of the Object to download
**Supports Expression Language: true** |
| **Requester Pays** | false | true
false | If true, indicates that the requester consents to pay any charges associated with retrieving objects from the S3 bucket. This sets the 'x-amz-request-payer' header to 'requester'. |
@@ -1384,6 +1390,7 @@ In the list below, the names of required properties appear in bold. Any other pr
| Name | Default Value | Allowable Values | Description |
|----------------------------------------|-----------------|-----------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. |
+| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. |
| **Container Name** | | | Name of the Azure Storage container. In case of PutAzureBlobStorage processor, container can be created if it does not exist.
**Supports Expression Language: true** |
| Storage Account Name | | | The storage account name.
**Supports Expression Language: true** |
| Storage Account Key | | | The storage account key. This is an admin-like password providing access to every container in this account. It is recommended one uses Shared Access Signature (SAS) token instead for fine-grained control with policies if Credential Configuration Strategy is set to From Properties. If set, SAS Token must be empty.
**Sensitive Property: true**
**Supports Expression Language: true** |
@@ -1415,6 +1422,7 @@ In the list below, the names of required properties appear in bold. Any other pr
| Name | Default Value | Allowable Values | Description |
|-----------------------------------|---------------|---------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. |
+| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. |
| **Filesystem Name** | | | Name of the Azure Storage File System. It is assumed to be already existing.
**Supports Expression Language: true** |
| Directory Name | | | Name of the Azure Storage Directory. The Directory Name cannot contain a leading '/'. If left empty it designates the root directory. The directory will be created if not already existing.
**Supports Expression Language: true** |
| **Recurse Subdirectories** | true | true
false | Indicates whether to list files from subdirectories of the directory |
@@ -1677,7 +1685,6 @@ In the list below, the names of required properties appear in bold. Any other pr
| Name | Default Value | Allowable Values | Description |
|----------------------------------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| **Bucket** | | | The S3 bucket
**Supports Expression Language: true** |
| Access Key | | | AWS account access key
**Supports Expression Language: true** |
| Secret Key | | | AWS account secret key
**Sensitive Property: true**
**Supports Expression Language: true** |
| Credentials File | | | Path to a file containing AWS access key and secret key in properties file format. Properties used: accessKey and secretKey |
@@ -1689,7 +1696,9 @@ In the list below, the names of required properties appear in bold. Any other pr
| Proxy Port | | | The port number of the proxy host
**Supports Expression Language: true** |
| Proxy Username | | | Username to set when authenticating against proxy
**Supports Expression Language: true** |
| Proxy Password | | | Password to set when authenticating against proxy
**Sensitive Property: true**
**Supports Expression Language: true** |
+| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. When used, this will override any values specified for Proxy Host, Proxy Port, Proxy Username, and Proxy Password properties. |
| **Use Default Credentials** | false | true
false | If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc. |
+| **Bucket** | | | The S3 bucket
**Supports Expression Language: true** |
| Delimiter | | | The string used to delimit directories within the bucket. Please consult the AWS documentation for the correct use of this field. |
| Prefix | | | The prefix used to filter the object list. In most cases, it should end with a forward slash ('/'). |
| **Use Versions** | false | true
false | Specifies whether to use S3 versions, if applicable. If false, only the latest version of each object will be returned. |
@@ -2312,6 +2321,7 @@ In the list below, the names of required properties appear in bold. Any other pr
| Name | Default Value | Allowable Values | Description |
|----------------------------------------|-----------------|-----------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. |
+| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. |
| **Container Name** | | | Name of the Azure Storage container. In case of PutAzureBlobStorage processor, container can be created if it does not exist.
**Supports Expression Language: true** |
| Storage Account Name | | | The storage account name.
**Supports Expression Language: true** |
| Storage Account Key | | | The storage account key. This is an admin-like password providing access to every container in this account. It is recommended one uses Shared Access Signature (SAS) token instead for fine-grained control with policies if Credential Configuration Strategy is set to From Properties. If set, SAS Token must be empty.
**Sensitive Property: true**
**Supports Expression Language: true** |
@@ -2344,6 +2354,7 @@ In the list below, the names of required properties appear in bold. Any other pr
| Name | Default Value | Allowable Values | Description |
|-----------------------------------|---------------|-----------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. |
+| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. |
| **Filesystem Name** | | | Name of the Azure Storage File System. It is assumed to be already existing.
**Supports Expression Language: true** |
| Directory Name | | | Name of the Azure Storage Directory. The Directory Name cannot contain a leading '/'. If left empty it designates the root directory. The directory will be created if not already existing.
**Supports Expression Language: true** |
| File Name | | | The filename in Azure Storage. If left empty the filename attribute will be used by default.
**Supports Expression Language: true** |
@@ -2582,7 +2593,6 @@ In the list below, the names of required properties appear in bold. Any other pr
| Name | Default Value | Allowable Values | Description |
|----------------------------------------|--------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| **Bucket** | | | The S3 bucket
**Supports Expression Language: true** |
| Access Key | | | AWS account access key
**Supports Expression Language: true** |
| Secret Key | | | AWS account secret key
**Sensitive Property: true**
**Supports Expression Language: true** |
| Credentials File | | | Path to a file containing AWS access key and secret key in properties file format. Properties used: accessKey and secretKey |
@@ -2594,7 +2604,9 @@ In the list below, the names of required properties appear in bold. Any other pr
| Proxy Port | | | The port number of the proxy host
**Supports Expression Language: true** |
| Proxy Username | | | Username to set when authenticating against proxy
**Supports Expression Language: true** |
| Proxy Password | | | Password to set when authenticating against proxy
**Sensitive Property: true**
**Supports Expression Language: true** |
+| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. When used, this will override any values specified for Proxy Host, Proxy Port, Proxy Username, and Proxy Password properties. |
| **Use Default Credentials** | false | true
false | If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc. |
+| **Bucket** | | | The S3 bucket
**Supports Expression Language: true** |
| Object Key | | | The key of the S3 object. If none is given the filename attribute will be used by default.
**Supports Expression Language: true** |
| Content Type | application/octet-stream | | Sets the Content-Type HTTP header indicating the type of content stored in the associated object. The value of this header is a standard MIME type. If no content type is provided the default content type "application/octet-stream" will be used.
**Supports Expression Language: true** |
| **Storage Class** | Standard | Standard
ReducedRedundancy
StandardIA
OnezoneIA
IntelligentTiering
Glacier
DeepArchive
Outposts
GlacierIR
Snow
ExpressOneZone | AWS S3 Storage Class |
diff --git a/behave_framework/src/minifi_test_framework/steps/flow_building_steps.py b/behave_framework/src/minifi_test_framework/steps/flow_building_steps.py
index e2ac9df508..1905f15d21 100644
--- a/behave_framework/src/minifi_test_framework/steps/flow_building_steps.py
+++ b/behave_framework/src/minifi_test_framework/steps/flow_building_steps.py
@@ -192,10 +192,25 @@ def step_impl(context: MinifiTestContext):
@step("the http proxy server is set up")
-def step_impl(context):
+def step_impl(context: MinifiTestContext):
context.containers["http-proxy"] = HttpProxy(context)
+@given("a ProxyConfigurationService controller service is set up with HTTP proxy configuration in the \"{container_name}\" flow")
+def step_impl(context: MinifiTestContext, container_name: str):
+ controller_service = ControllerService(class_name="ProxyConfigurationService", service_name="ProxyConfigurationService")
+ controller_service.add_property("Proxy Server Host", f"http-proxy-{context.scenario_id}")
+ controller_service.add_property("Proxy Server Port", "3128")
+ controller_service.add_property("Proxy User Name", "admin")
+ controller_service.add_property("Proxy User Password", "test101")
+ context.get_or_create_minifi_container(container_name).flow_definition.controller_services.append(controller_service)
+
+
+@given("a ProxyConfigurationService controller service is set up with HTTP proxy configuration")
+def step_impl(context: MinifiTestContext):
+ context.execute_steps(f"given a ProxyConfigurationService controller service is set up with HTTP proxy configuration in the \"{DEFAULT_MINIFI_CONTAINER_NAME}\" flow")
+
+
@step("the processors are connected up as described here")
def step_impl(context: MinifiTestContext):
for row in context.table:
diff --git a/core-framework/include/controllers/ProxyConfiguration.h b/core-framework/include/controllers/ProxyConfiguration.h
new file mode 100644
index 0000000000..0090cf45d0
--- /dev/null
+++ b/core-framework/include/controllers/ProxyConfiguration.h
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include
+#include
+
+#include "minifi-cpp/controllers/ProxyConfigurationServiceInterface.h"
+
+namespace org::apache::nifi::minifi::controllers {
+
+struct ProxyConfiguration {
+ ProxyType proxy_type;
+ std::string proxy_host;
+ std::optional proxy_port;
+ std::optional proxy_user;
+ std::optional proxy_password;
+};
+
+} // namespace org::apache::nifi::minifi::controllers
diff --git a/docker/test/integration/features/steps/steps.py b/docker/test/integration/features/steps/steps.py
index f317294f20..744f2b31ac 100644
--- a/docker/test/integration/features/steps/steps.py
+++ b/docker/test/integration/features/steps/steps.py
@@ -25,6 +25,7 @@
from minifi.controllers.XMLReader import XMLReader
from minifi.controllers.XMLRecordSetWriter import XMLRecordSetWriter
from minifi.controllers.XMLReader import XMLReader
+from minifi.controllers.ProxyConfigurationService import ProxyConfigurationService
from behave import given, then, when
from behave.model_describe import ModelDescriptor
@@ -406,6 +407,18 @@ def step_impl(context):
context.test.acquire_container(context=context, name="http-proxy", engine="http-proxy")
+@given("a ProxyConfigurationService controller service is set up with HTTP proxy configuration in the \"{container_name}\" flow")
+def step_impl(context, container_name):
+ proxy_service = ProxyConfigurationService("ProxyConfigurationService", host=f"http-proxy-{context.feature_id}", port=3128, username="admin", password="test101")
+ container = context.test.acquire_container(context=context, name=container_name)
+ container.add_controller(proxy_service)
+
+
+@given("a ProxyConfigurationService controller service is set up with HTTP proxy configuration")
+def step_impl(context):
+ context.execute_steps("given a ProxyConfigurationService controller service is set up with HTTP proxy configuration in the \"{container_name}\" flow".format(container_name="minifi-cpp-flow"))
+
+
# TLS
@given("an ssl context service is set up for {processor_name}")
@given("an ssl context service with a manual CA cert file is set up for {processor_name}")
diff --git a/docker/test/integration/minifi/controllers/ProxyConfigurationService.py b/docker/test/integration/minifi/controllers/ProxyConfigurationService.py
new file mode 100644
index 0000000000..d0e3f457ec
--- /dev/null
+++ b/docker/test/integration/minifi/controllers/ProxyConfigurationService.py
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from ..core.ControllerService import ControllerService
+
+
+class ProxyConfigurationService(ControllerService):
+ def __init__(self, name, host, port=None, username=None, password=None):
+ super(ProxyConfigurationService, self).__init__(name=name)
+
+ self.service_class = 'ProxyConfigurationService'
+
+ self.properties['Proxy Server Host'] = host
+
+ if port is not None:
+ self.properties['Proxy Server Port'] = port
+
+ if username is not None:
+ self.properties['Proxy User Name'] = username
+
+ if password is not None:
+ self.properties['Proxy User Password'] = password
diff --git a/extensions/aws/processors/AwsProcessor.cpp b/extensions/aws/processors/AwsProcessor.cpp
index 6c0a1b19e8..f2b2980f28 100644
--- a/extensions/aws/processors/AwsProcessor.cpp
+++ b/extensions/aws/processors/AwsProcessor.cpp
@@ -69,10 +69,21 @@ std::optional AwsProcessor::getAWSCredentials(
aws::ProxyOptions AwsProcessor::getProxy(core::ProcessContext& context, const core::FlowFile* const flow_file) {
aws::ProxyOptions proxy;
- proxy.host = minifi::utils::parseOptionalProperty(context, ProxyHost, flow_file).value_or("");
- proxy.port = gsl::narrow(minifi::utils::parseOptionalU64Property(context, ProxyPort, flow_file).value_or(0));
- proxy.username = minifi::utils::parseOptionalProperty(context, ProxyUsername, flow_file).value_or("");
- proxy.password = minifi::utils::parseOptionalProperty(context, ProxyPassword, flow_file).value_or("");
+ auto proxy_controller_service = minifi::utils::parseOptionalControllerService(context, ProxyConfigurationService, getUUID());
+ if (proxy_controller_service) {
+ proxy.host = proxy_controller_service->getHost();
+ auto port_opt = proxy_controller_service->getPort();
+ proxy.port = port_opt ? *port_opt : 0;
+ auto username_opt = proxy_controller_service->getUsername();
+ proxy.username = username_opt ? *username_opt : "";
+ auto password_opt = proxy_controller_service->getPassword();
+ proxy.password = password_opt ? *password_opt : "";
+ } else {
+ proxy.host = minifi::utils::parseOptionalProperty(context, ProxyHost, flow_file).value_or("");
+ proxy.port = gsl::narrow(minifi::utils::parseOptionalU64Property(context, ProxyPort, flow_file).value_or(0));
+ proxy.username = minifi::utils::parseOptionalProperty(context, ProxyUsername, flow_file).value_or("");
+ proxy.password = minifi::utils::parseOptionalProperty(context, ProxyPassword, flow_file).value_or("");
+ }
if (!proxy.host.empty()) {
logger_->log_info("Proxy for AwsProcessor was set.");
diff --git a/extensions/aws/processors/AwsProcessor.h b/extensions/aws/processors/AwsProcessor.h
index 3637fdc343..548f332795 100644
--- a/extensions/aws/processors/AwsProcessor.h
+++ b/extensions/aws/processors/AwsProcessor.h
@@ -33,6 +33,7 @@
#include "core/PropertyDefinitionBuilder.h"
#include "minifi-cpp/core/PropertyValidator.h"
#include "core/ProcessorImpl.h"
+#include "minifi-cpp/controllers/ProxyConfigurationServiceInterface.h"
namespace org::apache::nifi::minifi::aws::processors {
@@ -148,6 +149,11 @@ class AwsProcessor : public core::ProcessorImpl { // NOLINT(cppcoreguidelines-s
.supportsExpressionLanguage(true)
.isSensitive(true)
.build();
+ EXTENSIONAPI static constexpr auto ProxyConfigurationService = core::PropertyDefinitionBuilder<>::createProperty("Proxy Configuration Service")
+ .withDescription("Specifies the Proxy Configuration Controller Service to proxy network requests. When used, "
+ "this will override any values specified for Proxy Host, Proxy Port, Proxy Username, and Proxy Password properties.")
+ .withAllowedTypes()
+ .build();
EXTENSIONAPI static constexpr auto UseDefaultCredentials = core::PropertyDefinitionBuilder<>::createProperty("Use Default Credentials")
.withDescription("If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc.")
.withValidator(core::StandardPropertyValidators::BOOLEAN_VALIDATOR)
@@ -166,6 +172,7 @@ class AwsProcessor : public core::ProcessorImpl { // NOLINT(cppcoreguidelines-s
ProxyPort,
ProxyUsername,
ProxyPassword,
+ ProxyConfigurationService,
UseDefaultCredentials
});
diff --git a/extensions/aws/tests/DeleteS3ObjectTests.cpp b/extensions/aws/tests/DeleteS3ObjectTests.cpp
index bf06e060a2..de39899f9f 100644
--- a/extensions/aws/tests/DeleteS3ObjectTests.cpp
+++ b/extensions/aws/tests/DeleteS3ObjectTests.cpp
@@ -84,7 +84,12 @@ TEST_CASE_METHOD(DeleteS3ObjectTestsFixture, "Non blank validator tests") {
TEST_CASE_METHOD(DeleteS3ObjectTestsFixture, "Test proxy setting", "[awsS3Proxy]") {
setRequiredProperties();
- setProxy();
+ SECTION("Use proxy configuration service") {
+ setProxy(true);
+ }
+ SECTION("Use processor properties") {
+ setProxy(false);
+ }
test_controller.runSession(plan, true);
checkProxySettings();
}
diff --git a/extensions/aws/tests/FetchS3ObjectTests.cpp b/extensions/aws/tests/FetchS3ObjectTests.cpp
index 5920c13c57..856ccf68f0 100644
--- a/extensions/aws/tests/FetchS3ObjectTests.cpp
+++ b/extensions/aws/tests/FetchS3ObjectTests.cpp
@@ -101,7 +101,12 @@ TEST_CASE_METHOD(FetchS3ObjectTestsFixture, "Non blank validator tests") {
TEST_CASE_METHOD(FetchS3ObjectTestsFixture, "Test proxy setting", "[awsS3Proxy]") {
setRequiredProperties();
- setProxy();
+ SECTION("Use proxy configuration service") {
+ setProxy(true);
+ }
+ SECTION("Use processor properties") {
+ setProxy(false);
+ }
test_controller.runSession(plan, true);
checkProxySettings();
}
diff --git a/extensions/aws/tests/ListS3Tests.cpp b/extensions/aws/tests/ListS3Tests.cpp
index 424e83fd53..752b329449 100644
--- a/extensions/aws/tests/ListS3Tests.cpp
+++ b/extensions/aws/tests/ListS3Tests.cpp
@@ -74,7 +74,12 @@ TEST_CASE_METHOD(ListS3TestsFixture, "Non blank validator tests") {
TEST_CASE_METHOD(ListS3TestsFixture, "Test proxy setting", "[awsS3Proxy]") {
setRequiredProperties();
- setProxy();
+ SECTION("Use proxy configuration service") {
+ setProxy(true);
+ }
+ SECTION("Use processor properties") {
+ setProxy(false);
+ }
test_controller.runSession(plan, true);
checkProxySettings();
}
diff --git a/extensions/aws/tests/PutS3ObjectTests.cpp b/extensions/aws/tests/PutS3ObjectTests.cpp
index a78b0cdd07..75bbba02e7 100644
--- a/extensions/aws/tests/PutS3ObjectTests.cpp
+++ b/extensions/aws/tests/PutS3ObjectTests.cpp
@@ -200,7 +200,12 @@ TEST_CASE_METHOD(PutS3ObjectTestsFixture, "Test multiple user metadata", "[awsS3
TEST_CASE_METHOD(PutS3ObjectTestsFixture, "Test proxy setting", "[awsS3Proxy]") {
setRequiredProperties();
- setProxy();
+ SECTION("Use proxy configuration service") {
+ setProxy(true);
+ }
+ SECTION("Use processor properties") {
+ setProxy(false);
+ }
test_controller.runSession(plan);
checkProxySettings();
}
diff --git a/extensions/aws/tests/S3TestsFixture.h b/extensions/aws/tests/S3TestsFixture.h
index 360e2722f8..68d01aac3c 100644
--- a/extensions/aws/tests/S3TestsFixture.h
+++ b/extensions/aws/tests/S3TestsFixture.h
@@ -96,7 +96,7 @@ class S3TestsFixture {
virtual void setAccesKeyCredentialsInProcessor() = 0;
virtual void setBucket() = 0;
- virtual void setProxy() = 0;
+ virtual void setProxy(bool use_controller_service) = 0;
void setRequiredProperties() {
setAccesKeyCredentialsInProcessor();
@@ -136,7 +136,8 @@ class FlowProcessorS3TestsFixture : public S3TestsFixture {
auto mock_s3_request_sender = std::make_unique();
this->mock_s3_request_sender_ptr = mock_s3_request_sender.get();
auto uuid = utils::IdGenerator::getIdGenerator()->generate();
- auto impl = std::unique_ptr(new T(core::ProcessorMetadata{.uuid = uuid, .name = "S3Processor", .logger = core::logging::LoggerFactory::getLogger(uuid)}, std::move(mock_s3_request_sender)));
+ auto impl = std::unique_ptr(new T(core::ProcessorMetadata{ // NOLINT(clang-analyzer-cplusplus.NewDeleteLeaks)
+ .uuid = uuid, .name = "S3Processor", .logger = core::logging::LoggerFactory::getLogger(uuid)}, std::move(mock_s3_request_sender)));
auto s3_processor_unique_ptr = std::make_unique("S3Processor", uuid, std::move(impl));
this->s3_processor = s3_processor_unique_ptr.get();
@@ -178,15 +179,24 @@ class FlowProcessorS3TestsFixture : public S3TestsFixture {
this->plan->setProperty(this->s3_processor, "Bucket", "${test.bucket}");
}
- void setProxy() override {
- this->plan->setDynamicProperty(update_attribute, "test.proxyHost", "host");
- this->plan->setProperty(this->s3_processor, "Proxy Host", "${test.proxyHost}");
- this->plan->setDynamicProperty(update_attribute, "test.proxyPort", "1234");
- this->plan->setProperty(this->s3_processor, "Proxy Port", "${test.proxyPort}");
- this->plan->setDynamicProperty(update_attribute, "test.proxyUsername", "username");
- this->plan->setProperty(this->s3_processor, "Proxy Username", "${test.proxyUsername}");
- this->plan->setDynamicProperty(update_attribute, "test.proxyPassword", "password");
- this->plan->setProperty(this->s3_processor, "Proxy Password", "${test.proxyPassword}");
+ void setProxy(bool use_controller_service) override {
+ if (use_controller_service) {
+ auto proxy_configuration_service = this->plan->addController("ProxyConfigurationService", "ProxyConfigurationService");
+ this->plan->setProperty(proxy_configuration_service, "Proxy Server Host", "host");
+ this->plan->setProperty(proxy_configuration_service, "Proxy Server Port", "1234");
+ this->plan->setProperty(proxy_configuration_service, "Proxy User Name", "username");
+ this->plan->setProperty(proxy_configuration_service, "Proxy User Password", "password");
+ this->plan->setProperty(this->s3_processor, "Proxy Configuration Service", "ProxyConfigurationService");
+ } else {
+ this->plan->setDynamicProperty(update_attribute, "test.proxyHost", "host");
+ this->plan->setProperty(this->s3_processor, "Proxy Host", "${test.proxyHost}");
+ this->plan->setDynamicProperty(update_attribute, "test.proxyPort", "1234");
+ this->plan->setProperty(this->s3_processor, "Proxy Port", "${test.proxyPort}");
+ this->plan->setDynamicProperty(update_attribute, "test.proxyUsername", "username");
+ this->plan->setProperty(this->s3_processor, "Proxy Username", "${test.proxyUsername}");
+ this->plan->setDynamicProperty(update_attribute, "test.proxyPassword", "password");
+ this->plan->setProperty(this->s3_processor, "Proxy Password", "${test.proxyPassword}");
+ }
}
protected:
@@ -224,10 +234,19 @@ class FlowProducerS3TestsFixture : public S3TestsFixture {
this->plan->setProperty(this->s3_processor, "Bucket", this->S3_BUCKET);
}
- void setProxy() override {
- this->plan->setProperty(this->s3_processor, "Proxy Host", "host");
- this->plan->setProperty(this->s3_processor, "Proxy Port", "1234");
- this->plan->setProperty(this->s3_processor, "Proxy Username", "username");
- this->plan->setProperty(this->s3_processor, "Proxy Password", "password");
+ void setProxy(bool use_controller_service) override {
+ if (use_controller_service) {
+ auto proxy_configuration_service = this->plan->addController("ProxyConfigurationService", "ProxyConfigurationService");
+ this->plan->setProperty(proxy_configuration_service, "Proxy Server Host", "host");
+ this->plan->setProperty(proxy_configuration_service, "Proxy Server Port", "1234");
+ this->plan->setProperty(proxy_configuration_service, "Proxy User Name", "username");
+ this->plan->setProperty(proxy_configuration_service, "Proxy User Password", "password");
+ this->plan->setProperty(this->s3_processor, "Proxy Configuration Service", "ProxyConfigurationService");
+ } else {
+ this->plan->setProperty(this->s3_processor, "Proxy Host", "host");
+ this->plan->setProperty(this->s3_processor, "Proxy Port", "1234");
+ this->plan->setProperty(this->s3_processor, "Proxy Username", "username");
+ this->plan->setProperty(this->s3_processor, "Proxy Password", "password");
+ }
}
};
diff --git a/extensions/aws/tests/features/s3.feature b/extensions/aws/tests/features/s3.feature
index b16ea1f259..2fe2832cf8 100644
--- a/extensions/aws/tests/features/s3.feature
+++ b/extensions/aws/tests/features/s3.feature
@@ -44,7 +44,7 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
And a PutS3Object processor set up to communicate with an s3 server
And these processor properties are set
| processor name | property name | property value |
- | PutS3Object | Proxy Host | http-proxy-${scenario_id} |
+ | PutS3Object | Proxy Host | http-proxy-${scenario_id} |
| PutS3Object | Proxy Port | 3128 |
| PutS3Object | Proxy Username | admin |
| PutS3Object | Proxy Password | test101 |
@@ -61,7 +61,26 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
Then a single file with the content "LH_O#L|FD credentials;
std::tie(std::ignore, credentials) = getCredentialsFromControllerService(context);
if (!credentials) {
@@ -51,6 +52,7 @@ bool AzureDataLakeStorageProcessorBase::setCommonParameters(storage::AzureDataLa
}
params.directory_name = context.getProperty(DirectoryName, flow_file).value_or("");
+ params.proxy_configuration = proxy_configuration_;
return true;
}
diff --git a/extensions/azure/processors/AzureDataLakeStorageProcessorBase.h b/extensions/azure/processors/AzureDataLakeStorageProcessorBase.h
index dc2f55f3e2..88de0b1446 100644
--- a/extensions/azure/processors/AzureDataLakeStorageProcessorBase.h
+++ b/extensions/azure/processors/AzureDataLakeStorageProcessorBase.h
@@ -56,7 +56,7 @@ class AzureDataLakeStorageProcessorBase : public AzureStorageProcessorBase {
~AzureDataLakeStorageProcessorBase() override = default;
- void onSchedule(core::ProcessContext& context, core::ProcessSessionFactory& sessionFactory) override;
+ void onSchedule(core::ProcessContext& context, core::ProcessSessionFactory& session_factory) override;
protected:
explicit AzureDataLakeStorageProcessorBase(core::ProcessorMetadata metadata, std::unique_ptr data_lake_storage_client)
diff --git a/extensions/azure/processors/AzureStorageProcessorBase.cpp b/extensions/azure/processors/AzureStorageProcessorBase.cpp
index 7795a4786b..32ac26dc60 100644
--- a/extensions/azure/processors/AzureStorageProcessorBase.cpp
+++ b/extensions/azure/processors/AzureStorageProcessorBase.cpp
@@ -25,9 +25,24 @@
#include "minifi-cpp/core/ProcessContext.h"
#include "controllerservices/AzureStorageCredentialsService.h"
+#include "utils/ProcessorConfigUtils.h"
namespace org::apache::nifi::minifi::azure::processors {
+void AzureStorageProcessorBase::onSchedule(core::ProcessContext& context, core::ProcessSessionFactory&) {
+ auto proxy_controller_service = minifi::utils::parseOptionalControllerService(context, ProxyConfigurationService, getUUID());
+ if (proxy_controller_service) {
+ logger_->log_debug("Proxy configuration is set for Azure Storage processor");
+ proxy_configuration_ = minifi::controllers::ProxyConfiguration{
+ .proxy_type = minifi::controllers::ProxyType::HTTP,
+ .proxy_host = proxy_controller_service->getHost(),
+ .proxy_port = proxy_controller_service->getPort(),
+ .proxy_user = proxy_controller_service->getUsername(),
+ .proxy_password = proxy_controller_service->getPassword()
+ };
+ }
+}
+
std::tuple> AzureStorageProcessorBase::getCredentialsFromControllerService(
core::ProcessContext &context) const {
std::string service_name = context.getProperty(AzureStorageCredentialsService).value_or("");
diff --git a/extensions/azure/processors/AzureStorageProcessorBase.h b/extensions/azure/processors/AzureStorageProcessorBase.h
index 8f98f72a0c..ca86dc58f1 100644
--- a/extensions/azure/processors/AzureStorageProcessorBase.h
+++ b/extensions/azure/processors/AzureStorageProcessorBase.h
@@ -32,6 +32,8 @@
#include "core/ProcessorImpl.h"
#include "minifi-cpp/core/logging/Logger.h"
#include "storage/AzureStorageCredentials.h"
+#include "minifi-cpp/controllers/ProxyConfigurationServiceInterface.h"
+#include "controllers/ProxyConfiguration.h"
namespace org::apache::nifi::minifi::azure::processors {
@@ -40,10 +42,16 @@ class AzureStorageProcessorBase : public core::ProcessorImpl {
EXTENSIONAPI static constexpr auto AzureStorageCredentialsService = core::PropertyDefinitionBuilder<>::createProperty("Azure Storage Credentials Service")
.withDescription("Name of the Azure Storage Credentials Service used to retrieve the connection string from.")
.build();
- EXTENSIONAPI static constexpr auto Properties = std::to_array({AzureStorageCredentialsService});
+ EXTENSIONAPI static constexpr auto ProxyConfigurationService = core::PropertyDefinitionBuilder<>::createProperty("Proxy Configuration Service")
+ .withDescription("Specifies the Proxy Configuration Controller Service to proxy network requests.")
+ .withAllowedTypes()
+ .build();
+ EXTENSIONAPI static constexpr auto Properties = std::to_array({AzureStorageCredentialsService, ProxyConfigurationService});
using ProcessorImpl::ProcessorImpl;
+ void onSchedule(core::ProcessContext& context, core::ProcessSessionFactory& session_factory) override;
+
protected:
enum class GetCredentialsFromControllerResult {
OK,
@@ -52,6 +60,9 @@ class AzureStorageProcessorBase : public core::ProcessorImpl {
};
std::tuple> getCredentialsFromControllerService(core::ProcessContext &context) const;
+
+ protected:
+ std::optional proxy_configuration_;
};
} // namespace org::apache::nifi::minifi::azure::processors
diff --git a/extensions/azure/storage/AzureBlobStorageClient.cpp b/extensions/azure/storage/AzureBlobStorageClient.cpp
index 837efa4ac8..b4539ff4db 100644
--- a/extensions/azure/storage/AzureBlobStorageClient.cpp
+++ b/extensions/azure/storage/AzureBlobStorageClient.cpp
@@ -56,34 +56,47 @@ AzureBlobStorageClient::AzureBlobStorageClient() {
utils::AzureSdkLogger::initialize();
}
-Azure::Storage::Blobs::BlobContainerClient AzureBlobStorageClient::createClient(const AzureStorageCredentials &credentials, const std::string &container_name) {
+Azure::Storage::Blobs::BlobContainerClient AzureBlobStorageClient::createClient(const AzureStorageCredentials &credentials, const std::string &container_name,
+ const std::optional& proxy_configuration) {
+ Azure::Storage::Blobs::BlobClientOptions client_options;
+
+ if (proxy_configuration) {
+ client_options.Transport.HttpProxy = proxy_configuration->proxy_host + (proxy_configuration->proxy_port ? (":" + std::to_string(*proxy_configuration->proxy_port)) : "");
+ if (proxy_configuration->proxy_user) {
+ client_options.Transport.ProxyUserName = *proxy_configuration->proxy_user;
+ }
+ if (proxy_configuration->proxy_password) {
+ client_options.Transport.ProxyPassword = *proxy_configuration->proxy_password;
+ }
+ }
+
if (credentials.getCredentialConfigurationStrategy() == CredentialConfigurationStrategyOption::FromProperties) {
- return Azure::Storage::Blobs::BlobContainerClient::CreateFromConnectionString(credentials.buildConnectionString(), container_name);
+ return Azure::Storage::Blobs::BlobContainerClient::CreateFromConnectionString(credentials.buildConnectionString(), container_name, client_options);
}
auto storage_client = Azure::Storage::Blobs::BlobServiceClient("https://" + credentials.getStorageAccountName() + ".blob." + credentials.getEndpointSuffix(),
- credentials.createAzureTokenCredential());
+ credentials.createAzureTokenCredential(), client_options);
return storage_client.GetBlobContainerClient(container_name);
}
bool AzureBlobStorageClient::createContainerIfNotExists(const PutAzureBlobStorageParameters& params) {
- auto container_client = createClient(params.credentials, params.container_name);
+ auto container_client = createClient(params.credentials, params.container_name, params.proxy_configuration);
return container_client.CreateIfNotExists().Value.Created;
}
Azure::Storage::Blobs::Models::UploadBlockBlobResult AzureBlobStorageClient::uploadBlob(const PutAzureBlobStorageParameters& params, std::span buffer) {
- auto container_client = createClient(params.credentials, params.container_name);
+ auto container_client = createClient(params.credentials, params.container_name, params.proxy_configuration);
auto blob_client = container_client.GetBlockBlobClient(params.blob_name);
return blob_client.UploadFrom(reinterpret_cast(buffer.data()), buffer.size()).Value;
}
std::string AzureBlobStorageClient::getUrl(const AzureBlobStorageParameters& params) {
- auto container_client = createClient(params.credentials, params.container_name);
+ auto container_client = createClient(params.credentials, params.container_name, params.proxy_configuration);
return container_client.GetUrl();
}
bool AzureBlobStorageClient::deleteBlob(const DeleteAzureBlobStorageParameters& params) {
- auto container_client = createClient(params.credentials, params.container_name);
+ auto container_client = createClient(params.credentials, params.container_name, params.proxy_configuration);
Azure::Storage::Blobs::DeleteBlobOptions delete_options;
if (params.optional_deletion == OptionalDeletion::INCLUDE_SNAPSHOTS) {
delete_options.DeleteSnapshots = Azure::Storage::Blobs::Models::DeleteSnapshotsOption::IncludeSnapshots;
@@ -95,7 +108,7 @@ bool AzureBlobStorageClient::deleteBlob(const DeleteAzureBlobStorageParameters&
}
std::unique_ptr AzureBlobStorageClient::fetchBlob(const FetchAzureBlobStorageParameters& params) {
- auto container_client = createClient(params.credentials, params.container_name);
+ auto container_client = createClient(params.credentials, params.container_name, params.proxy_configuration);
auto blob_client = container_client.GetBlobClient(params.blob_name);
Azure::Storage::Blobs::DownloadBlobOptions options;
if (params.range_start || params.range_length) {
@@ -115,7 +128,7 @@ std::unique_ptr AzureBlobStorageClient::fetchBlob(const FetchAz
std::vector AzureBlobStorageClient::listContainer(const ListAzureBlobStorageParameters& params) {
std::vector result;
- auto container_client = createClient(params.credentials, params.container_name);
+ auto container_client = createClient(params.credentials, params.container_name, params.proxy_configuration);
Azure::Storage::Blobs::ListBlobsOptions options;
options.Prefix = params.prefix;
for (auto page_result = container_client.ListBlobs(options); page_result.HasPage(); page_result.MoveToNextPage()) {
diff --git a/extensions/azure/storage/AzureBlobStorageClient.h b/extensions/azure/storage/AzureBlobStorageClient.h
index 2b67c76356..ec3d62eea8 100644
--- a/extensions/azure/storage/AzureBlobStorageClient.h
+++ b/extensions/azure/storage/AzureBlobStorageClient.h
@@ -43,7 +43,8 @@ class AzureBlobStorageClient : public BlobStorageClient {
std::vector listContainer(const ListAzureBlobStorageParameters& params) override;
private:
- static Azure::Storage::Blobs::BlobContainerClient createClient(const AzureStorageCredentials& credentials, const std::string &container_name);
+ static Azure::Storage::Blobs::BlobContainerClient createClient(const AzureStorageCredentials& credentials, const std::string &container_name,
+ const std::optional& proxy_configuration);
std::shared_ptr logger_{core::logging::LoggerFactory::getLogger()};
};
diff --git a/extensions/azure/storage/AzureDataLakeStorageClient.cpp b/extensions/azure/storage/AzureDataLakeStorageClient.cpp
index d7618818ed..ace617b06f 100644
--- a/extensions/azure/storage/AzureDataLakeStorageClient.cpp
+++ b/extensions/azure/storage/AzureDataLakeStorageClient.cpp
@@ -35,13 +35,23 @@ AzureDataLakeStorageClient::AzureDataLakeStorageClient() {
utils::AzureSdkLogger::initialize();
}
-std::unique_ptr AzureDataLakeStorageClient::createClient(
- const AzureStorageCredentials& credentials, const std::string& file_system_name, std::optional number_of_retries) {
+std::unique_ptr AzureDataLakeStorageClient::createClient(const AzureStorageCredentials& credentials,
+ const std::string& file_system_name, std::optional number_of_retries, const std::optional& proxy_configuration) {
Azure::Storage::Files::DataLake::DataLakeClientOptions options;
if (number_of_retries) {
options.Retry.MaxRetries = gsl::narrow(*number_of_retries);
}
+ if (proxy_configuration) {
+ options.Transport.HttpProxy = proxy_configuration->proxy_host + (proxy_configuration->proxy_port ? (":" + std::to_string(*proxy_configuration->proxy_port)) : "");
+ if (proxy_configuration->proxy_user) {
+ options.Transport.ProxyUserName = *proxy_configuration->proxy_user;
+ }
+ if (proxy_configuration->proxy_password) {
+ options.Transport.ProxyPassword = *proxy_configuration->proxy_password;
+ }
+ }
+
if (credentials.getCredentialConfigurationStrategy() == CredentialConfigurationStrategyOption::FromProperties) {
return std::make_unique(
Azure::Storage::Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString(credentials.buildConnectionString(), file_system_name, options));
@@ -53,7 +63,7 @@ std::unique_ptr Azure
}
Azure::Storage::Files::DataLake::DataLakeDirectoryClient AzureDataLakeStorageClient::getDirectoryClient(const AzureDataLakeStorageParameters& params) {
- auto client = createClient(params.credentials, params.file_system_name, params.number_of_retries);
+ auto client = createClient(params.credentials, params.file_system_name, params.number_of_retries, params.proxy_configuration);
return client->GetDirectoryClient(params.directory_name);
}
@@ -104,7 +114,7 @@ std::unique_ptr AzureDataLakeStorageClient::fetchFile(const Fet
std::vector AzureDataLakeStorageClient::listDirectory(const ListAzureDataLakeStorageParameters& params) {
std::vector result;
if (params.directory_name.empty()) {
- auto client = createClient(params.credentials, params.file_system_name, params.number_of_retries);
+ auto client = createClient(params.credentials, params.file_system_name, params.number_of_retries, params.proxy_configuration);
for (auto page_result = client->ListPaths(params.recurse_subdirectories); page_result.HasPage(); page_result.MoveToNextPage()) {
result.insert(result.end(), page_result.Paths.begin(), page_result.Paths.end());
}
diff --git a/extensions/azure/storage/AzureDataLakeStorageClient.h b/extensions/azure/storage/AzureDataLakeStorageClient.h
index 9c9615869c..b9ab9a27b0 100644
--- a/extensions/azure/storage/AzureDataLakeStorageClient.h
+++ b/extensions/azure/storage/AzureDataLakeStorageClient.h
@@ -93,8 +93,8 @@ class AzureDataLakeStorageClient : public DataLakeStorageClient {
Azure::Storage::Files::DataLake::Models::DownloadFileResult result_;
};
- static std::unique_ptr createClient(
- const AzureStorageCredentials& credentials, const std::string& file_system_name, std::optional number_of_retries);
+ static std::unique_ptr createClient(const AzureStorageCredentials& credentials,
+ const std::string& file_system_name, std::optional number_of_retries, const std::optional& proxy_configuration);
static Azure::Storage::Files::DataLake::DataLakeDirectoryClient getDirectoryClient(const AzureDataLakeStorageParameters& params);
static Azure::Storage::Files::DataLake::DataLakeFileClient getFileClient(const AzureDataLakeStorageFileOperationParameters& params);
diff --git a/extensions/azure/storage/BlobStorageClient.h b/extensions/azure/storage/BlobStorageClient.h
index 007b3b86f3..4b1b0c3684 100644
--- a/extensions/azure/storage/BlobStorageClient.h
+++ b/extensions/azure/storage/BlobStorageClient.h
@@ -30,6 +30,8 @@
#include "minifi-cpp/utils/gsl.h"
#include "utils/Enum.h"
#include "minifi-cpp/io/InputStream.h"
+#include "minifi-cpp/controllers/ProxyConfigurationServiceInterface.h"
+#include "controllers/ProxyConfiguration.h"
namespace org::apache::nifi::minifi::azure::storage {
@@ -62,6 +64,7 @@ namespace org::apache::nifi::minifi::azure::storage {
struct AzureBlobStorageParameters {
AzureStorageCredentials credentials;
std::string container_name;
+ std::optional proxy_configuration;
};
struct AzureBlobStorageBlobOperationParameters : public AzureBlobStorageParameters {
diff --git a/extensions/azure/storage/DataLakeStorageClient.h b/extensions/azure/storage/DataLakeStorageClient.h
index 06445a2e8d..5347f92850 100644
--- a/extensions/azure/storage/DataLakeStorageClient.h
+++ b/extensions/azure/storage/DataLakeStorageClient.h
@@ -31,6 +31,8 @@
#include "azure/storage/files/datalake/datalake_responses.hpp"
#include "utils/Enum.h"
#include "utils/RegexUtils.h"
+#include "minifi-cpp/controllers/ProxyConfigurationServiceInterface.h"
+#include "controllers/ProxyConfiguration.h"
namespace org::apache::nifi::minifi::azure::storage {
@@ -39,6 +41,7 @@ struct AzureDataLakeStorageParameters {
std::string file_system_name;
std::string directory_name;
std::optional number_of_retries;
+ std::optional proxy_configuration;
};
struct AzureDataLakeStorageFileOperationParameters : public AzureDataLakeStorageParameters {
diff --git a/extensions/azure/tests/DeleteAzureBlobStorageTests.cpp b/extensions/azure/tests/DeleteAzureBlobStorageTests.cpp
index b554021bbe..cd91a592df 100644
--- a/extensions/azure/tests/DeleteAzureBlobStorageTests.cpp
+++ b/extensions/azure/tests/DeleteAzureBlobStorageTests.cpp
@@ -339,4 +339,28 @@ TEST_CASE_METHOD(DeleteAzureBlobStorageTestsFixture, "Test Azure blob delete wit
REQUIRE(failed_flowfiles[0] == TEST_DATA);
}
+TEST_CASE_METHOD(DeleteAzureBlobStorageTestsFixture, "Test Azure blob delete using proxy", "[azureBlobStorageDelete]") {
+ auto proxy_configuration_service = plan_->addController("ProxyConfigurationService", "ProxyConfigurationService");
+ plan_->setProperty(proxy_configuration_service, "Proxy Server Host", "host");
+ plan_->setProperty(proxy_configuration_service, "Proxy Server Port", "1234");
+ plan_->setProperty(proxy_configuration_service, "Proxy User Name", "username");
+ plan_->setProperty(proxy_configuration_service, "Proxy User Password", "password");
+ plan_->setProperty(azure_blob_storage_processor_, "Proxy Configuration Service", "ProxyConfigurationService");
+
+ plan_->setProperty(azure_blob_storage_processor_, "Container Name", "test.container");
+ plan_->setProperty(azure_blob_storage_processor_, "Blob", "test.blob");
+ setDefaultCredentials();
+ test_controller_.runSession(plan_, true);
+ auto passed_params = mock_blob_storage_ptr_->getPassedDeleteParams();
+ REQUIRE(passed_params.proxy_configuration);
+ REQUIRE(passed_params.proxy_configuration->proxy_host == "host");
+ REQUIRE(passed_params.proxy_configuration->proxy_port);
+ REQUIRE(*passed_params.proxy_configuration->proxy_port == 1234);
+ REQUIRE(passed_params.proxy_configuration->proxy_user);
+ REQUIRE(*passed_params.proxy_configuration->proxy_user == "username");
+ REQUIRE(passed_params.proxy_configuration->proxy_password);
+ REQUIRE(*passed_params.proxy_configuration->proxy_password == "password");
+ CHECK(getFailedFlowFileContents().empty());
+}
+
} // namespace
diff --git a/extensions/azure/tests/DeleteAzureDataLakeStorageTests.cpp b/extensions/azure/tests/DeleteAzureDataLakeStorageTests.cpp
index a241429c25..4f2a8e1698 100644
--- a/extensions/azure/tests/DeleteAzureDataLakeStorageTests.cpp
+++ b/extensions/azure/tests/DeleteAzureDataLakeStorageTests.cpp
@@ -153,4 +153,26 @@ TEST_CASE_METHOD(DeleteAzureDataLakeStorageTestsFixture, "Delete result is false
REQUIRE(failed_flowfiles[0] == TEST_DATA);
}
+TEST_CASE_METHOD(DeleteAzureDataLakeStorageTestsFixture, "Test Azure data lake storage delete using proxy", "[azureDataLakeStorageDelete]") {
+ auto proxy_configuration_service = plan_->addController("ProxyConfigurationService", "ProxyConfigurationService");
+ plan_->setProperty(proxy_configuration_service, "Proxy Server Host", "host");
+ plan_->setProperty(proxy_configuration_service, "Proxy Server Port", "1234");
+ plan_->setProperty(proxy_configuration_service, "Proxy User Name", "username");
+ plan_->setProperty(proxy_configuration_service, "Proxy User Password", "password");
+ plan_->setProperty(azure_data_lake_storage_, "Proxy Configuration Service", "ProxyConfigurationService");
+
+ test_controller_.runSession(plan_, true);
+
+ auto passed_params = mock_data_lake_storage_client_ptr_->getPassedDeleteParams();
+ REQUIRE(passed_params.proxy_configuration);
+ REQUIRE(passed_params.proxy_configuration->proxy_host == "host");
+ REQUIRE(passed_params.proxy_configuration->proxy_port);
+ REQUIRE(*passed_params.proxy_configuration->proxy_port == 1234);
+ REQUIRE(passed_params.proxy_configuration->proxy_user);
+ REQUIRE(*passed_params.proxy_configuration->proxy_user == "username");
+ REQUIRE(passed_params.proxy_configuration->proxy_password);
+ REQUIRE(*passed_params.proxy_configuration->proxy_password == "password");
+ CHECK(getFailedFlowFileContents().empty());
+}
+
} // namespace
diff --git a/extensions/azure/tests/FetchAzureBlobStorageTests.cpp b/extensions/azure/tests/FetchAzureBlobStorageTests.cpp
index cd57961cbf..1a145937d6 100644
--- a/extensions/azure/tests/FetchAzureBlobStorageTests.cpp
+++ b/extensions/azure/tests/FetchAzureBlobStorageTests.cpp
@@ -329,4 +329,28 @@ TEST_CASE_METHOD(FetchAzureBlobStorageTestsFixture, "Fetch full file fails", "[a
REQUIRE(failed_contents[0] == TEST_DATA);
}
+TEST_CASE_METHOD(FetchAzureBlobStorageTestsFixture, "Test Azure blob fetch using proxy", "[azureBlobStorageFetch]") {
+ auto proxy_configuration_service = plan_->addController("ProxyConfigurationService", "ProxyConfigurationService");
+ plan_->setProperty(proxy_configuration_service, "Proxy Server Host", "host");
+ plan_->setProperty(proxy_configuration_service, "Proxy Server Port", "1234");
+ plan_->setProperty(proxy_configuration_service, "Proxy User Name", "username");
+ plan_->setProperty(proxy_configuration_service, "Proxy User Password", "password");
+ plan_->setProperty(azure_blob_storage_processor_, "Proxy Configuration Service", "ProxyConfigurationService");
+
+ plan_->setProperty(azure_blob_storage_processor_, "Container Name", "test.container");
+ plan_->setProperty(azure_blob_storage_processor_, "Blob", "test.blob");
+ setDefaultCredentials();
+ test_controller_.runSession(plan_, true);
+ auto passed_params = mock_blob_storage_ptr_->getPassedFetchParams();
+ REQUIRE(passed_params.proxy_configuration);
+ REQUIRE(passed_params.proxy_configuration->proxy_host == "host");
+ REQUIRE(passed_params.proxy_configuration->proxy_port);
+ REQUIRE(*passed_params.proxy_configuration->proxy_port == 1234);
+ REQUIRE(passed_params.proxy_configuration->proxy_user);
+ REQUIRE(*passed_params.proxy_configuration->proxy_user == "username");
+ REQUIRE(passed_params.proxy_configuration->proxy_password);
+ REQUIRE(*passed_params.proxy_configuration->proxy_password == "password");
+ CHECK(getFailedFlowFileContents().empty());
+}
+
} // namespace
diff --git a/extensions/azure/tests/FetchAzureDataLakeStorageTests.cpp b/extensions/azure/tests/FetchAzureDataLakeStorageTests.cpp
index c5505f49e8..7a98f28c84 100644
--- a/extensions/azure/tests/FetchAzureDataLakeStorageTests.cpp
+++ b/extensions/azure/tests/FetchAzureDataLakeStorageTests.cpp
@@ -170,4 +170,26 @@ TEST_CASE_METHOD(FetchAzureDataLakeStorageTestsFixture, "Fetch full file fails",
REQUIRE(failed_contents[0] == TEST_DATA);
}
+TEST_CASE_METHOD(FetchAzureDataLakeStorageTestsFixture, "Test Azure data lake storage fetch using proxy", "[azureDataLakeStorageFetch]") {
+ auto proxy_configuration_service = plan_->addController("ProxyConfigurationService", "ProxyConfigurationService");
+ plan_->setProperty(proxy_configuration_service, "Proxy Server Host", "host");
+ plan_->setProperty(proxy_configuration_service, "Proxy Server Port", "1234");
+ plan_->setProperty(proxy_configuration_service, "Proxy User Name", "username");
+ plan_->setProperty(proxy_configuration_service, "Proxy User Password", "password");
+ plan_->setProperty(azure_data_lake_storage_, "Proxy Configuration Service", "ProxyConfigurationService");
+
+ test_controller_.runSession(plan_, true);
+
+ auto passed_params = mock_data_lake_storage_client_ptr_->getPassedFetchParams();
+ REQUIRE(passed_params.proxy_configuration);
+ REQUIRE(passed_params.proxy_configuration->proxy_host == "host");
+ REQUIRE(passed_params.proxy_configuration->proxy_port);
+ REQUIRE(*passed_params.proxy_configuration->proxy_port == 1234);
+ REQUIRE(passed_params.proxy_configuration->proxy_user);
+ REQUIRE(*passed_params.proxy_configuration->proxy_user == "username");
+ REQUIRE(passed_params.proxy_configuration->proxy_password);
+ REQUIRE(*passed_params.proxy_configuration->proxy_password == "password");
+ CHECK(getFailedFlowFileContents().empty());
+}
+
} // namespace
diff --git a/extensions/azure/tests/ListAzureBlobStorageTests.cpp b/extensions/azure/tests/ListAzureBlobStorageTests.cpp
index 31bdad9098..59be4b4a97 100644
--- a/extensions/azure/tests/ListAzureBlobStorageTests.cpp
+++ b/extensions/azure/tests/ListAzureBlobStorageTests.cpp
@@ -349,4 +349,29 @@ TEST_CASE_METHOD(ListAzureBlobStorageTestsFixture, "Do not list same files the s
REQUIRE_FALSE(LogTestController::getInstance().contains("key:azure", 0s, 0ms));
}
+TEST_CASE_METHOD(ListAzureBlobStorageTestsFixture, "List all files through a proxy", "[ListAzureBlobStorage]") {
+ setDefaultCredentials();
+ plan_->setProperty(list_azure_blob_storage_, minifi::azure::processors::ListAzureBlobStorage::ContainerName, CONTAINER_NAME);
+ plan_->setProperty(list_azure_blob_storage_, minifi::azure::processors::ListAzureBlobStorage::Prefix, PREFIX);
+ plan_->setProperty(list_azure_blob_storage_, minifi::azure::processors::ListAzureBlobStorage::ListingStrategy, magic_enum::enum_name(minifi::azure::EntityTracking::none));
+
+ auto proxy_configuration_service = plan_->addController("ProxyConfigurationService", "ProxyConfigurationService");
+ plan_->setProperty(proxy_configuration_service, "Proxy Server Host", "host");
+ plan_->setProperty(proxy_configuration_service, "Proxy Server Port", "1234");
+ plan_->setProperty(proxy_configuration_service, "Proxy User Name", "username");
+ plan_->setProperty(proxy_configuration_service, "Proxy User Password", "password");
+ plan_->setProperty(list_azure_blob_storage_, "Proxy Configuration Service", "ProxyConfigurationService");
+
+ test_controller_.runSession(plan_, true);
+ auto passed_params = mock_blob_storage_ptr_->getPassedListParams();
+ REQUIRE(passed_params.proxy_configuration);
+ REQUIRE(passed_params.proxy_configuration->proxy_host == "host");
+ REQUIRE(passed_params.proxy_configuration->proxy_port);
+ REQUIRE(*passed_params.proxy_configuration->proxy_port == 1234);
+ REQUIRE(passed_params.proxy_configuration->proxy_user);
+ REQUIRE(*passed_params.proxy_configuration->proxy_user == "username");
+ REQUIRE(passed_params.proxy_configuration->proxy_password);
+ REQUIRE(*passed_params.proxy_configuration->proxy_password == "password");
+}
+
} // namespace
diff --git a/extensions/azure/tests/ListAzureDataLakeStorageTests.cpp b/extensions/azure/tests/ListAzureDataLakeStorageTests.cpp
index 4f2a19b512..caf85449be 100644
--- a/extensions/azure/tests/ListAzureDataLakeStorageTests.cpp
+++ b/extensions/azure/tests/ListAzureDataLakeStorageTests.cpp
@@ -256,4 +256,25 @@ TEST_CASE_METHOD(ListAzureDataLakeStorageTestsFixture, "Both SAS Token and Stora
REQUIRE_THROWS_AS(test_controller_.runSession(plan_, true), minifi::Exception);
}
+TEST_CASE_METHOD(ListAzureDataLakeStorageTestsFixture, "List data lake storage files using proxy", "[azureDataLakeStorageParameters]") {
+ auto proxy_configuration_service = plan_->addController("ProxyConfigurationService", "ProxyConfigurationService");
+ plan_->setProperty(proxy_configuration_service, "Proxy Server Host", "host");
+ plan_->setProperty(proxy_configuration_service, "Proxy Server Port", "1234");
+ plan_->setProperty(proxy_configuration_service, "Proxy User Name", "username");
+ plan_->setProperty(proxy_configuration_service, "Proxy User Password", "password");
+ plan_->setProperty(list_azure_data_lake_storage_, "Proxy Configuration Service", "ProxyConfigurationService");
+
+ test_controller_.runSession(plan_, true);
+
+ auto passed_params = mock_data_lake_storage_client_ptr_->getPassedListParams();
+ REQUIRE(passed_params.proxy_configuration);
+ REQUIRE(passed_params.proxy_configuration->proxy_host == "host");
+ REQUIRE(passed_params.proxy_configuration->proxy_port);
+ REQUIRE(*passed_params.proxy_configuration->proxy_port == 1234);
+ REQUIRE(passed_params.proxy_configuration->proxy_user);
+ REQUIRE(*passed_params.proxy_configuration->proxy_user == "username");
+ REQUIRE(passed_params.proxy_configuration->proxy_password);
+ REQUIRE(*passed_params.proxy_configuration->proxy_password == "password");
+}
+
} // namespace
diff --git a/extensions/azure/tests/PutAzureBlobStorageTests.cpp b/extensions/azure/tests/PutAzureBlobStorageTests.cpp
index a4a0ed19ee..19fa585dcd 100644
--- a/extensions/azure/tests/PutAzureBlobStorageTests.cpp
+++ b/extensions/azure/tests/PutAzureBlobStorageTests.cpp
@@ -338,4 +338,28 @@ TEST_CASE_METHOD(PutAzureBlobStorageTestsFixture, "Test Azure blob upload failur
REQUIRE(failed_flowfiles[0] == TEST_DATA);
}
+TEST_CASE_METHOD(PutAzureBlobStorageTestsFixture, "Test Azure blob storage put using proxy", "[azureBlobStorageUpload]") {
+ auto proxy_configuration_service = plan_->addController("ProxyConfigurationService", "ProxyConfigurationService");
+ plan_->setProperty(proxy_configuration_service, "Proxy Server Host", "host");
+ plan_->setProperty(proxy_configuration_service, "Proxy Server Port", "1234");
+ plan_->setProperty(proxy_configuration_service, "Proxy User Name", "username");
+ plan_->setProperty(proxy_configuration_service, "Proxy User Password", "password");
+ plan_->setProperty(azure_blob_storage_processor_, "Proxy Configuration Service", "ProxyConfigurationService");
+
+ plan_->setProperty(azure_blob_storage_processor_, "Container Name", "test.container");
+ plan_->setProperty(azure_blob_storage_processor_, "Blob", "test.blob");
+ setDefaultCredentials();
+ test_controller_.runSession(plan_, true);
+ auto passed_params = mock_blob_storage_ptr_->getPassedPutParams();
+ REQUIRE(passed_params.proxy_configuration);
+ REQUIRE(passed_params.proxy_configuration->proxy_host == "host");
+ REQUIRE(passed_params.proxy_configuration->proxy_port);
+ REQUIRE(*passed_params.proxy_configuration->proxy_port == 1234);
+ REQUIRE(passed_params.proxy_configuration->proxy_user);
+ REQUIRE(*passed_params.proxy_configuration->proxy_user == "username");
+ REQUIRE(passed_params.proxy_configuration->proxy_password);
+ REQUIRE(*passed_params.proxy_configuration->proxy_password == "password");
+ CHECK(getFailedFlowFileContents().empty());
+}
+
} // namespace
diff --git a/extensions/azure/tests/PutAzureDataLakeStorageTests.cpp b/extensions/azure/tests/PutAzureDataLakeStorageTests.cpp
index 6938dc841f..32208afcd1 100644
--- a/extensions/azure/tests/PutAzureDataLakeStorageTests.cpp
+++ b/extensions/azure/tests/PutAzureDataLakeStorageTests.cpp
@@ -193,4 +193,26 @@ TEST_CASE_METHOD(PutAzureDataLakeStorageTestsFixture, "Upload to Azure Data Lake
CHECK(verifyLogLinePresenceInPollTime(1s, "key:azure.directory value:\n"));
}
+TEST_CASE_METHOD(PutAzureDataLakeStorageTestsFixture, "Test Azure data lake storage upload using proxy", "[azureDataLakeStorageUpload]") {
+ auto proxy_configuration_service = plan_->addController("ProxyConfigurationService", "ProxyConfigurationService");
+ plan_->setProperty(proxy_configuration_service, "Proxy Server Host", "host");
+ plan_->setProperty(proxy_configuration_service, "Proxy Server Port", "1234");
+ plan_->setProperty(proxy_configuration_service, "Proxy User Name", "username");
+ plan_->setProperty(proxy_configuration_service, "Proxy User Password", "password");
+ plan_->setProperty(azure_data_lake_storage_, "Proxy Configuration Service", "ProxyConfigurationService");
+
+ test_controller_.runSession(plan_, true);
+
+ auto passed_params = mock_data_lake_storage_client_ptr_->getPassedPutParams();
+ REQUIRE(passed_params.proxy_configuration);
+ REQUIRE(passed_params.proxy_configuration->proxy_host == "host");
+ REQUIRE(passed_params.proxy_configuration->proxy_port);
+ REQUIRE(*passed_params.proxy_configuration->proxy_port == 1234);
+ REQUIRE(passed_params.proxy_configuration->proxy_user);
+ REQUIRE(*passed_params.proxy_configuration->proxy_user == "username");
+ REQUIRE(passed_params.proxy_configuration->proxy_password);
+ REQUIRE(*passed_params.proxy_configuration->proxy_password == "password");
+ CHECK(getFailedFlowFileContents().empty());
+}
+
} // namespace
diff --git a/extensions/azure/tests/features/azure_storage.feature b/extensions/azure/tests/features/azure_storage.feature
index 26fb71dc18..687e41790a 100644
--- a/extensions/azure/tests/features/azure_storage.feature
+++ b/extensions/azure/tests/features/azure_storage.feature
@@ -108,3 +108,84 @@ Feature: Sending data from MiNiFi-C++ to an Azure storage server
Then the Minifi logs contain the following message: "key:azure.blobname value:test_1" in less than 60 seconds
Then the Minifi logs contain the following message: "key:azure.blobname value:test_2" in less than 60 seconds
And the Minifi logs do not contain the following message: "key:azure.blobname value:other_test" after 0 seconds
+
+ Scenario: A MiNiFi instance can upload data to Azure blob storage through a proxy
+ Given a GetFile processor with the "Input Directory" property set to "/tmp/input"
+ And a file with the content "#test_data$123$#" is present in "/tmp/input"
+ And a PutAzureBlobStorage processor set up to communicate with an Azure blob storage
+ And the "Proxy Configuration Service" property of the PutAzureBlobStorage processor is set to "ProxyConfigurationService"
+ And a PutFile processor with the "Directory" property set to "/tmp/output"
+ And the "success" relationship of the GetFile processor is connected to the PutAzureBlobStorage
+ And the "success" relationship of the PutAzureBlobStorage processor is connected to the PutFile
+ And the "failure" relationship of the PutAzureBlobStorage processor is connected to the PutAzureBlobStorage
+ And a ProxyConfigurationService controller service is set up with HTTP proxy configuration
+
+ And an Azure storage server is set up
+ And the http proxy server is set up
+
+ When all instances start up
+
+ Then a single file with the content "#test_data$123$#" is placed in the "/tmp/output" directory in less than 60 seconds
+ And the object on the Azure storage server is "#test_data$123$#"
+ And no errors were generated on the http-proxy regarding "http://azure-storage-server-${scenario_id}:10000/devstoreaccount1/test-container/test-blob"
+
+ Scenario: A MiNiFi instance can delete blob from Azure blob storage through a proxy
+ Given a GenerateFlowFile processor with the "File Size" property set to "0B"
+ And a DeleteAzureBlobStorage processor set up to communicate with an Azure blob storage
+ And the "Blob" property of the DeleteAzureBlobStorage processor is set to "test"
+ And the "Proxy Configuration Service" property of the DeleteAzureBlobStorage processor is set to "ProxyConfigurationService"
+ And the "success" relationship of the GenerateFlowFile processor is connected to the DeleteAzureBlobStorage
+ And a ProxyConfigurationService controller service is set up with HTTP proxy configuration
+
+ And an Azure storage server is set up
+ And the http proxy server is set up
+
+ When all instances start up
+ And test blob "test" is created on Azure blob storage
+
+ Then the Azure blob storage becomes empty in 30 seconds
+ And no errors were generated on the http-proxy regarding "http://azure-storage-server-${scenario_id}:10000/devstoreaccount1/test-container/test"
+
+ Scenario: A MiNiFi instance can fetch a blob from Azure blob storage through a proxy
+ Given a GetFile processor with the "Input Directory" property set to "/tmp/input"
+ And the "Keep Source File" property of the GetFile processor is set to "true"
+ And a file with the content "dummy" is present in "/tmp/input"
+ And a FetchAzureBlobStorage processor set up to communicate with an Azure blob storage
+ And the "Blob" property of the FetchAzureBlobStorage processor is set to "test"
+ And the "Range Start" property of the FetchAzureBlobStorage processor is set to "6"
+ And the "Range Length" property of the FetchAzureBlobStorage processor is set to "5"
+ And the "Proxy Configuration Service" property of the FetchAzureBlobStorage processor is set to "ProxyConfigurationService"
+ And a PutFile processor with the "Directory" property set to "/tmp/output"
+ And the "success" relationship of the GetFile processor is connected to the FetchAzureBlobStorage
+ And the "success" relationship of the FetchAzureBlobStorage processor is connected to the PutFile
+ And a ProxyConfigurationService controller service is set up with HTTP proxy configuration
+
+ And an Azure storage server is set up
+ And the http proxy server is set up
+
+ When all instances start up
+ And test blob "test" with the content "#test_data$123$#" is created on Azure blob storage
+
+ Then a single file with the content "data$" is placed in the "/tmp/output" directory in less than 60 seconds
+ And no errors were generated on the http-proxy regarding "http://azure-storage-server-${scenario_id}:10000/devstoreaccount1/test-container/test"
+
+ Scenario: A MiNiFi instance can list a container on Azure blob storage through a proxy
+ Given a ListAzureBlobStorage processor set up to communicate with an Azure blob storage
+ And the "Prefix" property of the ListAzureBlobStorage processor is set to "test"
+ And the "Proxy Configuration Service" property of the ListAzureBlobStorage processor is set to "ProxyConfigurationService"
+ And a LogAttribute processor with the "FlowFiles To Log" property set to "0"
+ And the "success" relationship of the ListAzureBlobStorage processor is connected to the LogAttribute
+ And a ProxyConfigurationService controller service is set up with HTTP proxy configuration
+
+ And an Azure storage server is set up
+ And the http proxy server is set up
+
+ When all instances start up
+ And test blob "test_1" with the content "data_1" is created on Azure blob storage
+ And test blob "test_2" with the content "data_2" is created on Azure blob storage
+ And test blob "other_test" with the content "data_3" is created on Azure blob storage
+
+ Then the Minifi logs contain the following message: "key:azure.blobname value:test_1" in less than 60 seconds
+ And the Minifi logs contain the following message: "key:azure.blobname value:test_2" in less than 60 seconds
+ And the Minifi logs do not contain the following message: "key:azure.blobname value:other_test" after 0 seconds
+ And no errors were generated on the http-proxy regarding "http://azure-storage-server-${scenario_id}:10000/devstoreaccount1/test-container"
diff --git a/libminifi/include/controllers/ProxyConfigurationService.h b/libminifi/include/controllers/ProxyConfigurationService.h
new file mode 100644
index 0000000000..f4b1468afe
--- /dev/null
+++ b/libminifi/include/controllers/ProxyConfigurationService.h
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include
+
+#include "minifi-cpp/controllers/ProxyConfigurationServiceInterface.h"
+#include "controllers/ProxyConfiguration.h"
+#include "core/controller/ControllerService.h"
+#include "core/PropertyDefinitionBuilder.h"
+#include "minifi-cpp/core/PropertyValidator.h"
+
+namespace org::apache::nifi::minifi::controllers {
+
+class ProxyConfigurationService : public core::controller::ControllerServiceImpl, public ProxyConfigurationServiceInterface {
+ public:
+ explicit ProxyConfigurationService(std::string_view name, const utils::Identifier& uuid = {})
+ : ControllerServiceImpl(name, uuid) {
+ }
+
+ MINIFIAPI static constexpr const char* Description = "Provides a set of configurations for different MiNiFi C++ components to use a proxy server. "
+ "Currently these properties can only be used for HTTP proxy configuration, not other protocols are supported at this time.";
+
+ MINIFIAPI static constexpr auto ProxyServerHost = core::PropertyDefinitionBuilder<>::createProperty("Proxy Server Host")
+ .withDescription("Proxy server hostname or ip-address.")
+ .isRequired(true)
+ .build();
+ MINIFIAPI static constexpr auto ProxyServerPort = core::PropertyDefinitionBuilder<>::createProperty("Proxy Server Port")
+ .withDescription("Proxy server port number.")
+ .withValidator(core::StandardPropertyValidators::PORT_VALIDATOR)
+ .build();
+ MINIFIAPI static constexpr auto ProxyUserName = core::PropertyDefinitionBuilder<>::createProperty("Proxy User Name")
+ .withDescription("The name of the proxy client for user authentication.")
+ .build();
+ MINIFIAPI static constexpr auto ProxyUserPassword = core::PropertyDefinitionBuilder<>::createProperty("Proxy User Password")
+ .withDescription("The password of the proxy client for user authentication.")
+ .isSensitive(true)
+ .build();
+ MINIFIAPI static constexpr auto Properties = std::to_array({
+ ProxyServerHost,
+ ProxyServerPort,
+ ProxyUserName,
+ ProxyUserPassword
+ });
+
+ MINIFIAPI static constexpr bool SupportsDynamicProperties = false;
+ MINIFIAPI static constexpr auto ImplementsApis = std::array{ ProxyConfigurationServiceInterface::ProvidesApi };
+ ADD_COMMON_VIRTUAL_FUNCTIONS_FOR_CONTROLLER_SERVICES
+
+ void yield() override {
+ }
+
+ bool isRunning() const override {
+ return getState() == core::controller::ControllerServiceState::ENABLED;
+ }
+
+ bool isWorkAvailable() override {
+ return false;
+ }
+
+ void initialize() override;
+ void onEnable() override;
+
+ ProxyType getProxyType() const override {
+ std::lock_guard lock(configuration_mutex_);
+ return proxy_configuration_.proxy_type;
+ }
+
+ std::string getHost() const override {
+ std::lock_guard lock(configuration_mutex_);
+ return proxy_configuration_.proxy_host;
+ }
+
+ std::optional getPort() const override {
+ std::lock_guard lock(configuration_mutex_);
+ return proxy_configuration_.proxy_port;
+ }
+
+ std::optional getUsername() const override {
+ std::lock_guard lock(configuration_mutex_);
+ return proxy_configuration_.proxy_user;
+ }
+
+ std::optional getPassword() const override {
+ std::lock_guard lock(configuration_mutex_);
+ return proxy_configuration_.proxy_password;
+ }
+
+ private:
+ mutable std::mutex configuration_mutex_;
+ ProxyConfiguration proxy_configuration_;
+ std::shared_ptr logger_ = core::logging::LoggerFactory::getLogger(uuid_);
+};
+
+} // namespace org::apache::nifi::minifi::controllers
diff --git a/libminifi/src/controllers/ProxyConfigurationService.cpp b/libminifi/src/controllers/ProxyConfigurationService.cpp
new file mode 100644
index 0000000000..69564f3fb4
--- /dev/null
+++ b/libminifi/src/controllers/ProxyConfigurationService.cpp
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "controllers/ProxyConfigurationService.h"
+
+#include "utils/ParsingUtils.h"
+#include "minifi-cpp/Exception.h"
+#include "core/Resource.h"
+
+namespace org::apache::nifi::minifi::controllers {
+
+void ProxyConfigurationService::initialize() {
+ setSupportedProperties(Properties);
+}
+
+void ProxyConfigurationService::onEnable() {
+ std::lock_guard lock(configuration_mutex_);
+ proxy_configuration_.proxy_type = ProxyType::HTTP;
+ proxy_configuration_.proxy_host = getProperty(ProxyServerHost.name).value_or("");
+ if (proxy_configuration_.proxy_host.empty()) {
+ logger_->log_error("Proxy Server Host is required");
+ throw minifi::Exception(ExceptionType::PROCESS_SCHEDULE_EXCEPTION, "Proxy Server Host is required");
+ }
+ if (auto proxy_port = getProperty(ProxyServerPort.name) | utils::andThen(parsing::parseIntegral)) {
+ proxy_configuration_.proxy_port = *proxy_port;
+ }
+ if (auto proxy_user = getProperty(ProxyUserName.name)) {
+ proxy_configuration_.proxy_user = *proxy_user;
+ }
+ if (auto proxy_password = getProperty(ProxyUserPassword.name)) {
+ proxy_configuration_.proxy_password = *proxy_password;
+ }
+}
+
+REGISTER_RESOURCE(ProxyConfigurationService, ControllerService);
+
+} // namespace org::apache::nifi::minifi::controllers
diff --git a/libminifi/test/unit/ProxyConfigurationServiceTests.cpp b/libminifi/test/unit/ProxyConfigurationServiceTests.cpp
new file mode 100644
index 0000000000..8bc8f593af
--- /dev/null
+++ b/libminifi/test/unit/ProxyConfigurationServiceTests.cpp
@@ -0,0 +1,63 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "unit/TestBase.h"
+#include "unit/Catch.h"
+#include "controllers/ProxyConfigurationService.h"
+
+namespace org::apache::nifi::minifi::test {
+
+struct ProxyConfigurationServiceTestFixture {
+ ProxyConfigurationServiceTestFixture() {
+ LogTestController::getInstance().clear();
+ LogTestController::getInstance().setTrace();
+ }
+
+ TestController test_controller_;
+ std::shared_ptr plan_ = test_controller_.createPlan();
+ std::shared_ptr proxy_configuration_node_ = plan_->addController("ProxyConfigurationService", "ProxyConfigurationService");
+ std::shared_ptr proxy_configuration_service_ =
+ std::dynamic_pointer_cast(proxy_configuration_node_->getControllerServiceImplementation());
+};
+
+TEST_CASE_METHOD(ProxyConfigurationServiceTestFixture, "ProxyConfigurationService onEnable throws when empty") {
+ REQUIRE_THROWS_WITH(proxy_configuration_service_->onEnable(), "Process Schedule Operation: Proxy Server Host is required");
+}
+
+TEST_CASE_METHOD(ProxyConfigurationServiceTestFixture, "Only required properties are set in ProxyConfigurationService") {
+ plan_->setProperty(proxy_configuration_node_, controllers::ProxyConfigurationService::ProxyServerHost, "192.168.1.123");
+ REQUIRE_NOTHROW(plan_->finalize());
+ CHECK(proxy_configuration_service_->getHost() == "192.168.1.123");
+ CHECK(proxy_configuration_service_->getPort() == std::nullopt);
+ CHECK(proxy_configuration_service_->getUsername() == std::nullopt);
+ CHECK(proxy_configuration_service_->getPassword() == std::nullopt);
+}
+
+TEST_CASE_METHOD(ProxyConfigurationServiceTestFixture, "All properties are set in ProxyConfigurationService") {
+ plan_->setProperty(proxy_configuration_node_, controllers::ProxyConfigurationService::ProxyServerHost, "192.168.1.123");
+ plan_->setProperty(proxy_configuration_node_, controllers::ProxyConfigurationService::ProxyServerPort, "8080");
+ plan_->setProperty(proxy_configuration_node_, controllers::ProxyConfigurationService::ProxyUserName, "user");
+ plan_->setProperty(proxy_configuration_node_, controllers::ProxyConfigurationService::ProxyUserPassword, "password");
+ REQUIRE_NOTHROW(plan_->finalize());
+ CHECK(proxy_configuration_service_->getHost() == "192.168.1.123");
+ CHECK(proxy_configuration_service_->getPort() == 8080);
+ CHECK(proxy_configuration_service_->getUsername() == "user");
+ CHECK(proxy_configuration_service_->getPassword() == "password");
+}
+
+} // namespace org::apache::nifi::minifi::test
diff --git a/minifi-api/include/minifi-cpp/controllers/ProxyConfigurationServiceInterface.h b/minifi-api/include/minifi-cpp/controllers/ProxyConfigurationServiceInterface.h
new file mode 100644
index 0000000000..a39d9572a3
--- /dev/null
+++ b/minifi-api/include/minifi-cpp/controllers/ProxyConfigurationServiceInterface.h
@@ -0,0 +1,43 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include "minifi-cpp/core/controller/ControllerService.h"
+#include "minifi-cpp/core/ControllerServiceApiDefinition.h"
+
+namespace org::apache::nifi::minifi::controllers {
+
+enum class ProxyType {
+ HTTP
+};
+
+class ProxyConfigurationServiceInterface : public virtual core::controller::ControllerService {
+ public:
+ static constexpr auto ProvidesApi = core::ControllerServiceApiDefinition {
+ .artifact = "minifi-system",
+ .group = "org.apache.nifi.minifi",
+ .type = "org.apache.nifi.minifi.controllers.ProxyConfigurationServiceInterface",
+ };
+
+ virtual std::string getHost() const = 0;
+ virtual std::optional getPort() const = 0;
+ virtual std::optional getUsername() const = 0;
+ virtual std::optional getPassword() const = 0;
+ virtual ProxyType getProxyType() const = 0;
+};
+
+} // namespace org::apache::nifi::minifi::controllers