Skip to content

Commit 132ce2a

Browse files
lokeshranginenilokeshrangineni
authored
feat: Loading the CA trusted store certificate into Feast to verify the public certificate. (feast-dev#4852)
* Initial Draft version to load the CA trusted store code. Signed-off-by: lrangine <[email protected]> * Initial Draft version to load the CA trusted store code. Signed-off-by: lrangine <[email protected]> * Fixing the lint error. Signed-off-by: lrangine <[email protected]> * Trying to fix the online store test cases. Signed-off-by: lrangine <[email protected]> * Formatted the python to fix lint errors. Signed-off-by: lrangine <[email protected]> * Fixing the unit test cases. Signed-off-by: lrangine <[email protected]> * Fixing the unit test cases. Signed-off-by: lrangine <[email protected]> * removing unnecessary cli args. Signed-off-by: lrangine <[email protected]> * Now configuring the SSL ca store configurations on the feast client side rather than on the server side. And also fixing the integration tests. Signed-off-by: lrangine <[email protected]> * Renamed the remote registry is_tls_mode variable to is_tls. Changed the offline store TLS setting decision from cert to scheme. Signed-off-by: lrangine <[email protected]> * Adding the existing trust store certificates to the newly created trust store. Signed-off-by: lrangine <[email protected]> * Clearing the existing trust store configuration to see if it fixes the PR integration failures. Signed-off-by: lrangine <[email protected]> * Clearing the existing trust store configuration to see if it fixes the PR integration failures. Signed-off-by: lrangine <[email protected]> * Clearing the existing trust store configuration to see if it fixes the PR integration failures. Signed-off-by: lrangine <[email protected]> * combining the default system ca store with the custom one to fix the integration tests. Signed-off-by: lrangine <[email protected]> * Final clean up and adding documentation. Signed-off-by: lrangine <[email protected]> * Incorporating the code review comments from Francisco. Signed-off-by: lrangine <[email protected]> --------- Signed-off-by: lrangine <[email protected]>
1 parent 739eaa7 commit 132ce2a

File tree

13 files changed

+320
-115
lines changed

13 files changed

+320
-115
lines changed

‎docs/how-to-guides/starting-feast-servers-tls-mode.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,3 +189,8 @@ INFO: Waiting for application startup.
189189
INFO: Application startup complete.
190190
INFO: Uvicorn running on https://0.0.0.0:8888 (Press CTRL+C to quit)
191191
```
192+
193+
194+
## Adding public key to CA trust store and configuring the feast to use the trust store.
195+
You can pass the public key for SSL verification using the `cert` parameter, however, it is sometimes difficult to maintain individual certificates and pass them individually.
196+
The alternative recommendation is to add the public certificate to CA trust store and set the path as an environment variable (e.g., `FEAST_CA_CERT_FILE_PATH`). Feast will use the trust store path in the `FEAST_CA_CERT_FILE_PATH` environment variable.

‎sdk/python/feast/cli.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -982,7 +982,6 @@ def serve_command(
982982
raise click.BadParameter(
983983
"Please pass --cert and --key args to start the feature server in TLS mode."
984984
)
985-
986985
store = create_feature_store(ctx)
987986

988987
store.serve(

‎sdk/python/feast/feature_store.py

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@
8686
from feast.repo_config import RepoConfig, load_repo_config
8787
from feast.repo_contents import RepoContents
8888
from feast.saved_dataset import SavedDataset, SavedDatasetStorage, ValidationReference
89+
from feast.ssl_ca_trust_store_setup import configure_ca_trust_store_env_variables
8990
from feast.stream_feature_view import StreamFeatureView
9091
from feast.utils import _utc_now
9192

@@ -129,6 +130,8 @@ def __init__(
129130
if fs_yaml_file is not None and config is not None:
130131
raise ValueError("You cannot specify both fs_yaml_file and config.")
131132

133+
configure_ca_trust_store_env_variables()
134+
132135
if repo_path:
133136
self.repo_path = Path(repo_path)
134137
else:
@@ -1949,13 +1952,19 @@ def serve_ui(
19491952
)
19501953

19511954
def serve_registry(
1952-
self, port: int, tls_key_path: str = "", tls_cert_path: str = ""
1955+
self,
1956+
port: int,
1957+
tls_key_path: str = "",
1958+
tls_cert_path: str = "",
19531959
) -> None:
19541960
"""Start registry server locally on a given port."""
19551961
from feast import registry_server
19561962

19571963
registry_server.start_server(
1958-
self, port=port, tls_key_path=tls_key_path, tls_cert_path=tls_cert_path
1964+
self,
1965+
port=port,
1966+
tls_key_path=tls_key_path,
1967+
tls_cert_path=tls_cert_path,
19591968
)
19601969

19611970
def serve_offline(

‎sdk/python/feast/infra/offline_stores/remote.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ def build_arrow_flight_client(
7474
scheme: str, host: str, port, auth_config: AuthConfig, cert: str = ""
7575
):
7676
arrow_scheme = "grpc+tcp"
77-
if cert:
77+
if scheme == "https":
7878
logger.info(
7979
"Scheme is https so going to connect offline server in SSL(TLS) mode."
8080
)

‎sdk/python/feast/infra/registry/remote.py

Lines changed: 28 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import os
12
from datetime import datetime
23
from pathlib import Path
34
from typing import List, Optional, Union
@@ -59,6 +60,12 @@ class RemoteRegistryConfig(RegistryConfig):
5960
""" str: Path to the public certificate when the registry server starts in TLS(SSL) mode. This may be needed if the registry server started with a self-signed certificate, typically this file ends with `*.crt`, `*.cer`, or `*.pem`.
6061
If registry_type is 'remote', then this configuration is needed to connect to remote registry server in TLS mode. If the remote registry started in non-tls mode then this configuration is not needed."""
6162

63+
is_tls: bool = False
64+
""" bool: Set to `True` if you plan to connect to a registry server running in TLS (SSL) mode.
65+
If you intend to add the public certificate to the trust store instead of passing it via the `cert` parameter, this field must be set to `True`.
66+
If you are planning to add the public certificate as part of the trust store instead of passing it as a `cert` parameters then setting this field to `true` is mandatory.
67+
"""
68+
6269

6370
class RemoteRegistry(BaseRegistry):
6471
def __init__(
@@ -70,20 +77,32 @@ def __init__(
7077
):
7178
self.auth_config = auth_config
7279
assert isinstance(registry_config, RemoteRegistryConfig)
73-
if registry_config.cert:
74-
with open(registry_config.cert, "rb") as cert_file:
75-
trusted_certs = cert_file.read()
76-
tls_credentials = grpc.ssl_channel_credentials(
77-
root_certificates=trusted_certs
78-
)
79-
self.channel = grpc.secure_channel(registry_config.path, tls_credentials)
80-
else:
81-
self.channel = grpc.insecure_channel(registry_config.path)
80+
self.channel = self._create_grpc_channel(registry_config)
8281

8382
auth_header_interceptor = GrpcClientAuthHeaderInterceptor(auth_config)
8483
self.channel = grpc.intercept_channel(self.channel, auth_header_interceptor)
8584
self.stub = RegistryServer_pb2_grpc.RegistryServerStub(self.channel)
8685

86+
def _create_grpc_channel(self, registry_config):
87+
assert isinstance(registry_config, RemoteRegistryConfig)
88+
if registry_config.cert or registry_config.is_tls:
89+
cafile = os.getenv("SSL_CERT_FILE") or os.getenv("REQUESTS_CA_BUNDLE")
90+
if not cafile and not registry_config.cert:
91+
raise EnvironmentError(
92+
"SSL_CERT_FILE or REQUESTS_CA_BUNDLE environment variable must be set to use secure TLS or set the cert parameter in feature_Store.yaml file under remote registry configuration."
93+
)
94+
with open(
95+
registry_config.cert if registry_config.cert else cafile, "rb"
96+
) as cert_file:
97+
trusted_certs = cert_file.read()
98+
tls_credentials = grpc.ssl_channel_credentials(
99+
root_certificates=trusted_certs
100+
)
101+
return grpc.secure_channel(registry_config.path, tls_credentials)
102+
else:
103+
# Create an insecure gRPC channel
104+
return grpc.insecure_channel(registry_config.path)
105+
87106
def close(self):
88107
if self.channel:
89108
self.channel.close()

‎sdk/python/feast/ssl_ca_trust_store_setup.py

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
import logging
2+
import os
3+
4+
logger = logging.getLogger(__name__)
5+
logger.setLevel(logging.INFO)
6+
7+
8+
def configure_ca_trust_store_env_variables():
9+
"""
10+
configures the environment variable so that other libraries or servers refer to the TLS ca file path.
11+
:param ca_file_path:
12+
:return:
13+
"""
14+
if (
15+
"FEAST_CA_CERT_FILE_PATH" in os.environ
16+
and os.environ["FEAST_CA_CERT_FILE_PATH"]
17+
):
18+
logger.info(
19+
f"Feast CA Cert file path found in environment variable FEAST_CA_CERT_FILE_PATH={os.environ['FEAST_CA_CERT_FILE_PATH']}. Going to refer this path."
20+
)
21+
os.environ["SSL_CERT_FILE"] = os.environ["FEAST_CA_CERT_FILE_PATH"]
22+
os.environ["REQUESTS_CA_BUNDLE"] = os.environ["FEAST_CA_CERT_FILE_PATH"]

‎sdk/python/tests/conftest.py

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,12 @@
5757
location,
5858
)
5959
from tests.utils.auth_permissions_util import default_store
60-
from tests.utils.generate_self_signed_certifcate_util import generate_self_signed_cert
6160
from tests.utils.http_server import check_port_open, free_port # noqa: E402
61+
from tests.utils.ssl_certifcates_util import (
62+
combine_trust_stores,
63+
create_ca_trust_store,
64+
generate_self_signed_cert,
65+
)
6266

6367
logger = logging.getLogger(__name__)
6468

@@ -514,17 +518,36 @@ def auth_config(request, is_integration_test):
514518
return auth_configuration
515519

516520

517-
@pytest.fixture(params=[True, False], scope="module")
521+
@pytest.fixture(scope="module")
518522
def tls_mode(request):
519-
is_tls_mode = request.param
523+
is_tls_mode = request.param[0]
524+
output_combined_truststore_path = ""
520525

521526
if is_tls_mode:
522527
certificates_path = tempfile.mkdtemp()
523528
tls_key_path = os.path.join(certificates_path, "key.pem")
524529
tls_cert_path = os.path.join(certificates_path, "cert.pem")
530+
525531
generate_self_signed_cert(cert_path=tls_cert_path, key_path=tls_key_path)
532+
is_ca_trust_store_set = request.param[1]
533+
if is_ca_trust_store_set:
534+
# Paths
535+
feast_ca_trust_store_path = os.path.join(
536+
certificates_path, "feast_trust_store.pem"
537+
)
538+
create_ca_trust_store(
539+
public_key_path=tls_cert_path,
540+
private_key_path=tls_key_path,
541+
output_trust_store_path=feast_ca_trust_store_path,
542+
)
543+
544+
# Combine trust stores
545+
output_combined_path = os.path.join(
546+
certificates_path, "combined_trust_store.pem"
547+
)
548+
combine_trust_stores(feast_ca_trust_store_path, output_combined_path)
526549
else:
527550
tls_key_path = ""
528551
tls_cert_path = ""
529552

530-
return is_tls_mode, tls_key_path, tls_cert_path
553+
return is_tls_mode, tls_key_path, tls_cert_path, output_combined_truststore_path

‎sdk/python/tests/integration/feature_repos/universal/data_sources/file.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,8 @@
3434
DataSourceCreator,
3535
)
3636
from tests.utils.auth_permissions_util import include_auth_config
37-
from tests.utils.generate_self_signed_certifcate_util import generate_self_signed_cert
3837
from tests.utils.http_server import check_port_open, free_port # noqa: E402
38+
from tests.utils.ssl_certifcates_util import generate_self_signed_cert
3939

4040
logger = logging.getLogger(__name__)
4141

‎sdk/python/tests/integration/online_store/test_remote_online_store.py

Lines changed: 28 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@
2222

2323

2424
@pytest.mark.integration
25+
@pytest.mark.parametrize(
26+
"tls_mode", [("True", "True"), ("True", "False"), ("False", "")], indirect=True
27+
)
2528
def test_remote_online_store_read(auth_config, tls_mode):
2629
with (
2730
tempfile.TemporaryDirectory() as remote_server_tmp_dir,
@@ -56,13 +59,13 @@ def test_remote_online_store_read(auth_config, tls_mode):
5659
)
5760
)
5861
assert None not in (server_store, server_url, registry_path)
59-
_, _, tls_cert_path = tls_mode
62+
6063
client_store = _create_remote_client_feature_store(
6164
temp_dir=remote_client_tmp_dir,
6265
server_registry_path=str(registry_path),
6366
feature_server_url=server_url,
6467
auth_config=auth_config,
65-
tls_cert_path=tls_cert_path,
68+
tls_mode=tls_mode,
6669
)
6770
assert client_store is not None
6871
_assert_non_existing_entity_feature_views_entity(
@@ -172,14 +175,15 @@ def _create_server_store_spin_feature_server(
172175
):
173176
store = default_store(str(temp_dir), auth_config, permissions_list)
174177
feast_server_port = free_port()
175-
is_tls_mode, tls_key_path, tls_cert_path = tls_mode
178+
is_tls_mode, tls_key_path, tls_cert_path, ca_trust_store_path = tls_mode
176179

177180
server_url = next(
178181
start_feature_server(
179182
repo_path=str(store.repo_path),
180183
server_port=feast_server_port,
181184
tls_key_path=tls_key_path,
182185
tls_cert_path=tls_cert_path,
186+
ca_trust_store_path=ca_trust_store_path,
183187
)
184188
)
185189
if is_tls_mode:
@@ -203,20 +207,33 @@ def _create_remote_client_feature_store(
203207
server_registry_path: str,
204208
feature_server_url: str,
205209
auth_config: str,
206-
tls_cert_path: str = "",
210+
tls_mode,
207211
) -> FeatureStore:
208212
project_name = "REMOTE_ONLINE_CLIENT_PROJECT"
209213
runner = CliRunner()
210214
result = runner.run(["init", project_name], cwd=temp_dir)
211215
assert result.returncode == 0
212216
repo_path = os.path.join(temp_dir, project_name, "feature_repo")
213-
_overwrite_remote_client_feature_store_yaml(
214-
repo_path=str(repo_path),
215-
registry_path=server_registry_path,
216-
feature_server_url=feature_server_url,
217-
auth_config=auth_config,
218-
tls_cert_path=tls_cert_path,
219-
)
217+
is_tls_mode, _, tls_cert_path, ca_trust_store_path = tls_mode
218+
if is_tls_mode and not ca_trust_store_path:
219+
_overwrite_remote_client_feature_store_yaml(
220+
repo_path=str(repo_path),
221+
registry_path=server_registry_path,
222+
feature_server_url=feature_server_url,
223+
auth_config=auth_config,
224+
tls_cert_path=tls_cert_path,
225+
)
226+
else:
227+
_overwrite_remote_client_feature_store_yaml(
228+
repo_path=str(repo_path),
229+
registry_path=server_registry_path,
230+
feature_server_url=feature_server_url,
231+
auth_config=auth_config,
232+
)
233+
234+
if is_tls_mode and ca_trust_store_path:
235+
# configure trust store path only when is_tls_mode and ca_trust_store_path exists.
236+
os.environ["FEAST_CA_CERT_FILE_PATH"] = ca_trust_store_path
220237

221238
return FeatureStore(repo_path=repo_path)
222239

‎sdk/python/tests/unit/permissions/auth/server/test_auth_registry_server.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ def start_registry_server(
4444

4545
assertpy.assert_that(server_port).is_not_equal_to(0)
4646

47-
is_tls_mode, tls_key_path, tls_cert_path = tls_mode
47+
is_tls_mode, tls_key_path, tls_cert_path, tls_ca_file_path = tls_mode
4848
if is_tls_mode:
4949
print(f"Starting Registry in TLS mode at {server_port}")
5050
server = start_server(
@@ -74,6 +74,9 @@ def start_registry_server(
7474
server.stop(grace=None) # Teardown server
7575

7676

77+
@pytest.mark.parametrize(
78+
"tls_mode", [("True", "True"), ("True", "False"), ("False", "")], indirect=True
79+
)
7780
def test_registry_apis(
7881
auth_config,
7982
tls_mode,

0 commit comments

Comments
 (0)