Skip to content

Commit f3cc68b

Browse files
PasichniukPasichniuk
authored
feat: default allowed domains for huggingface inference deployments (#189)
1 parent c77dce5 commit f3cc68b

File tree

7 files changed

+245
-11
lines changed

7 files changed

+245
-11
lines changed

‎docs/configuration.md‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -242,11 +242,12 @@ TLS and routing rules are generated automatically from `app.nim.deploy.cluster-h
242242
| `app.mcp-registry.base-url` | `MCP_REGISTRY_BASE_URL` | `https://registry.modelcontextprotocol.io` | No | - | Base URL of the MCP Registry API (used by the proxy). |
243243

244244
### Hugging Face Configuration
245-
| Property | Environment Variable | Default Value | Required | Applied when | Description |
246-
|----------------------------------|----------------------------------|--------------------------|----------|--------------|--------------------------------------------|
247-
| `huggingface.base-url` | `HUGGINGFACE_BASE_URL` | `https://huggingface.co` | No | - | Base URL for Hugging Face API |
248-
| `huggingface.api-token` | `HUGGINGFACE_API_TOKEN` | - | No | - | API token for authentication |
249-
| `huggingface.tag-cache-duration` | `HUGGINGFACE_TAG_CACHE_DURATION` | `24h` | No | - | Duration to cache tag data (e.g. 24h, 60m) |
245+
| Property | Environment Variable | Default Value | Required | Applied when | Description |
246+
|--------------------------------------------|----------------------------------------|-----------------------------------------------------------------|----------|--------------|----------------------------------------------------------------------------------------------------------------------------------|
247+
| `app.huggingface.base-url` | `HUGGINGFACE_BASE_URL` | `https://huggingface.co` | No | - | Base URL for Hugging Face API |
248+
| `app.huggingface.api-token` | `HUGGINGFACE_API_TOKEN` | - | No | - | API token for authentication |
249+
| `app.huggingface.tag-cache-duration` | `HUGGINGFACE_TAG_CACHE_DURATION` | `24h` | No | - | Duration to cache tag data (e.g. 24h, 60m) |
250+
| `app.huggingface.default-allowed-domains` | `HUGGINGFACE_DEFAULT_ALLOWED_DOMAINS` | `huggingface.co,transfer.xethub.hf.co,cas-server.xethub.hf.co` | No | - | Comma-separated list of default domains added to Cilium network policy egress for inference deployments with HuggingFace source. |
250251

251252
### Validation Configuration
252253

‎src/main/java/com/epam/aidial/deployment/manager/huggingface/properties/HuggingFaceProperties.java‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,56 @@
11
package com.epam.aidial.deployment.manager.huggingface.properties;
22

3+
import lombok.AccessLevel;
34
import lombok.Getter;
45
import lombok.Setter;
6+
import org.apache.commons.lang3.StringUtils;
57
import org.springframework.boot.context.properties.ConfigurationProperties;
68
import org.springframework.stereotype.Component;
79

810
import java.time.Duration;
11+
import java.util.Collections;
12+
import java.util.List;
13+
import java.util.stream.Stream;
14+
import javax.annotation.PostConstruct;
15+
16+
import static java.util.stream.Collectors.toList;
917

1018
@Getter
1119
@Setter
1220
@Component
1321
@ConfigurationProperties(prefix = "app.huggingface")
1422
public class HuggingFaceProperties {
23+
1524
private String baseUrl;
1625
private String apiToken;
1726
private Duration tagCacheDuration;
27+
28+
@Getter(AccessLevel.NONE)
29+
private String defaultAllowedDomains;
30+
31+
@Getter(AccessLevel.NONE)
32+
private List<String> parsedDefaultAllowedDomains;
33+
34+
@PostConstruct
35+
void initDefaultAllowedDomains() {
36+
if (StringUtils.isBlank(defaultAllowedDomains)) {
37+
parsedDefaultAllowedDomains = List.of();
38+
} else {
39+
parsedDefaultAllowedDomains = Stream.of(defaultAllowedDomains.split(","))
40+
.map(String::trim)
41+
.filter(s -> !s.isEmpty())
42+
.collect(toList());
43+
}
44+
}
45+
46+
/**
47+
* Returns the default allowed domains for HuggingFace inference deployments egress.
48+
* Parsed once at startup from the comma-separated {@link #defaultAllowedDomains} property.
49+
* Null check is required for cases when method is called before init is done.
50+
*/
51+
public List<String> getDefaultAllowedDomains() {
52+
return parsedDefaultAllowedDomains == null
53+
? List.of()
54+
: Collections.unmodifiableList(parsedDefaultAllowedDomains);
55+
}
1856
}

‎src/main/java/com/epam/aidial/deployment/manager/service/deployment/AbstractDeploymentManager.java‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ public Deployment deploy(String id) {
113113
@Override
114114
public void afterCommit() {
115115
try {
116-
createCiliumNetworkPolicy(id, deployment.getAllowedDomains(), getCiliumIngressPorts(deployment));
116+
createCiliumNetworkPolicy(id, getEffectiveDeploymentAllowedDomains(deployment), getCiliumIngressPorts(deployment));
117117
createService(namespace, serviceSpec);
118118
} catch (Exception e) {
119119
var errorMessage = "Failed to deploy service '%s'".formatted(id);
@@ -741,7 +741,7 @@ public void updateCiliumNetworkPolicy(String id) {
741741
var deployment = getDeployment(id);
742742

743743
try {
744-
updateCiliumNetworkPolicy(id, deployment.getAllowedDomains(), getCiliumIngressPorts(deployment));
744+
updateCiliumNetworkPolicy(id, getEffectiveDeploymentAllowedDomains(deployment), getCiliumIngressPorts(deployment));
745745
} catch (Exception e) {
746746
var errorMessage = "Cilium Network Policy update failed for deployment '%s'".formatted(id);
747747
log.warn(errorMessage, e);
@@ -785,10 +785,15 @@ private void deleteCiliumNetworkPolicy(String name) {
785785
protected Set<Integer> getCiliumIngressPorts(D deployment) {
786786
Set<Integer> ports = new HashSet<>();
787787
Optional.ofNullable(deployment.getContainerPort()).ifPresent(ports::add);
788-
Optional.ofNullable(defaultContainerPort).ifPresent(ports::add);
788+
ports.add(defaultContainerPort);
789789
return ports;
790790
}
791791

792+
protected List<String> getEffectiveDeploymentAllowedDomains(D deployment) {
793+
var domains = deployment.getAllowedDomains();
794+
return domains != null ? new ArrayList<>(domains) : new ArrayList<>();
795+
}
796+
792797
private Map<String, String> transformEnvs(Map<String, EnvVarValue> envs) {
793798
return MapUtils.emptyIfNull(envs).entrySet().stream()
794799
.map(entry -> {

‎src/main/java/com/epam/aidial/deployment/manager/service/deployment/InferenceDeploymentManager.java‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,15 @@
55
import com.epam.aidial.deployment.manager.configuration.KserveDeployProperties;
66
import com.epam.aidial.deployment.manager.configuration.logging.LogExecution;
77
import com.epam.aidial.deployment.manager.dao.repository.DeploymentRepository;
8+
import com.epam.aidial.deployment.manager.huggingface.properties.HuggingFaceProperties;
89
import com.epam.aidial.deployment.manager.kubernetes.K8sClient;
910
import com.epam.aidial.deployment.manager.kubernetes.kserve.K8sKserveClient;
1011
import com.epam.aidial.deployment.manager.model.DeploymentStatus;
1112
import com.epam.aidial.deployment.manager.model.SensitiveEnvVar;
1213
import com.epam.aidial.deployment.manager.model.SimpleEnvVar;
1314
import com.epam.aidial.deployment.manager.model.deployment.Deployment;
1415
import com.epam.aidial.deployment.manager.model.deployment.InferenceDeployment;
16+
import com.epam.aidial.deployment.manager.model.deployment.InferenceDeploymentHuggingFaceSource;
1517
import com.epam.aidial.deployment.manager.service.manifest.InferenceManifestGenerator;
1618
import com.epam.aidial.deployment.manager.service.manifest.ManifestGenerator;
1719
import com.epam.aidial.deployment.manager.service.pipeline.specification.CiliumNetworkPolicyCreator;
@@ -22,11 +24,13 @@
2224
import io.kserve.serving.v1beta1.inferenceservicestatus.ModelStatus;
2325
import io.kserve.serving.v1beta1.inferenceservicestatus.modelstatus.States;
2426
import lombok.extern.slf4j.Slf4j;
27+
import org.apache.commons.collections4.CollectionUtils;
2528
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
2629
import org.springframework.stereotype.Component;
2730

2831
import java.util.List;
2932
import java.util.Optional;
33+
import java.util.stream.Stream;
3034

3135
@Slf4j
3236
@Component
@@ -40,6 +44,7 @@ public class InferenceDeploymentManager extends AbstractModelDeploymentManager<I
4044
private final InferenceManifestGenerator inferenceManifestGenerator;
4145
private final K8sKserveClient k8sKserveClient;
4246
private final boolean useClusterInternalUrl;
47+
private final List<String> defaultAllowedDomains;
4348

4449
public InferenceDeploymentManager(
4550
K8sClient k8sClient,
@@ -50,14 +55,16 @@ public InferenceDeploymentManager(
5055
CiliumNetworkPolicyCreator ciliumNetworkPolicyCreator,
5156
DeploymentRepository deploymentRepository,
5257
K8sKserveClient k8sKserveClient,
53-
KserveDeployProperties kserveDeployProperties
58+
KserveDeployProperties kserveDeployProperties,
59+
HuggingFaceProperties huggingFaceProperties
5460
) {
5561
super(k8sClient, disposableResourceManager, manifestGenerator, deploymentRepository,
5662
containerPortResolver, ciliumNetworkPolicyCreator, kserveDeployProperties.getNamespace(),
5763
kserveDeployProperties.getStartupTimeout(), DEFAULT_KSERVE_SERVICE_PORT);
5864
this.inferenceManifestGenerator = inferenceManifestGenerator;
5965
this.k8sKserveClient = k8sKserveClient;
6066
this.useClusterInternalUrl = kserveDeployProperties.isUseClusterInternalUrl();
67+
this.defaultAllowedDomains = huggingFaceProperties.getDefaultAllowedDomains();
6168
}
6269

6370
@Override
@@ -228,6 +235,22 @@ protected String getServiceNameLabel() {
228235
return SERVICE_NAME_LABEL;
229236
}
230237

238+
@Override
239+
protected List<String> getEffectiveDeploymentAllowedDomains(InferenceDeployment deployment) {
240+
List<String> allowedDomains = super.getEffectiveDeploymentAllowedDomains(deployment);
241+
242+
if (!(deployment.getSource() instanceof InferenceDeploymentHuggingFaceSource)
243+
|| CollectionUtils.isEmpty(defaultAllowedDomains)) {
244+
return allowedDomains;
245+
}
246+
247+
return Stream.concat(
248+
allowedDomains.stream(),
249+
defaultAllowedDomains.stream())
250+
.distinct()
251+
.toList();
252+
}
253+
231254
private String getClusterInternalUrl(Components predictor, String serviceName) {
232255
var address = predictor.getAddress();
233256
if (address == null) {

‎src/main/resources/application.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ app:
3030
base-url: ${HUGGINGFACE_BASE_URL:https://huggingface.co}
3131
api-token: ${HUGGINGFACE_API_TOKEN:}
3232
tag-cache-duration: ${HUGGINGFACE_TAG_CACHE_DURATION:24h}
33+
default-allowed-domains: ${HUGGINGFACE_DEFAULT_ALLOWED_DOMAINS:huggingface.co,transfer.xethub.hf.co,cas-server.xethub.hf.co}
3334

3435
mcp-registry:
3536
base-url: ${MCP_REGISTRY_BASE_URL:https://registry.modelcontextprotocol.io}

‎src/test/java/com/epam/aidial/deployment/manager/functional/config/FunctionalTestConfiguration.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
"com.epam.aidial.deployment.manager.configuration",
3030
"com.epam.aidial.deployment.manager.dao",
3131
"com.epam.aidial.deployment.manager.docker",
32+
"com.epam.aidial.deployment.manager.huggingface",
3233
"com.epam.aidial.deployment.manager.kubernetes",
3334
"com.epam.aidial.deployment.manager.mapper",
3435
"com.epam.aidial.deployment.manager.model",

0 commit comments

Comments
 (0)