Skip to content

Commit 072d286

Browse files
MarcinFalkowskiMarcinFalkowski
authored
add x-service-tag-preference header to upstream request (#370)
1 parent 19d8dab commit 072d286

File tree

12 files changed

+183
-42
lines changed

12 files changed

+183
-42
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
77

88
### Changed
99
- move min & max envoy versions inside artifact to be accessible for dependant projects
10+
- add x-service-tag-preference header to upstream request
1011

1112
## [0.19.30]
1213

‎docs/configuration.md‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -139,16 +139,17 @@ Property
139139
**envoy-control.envoy.snapshot.load-balancing.use-keys-subset-fallback-policy** | KEYS_SUBSET fallback policy is used by default when canary and service-tags are enabled. It is not supported in Envoy <= 1.12.x. Set to false for compatibility with Envoy 1.12.x | true
140140

141141
## Routing
142-
Property | Description | Default value
143-
------------------------------------------------------------------------------------------- |----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| ---------
144-
**envoy-control.envoy.snapshot.routing.service-tags.enabled** | If set to true, service tags routing will be enabled | false
145-
**envoy-control.envoy.snapshot.routing.service-tags.metadata-key** | What key to use in endpoint metadata to store its service tags | tag
146-
**envoy-control.envoy.snapshot.routing.service-tags.header** | What header to use in service tag rules | x-service-tag
142+
Property | Description | Default value
143+
------------------------------------------------------------------------------------------- |-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| ---------
144+
**envoy-control.envoy.snapshot.routing.service-tags.enabled** | If set to true, service tags routing will be enabled | false
145+
**envoy-control.envoy.snapshot.routing.service-tags.metadata-key** | What key to use in endpoint metadata to store its service tags | tag
146+
**envoy-control.envoy.snapshot.routing.service-tags.header** | What header to use in service tag rules | x-service-tag
147+
**envoy-control.envoy.snapshot.routing.service-tags.preference-header** | What header to use for service tag preference list. Used for sending info to upstream if 'auto service tags' is in force. In the future also read from downstream request. | x-service-tag-preference
147148
**envoy-control.envoy.snapshot.routing.service-tags.routing-excluded-tags** | List of tags predicates that cannot be used for routing. This supports an exact matching (just "string" - EXACT matching) prefixes (PREFIX matching) and regexes (REGEX matching) | empty list
148-
**envoy-control.envoy.snapshot.routing.service-tags.allowed-tags-combinations** | List of rules, which tags can be conbined together and requested together. Details below | empty list
149-
**(...).allowed-tags-combinations[].service-name** | The rule will apply only for this service | ""
150-
**(...).allowed-tags-combinations[].tags** | List of tag patterns, that can be combined and requested together | empty list
151-
**envoy-control.envoy.snapshot.routing.service-tags.auto-service-tag-enabled** | Enable auto service tag feature. (`enabled` needs also be true) | false
149+
**envoy-control.envoy.snapshot.routing.service-tags.allowed-tags-combinations** | List of rules, which tags can be conbined together and requested together. Details below | empty list
150+
**(...).allowed-tags-combinations[].service-name** | The rule will apply only for this service | ""
151+
**(...).allowed-tags-combinations[].tags** | List of tag patterns, that can be combined and requested together | empty list
152+
**envoy-control.envoy.snapshot.routing.service-tags.auto-service-tag-enabled** | Enable auto service tag feature. (`enabled` needs also be true) | false
152153

153154
## Outlier detection
154155
Property | Description | Default value

‎envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/SnapshotProperties.kt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -237,9 +237,12 @@ class ServiceTagsProperties {
237237
var enabled = false
238238
var metadataKey = "tag"
239239
var header = "x-service-tag"
240+
var preferenceHeader = "x-service-tag-preference"
240241
var routingExcludedTags: MutableList<StringMatcher> = mutableListOf()
241242
var allowedTagsCombinations: MutableList<ServiceTagsCombinationsProperties> = mutableListOf()
242243
var autoServiceTagEnabled = false
244+
245+
fun isAutoServiceTagEffectivelyEnabled() = enabled && autoServiceTagEnabled
243246
}
244247

245248
class StringMatcher {

‎envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/endpoints/EnvoyEndpointsFactory.kt‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ class EnvoyEndpointsFactory(
5252
clusterLoadAssignment: ClusterLoadAssignment,
5353
routingPolicy: RoutingPolicy
5454
): ClusterLoadAssignment {
55-
if (!routingPolicy.autoServiceTag || !isAutoServiceTagEnabled()) {
55+
if (!routingPolicy.autoServiceTag || !properties.routing.serviceTags.isAutoServiceTagEffectivelyEnabled()) {
5656
return clusterLoadAssignment
5757
}
5858

@@ -67,8 +67,6 @@ class EnvoyEndpointsFactory(
6767
}
6868
}
6969

70-
private fun isAutoServiceTagEnabled() = properties.routing.serviceTags.run { enabled && autoServiceTagEnabled }
71-
7270
private fun filterEndpoints(loadAssignment: ClusterLoadAssignment, tag: String): ClusterLoadAssignment? {
7371
var allEndpointMatched = true
7472
val filteredEndpoints = loadAssignment.endpointsList.mapNotNull { localityLbEndpoint ->

‎envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/routes/EnvoyEgressRoutesFactory.kt‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,24 @@ class EnvoyEgressRoutesFactory(
139139
.build()
140140
)
141141
}
142+
143+
if (properties.routing.serviceTags.isAutoServiceTagEffectivelyEnabled()) {
144+
val routingPolicy = routeSpecification.settings.routingPolicy
145+
if (routingPolicy.autoServiceTag) {
146+
val tagsPreferenceJoined = routingPolicy.serviceTagPreference.joinToString("|")
147+
virtualHost.addRequestHeadersToAdd(
148+
HeaderValueOption.newBuilder()
149+
.setHeader(
150+
HeaderValue.newBuilder()
151+
.setKey(properties.routing.serviceTags.preferenceHeader)
152+
.setValue(tagsPreferenceJoined)
153+
)
154+
.setAppendAction(HeaderValueOption.HeaderAppendAction.OVERWRITE_IF_EXISTS_OR_ADD)
155+
.setKeepEmptyValue(false)
156+
)
157+
}
158+
}
159+
142160
return virtualHost.build()
143161
}
144162

‎envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/config/envoy/EnvoyExtension.kt‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import pl.allegro.tech.servicemesh.envoycontrol.config.RandomConfigFile
1414
import pl.allegro.tech.servicemesh.envoycontrol.config.envoycontrol.EnvoyControlExtensionBase
1515
import pl.allegro.tech.servicemesh.envoycontrol.config.service.ServiceExtension
1616
import pl.allegro.tech.servicemesh.envoycontrol.logger
17+
import java.time.Duration
1718

1819
class EnvoyExtension(
1920
private val envoyControl: EnvoyControlExtensionBase,
@@ -82,6 +83,38 @@ class EnvoyExtension(
8283
}
8384
}
8485

86+
fun waitForClusterEndpointHealthy(
87+
cluster: String,
88+
endpointIp: String
89+
) {
90+
untilAsserted(wait = Duration.ofSeconds(5)) {
91+
assertThat(container.admin().isEndpointHealthy(cluster, endpointIp))
92+
.withFailMessage {
93+
"Expected to see healthy endpoint of cluster '$cluster' with address " +
94+
"'$endpointIp' in envoy ${container.adminUrl()}/clusters, " +
95+
"but it's not present. Found following endpoints: " +
96+
"${container.admin().endpointsAddress(cluster)}"
97+
}
98+
.isTrue()
99+
}
100+
}
101+
102+
fun waitForClusterEndpointNotHealthy(
103+
cluster: String,
104+
endpointIp: String
105+
) {
106+
untilAsserted(wait = Duration.ofSeconds(5)) {
107+
assertThat(container.admin().isEndpointHealthy(cluster, endpointIp))
108+
.withFailMessage {
109+
"Expected to not see endpoint of cluster '$cluster' with address " +
110+
"'$endpointIp' in envoy ${container.adminUrl()}/clusters, " +
111+
"but it's still present. Found following endpoints: " +
112+
"${container.admin().endpointsAddress(cluster)}"
113+
}
114+
.isFalse()
115+
}
116+
}
117+
85118
fun recordRBACLogs() {
86119
container.logRecorder.recordLogs(::isRbacAccessLog)
87120
}

envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/CanaryLoadBalancingTest.kt renamed to envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/routing/CanaryLoadBalancingTest.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package pl.allegro.tech.servicemesh.envoycontrol
1+
package pl.allegro.tech.servicemesh.envoycontrol.routing
22

33
import org.assertj.core.api.Assertions.assertThat
44
import org.junit.jupiter.api.Test

envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/RoutingPolicyTest.kt renamed to envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/routing/RoutingPolicyTest.kt

Lines changed: 6 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package pl.allegro.tech.servicemesh.envoycontrol
1+
package pl.allegro.tech.servicemesh.envoycontrol.routing
22

33
import org.assertj.core.api.Assertions.assertThat
44
import org.junit.jupiter.api.Test
@@ -306,35 +306,18 @@ class RoutingPolicyTest {
306306
serviceInstance: EchoServiceExtension,
307307
envoy: EnvoyExtension
308308
) {
309-
untilAsserted(wait = Duration.ofSeconds(5)) {
310-
assertThat(envoy.container.admin().isEndpointHealthy(serviceName, serviceInstance.container().ipAddress()))
311-
.withFailMessage {
312-
"Expected to see healthy endpoint of cluster '$serviceName' with address " +
313-
"'${serviceInstance.container().address()}' in envoy " +
314-
"${serviceInstance.container().address()}/clusters, " +
315-
"but it's not present. Found following endpoints: " +
316-
"${envoy.container.admin().endpointsAddress(serviceName)}"
317-
}
318-
.isTrue()
319-
}
309+
envoy.waitForClusterEndpointHealthy(cluster = serviceName, endpointIp = serviceInstance.container().ipAddress())
320310
}
321311

322312
private fun waitForEndpointRemoved(
323313
serviceName: String,
324314
serviceInstance: EchoServiceExtension,
325315
envoy: EnvoyExtension
326316
) {
327-
untilAsserted(wait = Duration.ofSeconds(5)) {
328-
assertThat(envoy.container.admin().isEndpointHealthy(serviceName, serviceInstance.container().ipAddress()))
329-
.withFailMessage {
330-
"Expected to not see endpoint of cluster '$serviceName' with address " +
331-
"'${serviceInstance.container().address()}' in envoy " +
332-
"${serviceInstance.container().address()}/clusters, " +
333-
"but it's still present. Found following endpoints: " +
334-
"${envoy.container.admin().endpointsAddress(serviceName)}"
335-
}
336-
.isFalse()
337-
}
317+
envoy.waitForClusterEndpointNotHealthy(
318+
cluster = serviceName,
319+
endpointIp = serviceInstance.container().ipAddress()
320+
)
338321
}
339322

340323
private fun callStats() = CallStats(listOf(ipsumEchoService, loremEchoService, otherEchoService))

‎envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/routing/ServiceTagPreferenceTest.kt‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
package pl.allegro.tech.servicemesh.envoycontrol.routing
2+
3+
import org.assertj.core.api.Assertions.assertThat
4+
import org.junit.jupiter.api.Test
5+
import org.junit.jupiter.api.extension.RegisterExtension
6+
import pl.allegro.tech.servicemesh.envoycontrol.assertions.isOk
7+
import pl.allegro.tech.servicemesh.envoycontrol.config.RandomConfigFile
8+
import pl.allegro.tech.servicemesh.envoycontrol.config.consul.ConsulExtension
9+
import pl.allegro.tech.servicemesh.envoycontrol.config.envoy.EnvoyExtension
10+
import pl.allegro.tech.servicemesh.envoycontrol.config.envoycontrol.EnvoyControlExtension
11+
import pl.allegro.tech.servicemesh.envoycontrol.config.service.GenericServiceExtension
12+
import pl.allegro.tech.servicemesh.envoycontrol.config.service.HttpsEchoContainer
13+
import pl.allegro.tech.servicemesh.envoycontrol.config.service.ServiceExtension
14+
import pl.allegro.tech.servicemesh.envoycontrol.config.service.asHttpsEchoResponse
15+
16+
class ServiceTagPreferenceTest {
17+
18+
companion object {
19+
private val properties = mapOf(
20+
"envoy-control.envoy.snapshot.routing.service-tags.enabled" to true,
21+
"envoy-control.envoy.snapshot.routing.service-tags.auto-service-tag-enabled" to true
22+
)
23+
24+
@JvmField
25+
@RegisterExtension
26+
val consul = ConsulExtension()
27+
28+
@JvmField
29+
@RegisterExtension
30+
val envoyControl = EnvoyControlExtension(consul, properties)
31+
32+
@JvmField
33+
@RegisterExtension
34+
val echoService = GenericServiceExtension(HttpsEchoContainer())
35+
36+
// language=yaml
37+
private var proxySettings = """
38+
node:
39+
metadata:
40+
proxy_settings:
41+
outgoing:
42+
routingPolicy:
43+
autoServiceTag: true
44+
serviceTagPreference: ["ipsum", "lorem"]
45+
fallbackToAnyInstance: true
46+
dependencies:
47+
- service: "echo"
48+
- service: "echo-disabled"
49+
routingPolicy:
50+
autoServiceTag: false
51+
- service: "echo-one-tag"
52+
routingPolicy:
53+
serviceTagPreference: ["one"]
54+
- service: "echo-no-tag"
55+
routingPolicy:
56+
serviceTagPreference: []
57+
58+
""".trimIndent()
59+
60+
@JvmField
61+
@RegisterExtension
62+
val envoy = EnvoyExtension(envoyControl, config = RandomConfigFile.copy(configOverride = proxySettings))
63+
64+
private val allTags = listOf("ipsum", "lorem", "one")
65+
}
66+
67+
@Test
68+
fun `should add correct x-service-tag-preference header to upstream request`() {
69+
// given
70+
listOf("echo", "echo-disabled", "echo-one-tag", "echo-no-tag").forEach { service ->
71+
consul.server.operations.registerService(name = service, extension = echoService, tags = allTags)
72+
}
73+
listOf("echo", "echo-disabled", "echo-one-tag", "echo-no-tag").forEach { service ->
74+
waitForEndpointReady(service, echoService, envoy)
75+
}
76+
77+
// when
78+
val echoResponse = envoy.egressOperations.callService("echo").asHttpsEchoResponse()
79+
val echoDisabledResponse = envoy.egressOperations.callService("echo-disabled").asHttpsEchoResponse()
80+
val echoOneTagResponse = envoy.egressOperations.callService("echo-one-tag").asHttpsEchoResponse()
81+
val echoNoTagResponse = envoy.egressOperations.callService("echo-no-tag").asHttpsEchoResponse()
82+
83+
// then
84+
assertThat(echoResponse).isOk()
85+
assertThat(echoResponse.requestHeaders).containsEntry("x-service-tag-preference", "ipsum|lorem")
86+
87+
assertThat(echoDisabledResponse).isOk()
88+
assertThat(echoDisabledResponse.requestHeaders).doesNotContainKey("x-service-tag-preference")
89+
90+
assertThat(echoOneTagResponse).isOk()
91+
assertThat(echoOneTagResponse.requestHeaders).containsEntry("x-service-tag-preference", "one")
92+
93+
assertThat(echoNoTagResponse).isOk()
94+
assertThat(echoNoTagResponse.requestHeaders).doesNotContainKey("x-service-tag-preference")
95+
}
96+
97+
private fun waitForEndpointReady(
98+
serviceName: String,
99+
serviceInstance: ServiceExtension<*>,
100+
envoy: EnvoyExtension
101+
) {
102+
envoy.waitForClusterEndpointHealthy(cluster = serviceName, endpointIp = serviceInstance.container().ipAddress())
103+
}
104+
}

envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/ServiceTagsAndCanaryTest.kt renamed to envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/routing/ServiceTagsAndCanaryTest.kt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
1-
package pl.allegro.tech.servicemesh.envoycontrol
1+
package pl.allegro.tech.servicemesh.envoycontrol.routing
22

33
import org.junit.jupiter.api.extension.RegisterExtension
4-
import pl.allegro.tech.servicemesh.envoycontrol.config.envoycontrol.EnvoyControlExtension
54
import pl.allegro.tech.servicemesh.envoycontrol.config.consul.ConsulExtension
65
import pl.allegro.tech.servicemesh.envoycontrol.config.envoy.EnvoyExtension
6+
import pl.allegro.tech.servicemesh.envoycontrol.config.envoycontrol.EnvoyControlExtension
77

88
class ServiceTagsAndCanaryTest : ServiceTagsAndCanaryTestBase {
99
companion object {

0 commit comments

Comments
 (0)