Skip to content

Th2 3673 common need to change the filter #201

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
fiodar-r wants to merge 18 commits into
base: master
Choose a base branch
from
Open

Th2 3673 common need to change the filter #201

Show file tree
Hide file tree
Changes from 13 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
35f6fad
[TH2-3673] Fixed filters logic
May 13, 2022
952232b
[TH2-3673] Added tests for new filtering logic
May 13, 2022
997dbbb
[TH2-3673] Added tests for new filtering logic
May 13, 2022
b3a2a18
[TH2-3673] Minor optimization for verifying against multiple filters
May 13, 2022
abd9642
[TH2-3673] Tests for multiple same values
May 14, 2022
ae7e315
[TH2-3673] minor test code style improvements
May 17, 2022
a517c7c
Merge branch 'master' into TH2-3673-common-need-to-change-the-filter
May 17, 2022
76b908c
additional tests
May 18, 2022
e3a5c0b
changed logic for filtering message batches
May 18, 2022
c88af55
tests for filtering message batches
May 18, 2022
b49399e
added properties field in filter; added related tests
May 19, 2022
11a848b
tests refactor
May 19, 2022
259e0de
added additional tests for properties filtering
May 19, 2022
b6b2f2d
tests refactoring with util methods
May 19, 2022
8f44ca4
added protocol field to metadata, added according tests
May 20, 2022
04a6ff8
[TH2-3673] run ci
May 23, 2022
4d26cee
[TH2-3673] added aliases for message filter configuration
Jul 25, 2022
6e105ba
Merge branch 'master' into TH2-3673-common-need-to-change-the-filter
Jul 25, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 19 additions & 7 deletions ...main/java/com/exactpro/th2/common/schema/filter/strategy/impl/AbstractFilterStrategy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import com.google.protobuf.Message;
import org.apache.commons.collections4.MultiMapUtils;
import org.apache.commons.collections4.MultiValuedMap;
import org.jetbrains.annotations.NotNull;

import java.util.Collection;
import java.util.List;
Expand All @@ -30,29 +31,40 @@ public abstract class AbstractFilterStrategy<T extends Message> implements Filte

@Override
public boolean verify(T message, RouterFilter routerFilter) {

MultiValuedMap<String, FieldFilterConfiguration> msgFieldFilters = MultiMapUtils.newListValuedHashMap();
msgFieldFilters.putAll(routerFilter.getMessage());
msgFieldFilters.putAll(routerFilter.getMetadata());

MultiValuedMap<String, FieldFilterConfiguration> msgFieldFilters = extractConfigMapFromRouterFilter(routerFilter);
return checkValues(getFields(message), msgFieldFilters);
}

@Override
public boolean verify(T message, List<? extends RouterFilter> routerFilters) {
Map<String, String> fields = getFields(message);
for (var fieldsFilter : routerFilters) {
if (verify(message, fieldsFilter)) {
if (verify(fields, fieldsFilter)) {
return true;
}
}

return false;
}

private boolean verify(Map<String, String> fields, RouterFilter routerFilter) {
MultiValuedMap<String, FieldFilterConfiguration> msgFieldFilters = extractConfigMapFromRouterFilter(routerFilter);
return checkValues(fields, msgFieldFilters);
}

@NotNull
private MultiValuedMap<String, FieldFilterConfiguration> extractConfigMapFromRouterFilter(RouterFilter routerFilter) {
MultiValuedMap<String, FieldFilterConfiguration> msgFieldFilters = MultiMapUtils.newListValuedHashMap();
msgFieldFilters.putAll(routerFilter.getMessage());
msgFieldFilters.putAll(routerFilter.getMetadata());
msgFieldFilters.putAll(routerFilter.getProperties());
return msgFieldFilters;
}

protected abstract Map<String, String> getFields(T message);

private boolean checkValues(Map<String, String> messageFields, MultiValuedMap<String, FieldFilterConfiguration> fieldFilters) {
return fieldFilters.isEmpty() || fieldFilters.keys().stream().anyMatch(fieldName -> {
return fieldFilters.isEmpty() || fieldFilters.keys().stream().allMatch(fieldName -> {
String messageValue = messageFields.get(fieldName);
Collection<FieldFilterConfiguration> filters = fieldFilters.get(fieldName);
return !filters.isEmpty() && filters.stream().allMatch(filter -> FieldValueChecker.checkFieldValue(filter, messageValue));
Expand Down
1 change: 1 addition & 0 deletions ...ava/com/exactpro/th2/common/schema/filter/strategy/impl/AbstractTh2MsgFilterStrategy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ public Map<String, String> getFields(com.google.protobuf.Message message) {
DIRECTION_KEY, messageID.getDirection().name()
);

messageFields.putAll(th2Msg.getMetadata().getPropertiesMap());
messageFields.putAll(metadataMsgFields);

return messageFields;
Expand Down
4 changes: 3 additions & 1 deletion ...in/kotlin/com/exactpro/th2/common/schema/filter/strategy/impl/AnyMessageFilterStrategy.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,19 +25,21 @@ class AnyMessageFilterStrategy : AbstractFilterStrategy<Message>() {
override fun getFields(message: Message): MutableMap<String, String> {
check(message is AnyMessage) { "Message is not an ${AnyMessage::class.qualifiedName}: ${message.toJson()}" }

val result = HashMap<String, String>();
val result = HashMap<String, String>()

when {
message.hasMessage() -> {
result.putAll(message.message.fieldsMap.mapValues { it.value.simpleValue })

val metadata = message.message.metadata
result.putAll(message.message.metadata.propertiesMap)
result[AbstractTh2MsgFilterStrategy.SESSION_ALIAS_KEY] = metadata.id.connectionId.sessionAlias
result[AbstractTh2MsgFilterStrategy.MESSAGE_TYPE_KEY] = metadata.messageType
result[AbstractTh2MsgFilterStrategy.DIRECTION_KEY] = metadata.id.direction.name
}
message.hasRawMessage() -> {
val metadata = message.rawMessage.metadata
result.putAll(message.rawMessage.metadata.propertiesMap)
result[AbstractTh2MsgFilterStrategy.SESSION_ALIAS_KEY] = metadata.id.connectionId.sessionAlias
result[AbstractTh2MsgFilterStrategy.DIRECTION_KEY] = metadata.id.direction.name
}
Expand Down
6 changes: 5 additions & 1 deletion src/main/kotlin/com/exactpro/th2/common/schema/grpc/configuration/GrpcRawFilterStrategy.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@ data class GrpcRawFilterStrategy(var filters: List<GrpcRouterFilterConfiguration
data class GrpcRouterFilterConfiguration(
@JsonProperty(required = true) var endpoint: String,
@JsonDeserialize(using = MultiMapFiltersDeserializer::class) override var metadata: MultiValuedMap<String, FieldFilterConfiguration> = emptyMultiMap(),
@JsonDeserialize(using = MultiMapFiltersDeserializer::class) override var message: MultiValuedMap<String, FieldFilterConfiguration> = emptyMultiMap()
@JsonDeserialize(using = MultiMapFiltersDeserializer::class) override var message: MultiValuedMap<String, FieldFilterConfiguration> = emptyMultiMap(),
@JsonDeserialize(using = MultiMapFiltersDeserializer::class) override var properties: MultiValuedMap<String, FieldFilterConfiguration> = emptyMultiMap()
) : RouterFilter, Configuration() {

@JsonGetter("metadata")
Expand All @@ -39,4 +40,7 @@ data class GrpcRouterFilterConfiguration(
@JsonGetter("message")
fun getJsonMessage(): Collection<FieldFilterConfiguration> = message.values()

@JsonGetter("properties")
fun getJsonProperties(): Collection<FieldFilterConfiguration> = properties.values()

}
14 changes: 12 additions & 2 deletions ...kotlin/com/exactpro/th2/common/schema/message/configuration/MessageRouterConfiguration.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,14 @@ data class QueueConfiguration(

data class MqRouterFilterConfiguration(
@JsonDeserialize(using = MultiMapFiltersDeserializer::class) override var metadata: MultiValuedMap<String, FieldFilterConfiguration> = emptyMultiMap(),
@JsonDeserialize(using = MultiMapFiltersDeserializer::class) override var message: MultiValuedMap<String, FieldFilterConfiguration> = emptyMultiMap()
@JsonDeserialize(using = MultiMapFiltersDeserializer::class) override var message: MultiValuedMap<String, FieldFilterConfiguration> = emptyMultiMap(),
@JsonDeserialize(using = MultiMapFiltersDeserializer::class) override var properties: MultiValuedMap<String, FieldFilterConfiguration> = emptyMultiMap()
) : Configuration(), RouterFilter {

constructor(
metadata: Collection<FieldFilterConfiguration> = emptyList(),
message: Collection<FieldFilterConfiguration> = emptyList()
message: Collection<FieldFilterConfiguration> = emptyList(),
properties: Collection<FieldFilterConfiguration> = emptyList()
) : this(
MultiMapUtils.newListValuedHashMap<String, FieldFilterConfiguration>().also { metadataMultiMap ->
metadata.forEach {
Expand All @@ -63,6 +65,11 @@ data class MqRouterFilterConfiguration(
message.forEach {
messageMultiMap.put(it.fieldName, it)
}
},
MultiMapUtils.newListValuedHashMap<String, FieldFilterConfiguration>().also { propertiesMultiMap ->
properties.forEach {
propertiesMultiMap.put(it.fieldName, it)
}
}
)

Expand All @@ -71,6 +78,9 @@ data class MqRouterFilterConfiguration(

@JsonGetter("message")
fun getJsonMessage(): Collection<FieldFilterConfiguration> = message.values()

@JsonGetter("properties")
fun getJsonProperties(): Collection<FieldFilterConfiguration> = properties.values()
}

data class FieldFilterConfigurationOld(
Expand Down
1 change: 1 addition & 0 deletions src/main/kotlin/com/exactpro/th2/common/schema/message/configuration/RouterFilter.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,4 +20,5 @@ import org.apache.commons.collections4.MultiValuedMap
interface RouterFilter {
val metadata: MultiValuedMap<String, FieldFilterConfiguration>
val message: MultiValuedMap<String, FieldFilterConfiguration>
val properties: MultiValuedMap<String, FieldFilterConfiguration>
}
2 changes: 1 addition & 1 deletion ...m/exactpro/th2/common/schema/message/impl/rabbitmq/group/RabbitMessageGroupBatchRouter.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ class RabbitMessageGroupBatchRouter : AbstractRabbitRouter<MessageGroupBatch>()

val builder = MessageGroupBatch.newBuilder()
message.groupsList.forEach { group ->
if (group.messagesList.all { filterMessage(it, pinConfiguration.filters) }) {
if (group.messagesList.any { filterMessage(it, pinConfiguration.filters) }) {
builder.addGroups(group)
} else {
incrementDroppedMetrics(
Expand Down
13 changes: 12 additions & 1 deletion src/test/kotlin/com/exactpro/th2/common/schema/TestJsonConfiguration.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,14 @@ class TestJsonConfiguration {
FieldFilterOperation.EQUAL
)
),
listOf(FieldFilterConfiguration("test_field", "test_value", FieldFilterOperation.EQUAL))
listOf(FieldFilterConfiguration("test_field", "test_value", FieldFilterOperation.EQUAL)),
listOf(
FieldFilterConfiguration(
"method",
"POST",
FieldFilterOperation.EQUAL
)
),
),
MqRouterFilterConfiguration(
listOf(
Expand All @@ -196,6 +203,10 @@ class TestJsonConfiguration {
listOf(
FieldFilterConfiguration("test_field", "test_value0", FieldFilterOperation.EQUAL),
FieldFilterConfiguration("test_field", "test_value1", FieldFilterOperation.EQUAL)
),
listOf(
FieldFilterConfiguration("method1", "POST", FieldFilterOperation.EQUAL),
FieldFilterConfiguration("method2", "GET", FieldFilterOperation.NOT_EQUAL)
)
)
)
Expand Down
Loading