Skip to content

Commit 1a16639

Browse files
authored
Merge branch 'master' into master
2 parents 16edbaa + d0f8753 commit 1a16639

File tree

231 files changed

+20680
-2473
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

231 files changed

+20680
-2473
lines changed

.github/CODEOWNERS

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
* @confluentinc/clients @confluentinc/data-governance

.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ obj/
66
*.dylib
77
*.csproj.user
88
*.xproj.user
9+
*.sln.*.user
10+
.idea
911
.vs
1012
.vscode
1113
todo.txt

.semaphore/semaphore.yml

Lines changed: 56 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ version: v1.0
22
name: 'confluent-kafka-dotnet build pipeline'
33
agent:
44
machine:
5-
type: s1-prod-ubuntu20-04-amd64-1
5+
type: s1-prod-ubuntu22-04-amd64-1
66
global_job_config:
77
prologue:
88
commands:
@@ -19,10 +19,15 @@ blocks:
1919
task:
2020
agent:
2121
machine:
22-
type: s1-prod-ubuntu20-04-amd64-2
22+
type: s1-prod-ubuntu22-04-amd64-2
2323
jobs:
2424
- name: 'Build and test'
2525
commands:
26+
# TODO: install .NET 8.0 SDK in this agent image
27+
- sudo apt remove dotnet-sdk* dotnet-host* dotnet* aspnetcore* netstandard* -y
28+
# TODO: remove incorrect focal repos from this agent
29+
- sudo rm /etc/apt/sources.list.d/*microsoft*
30+
- sudo apt update && sudo apt install -y dotnet-sdk-6.0 dotnet-sdk-8.0
2631
- dotnet restore
2732
- make build
2833
- make test
@@ -36,9 +41,15 @@ blocks:
3641
- name: 'Build and test'
3742
commands:
3843
- ulimit -n 1024
39-
- dotnet restore
40-
- make build
41-
- make test
44+
- curl -sSL https://dot.net/v1/dotnet-install.sh | bash /dev/stdin --channel 8.0 --install-dir ~/Applications/dotnet
45+
- export DOTNET_ROOT=~/Applications/dotnet
46+
- export PATH=~/Applications/dotnet:$PATH
47+
- export DOTNET_MULTILEVEL_LOOKUP=0
48+
- which dotnet
49+
- dotnet --version
50+
- dotnet --list-sdks
51+
- dotnet restore -p:TargetFramework=net8.0
52+
- make test-latest
4253
- name: 'Windows x64'
4354
dependencies: [ ]
4455
task:
@@ -49,12 +60,12 @@ blocks:
4960
- name: 'Build and test'
5061
commands:
5162
- wget https://dot.net/v1/dotnet-install.ps1 -OutFile dotnet-install.ps1
52-
- powershell -ExecutionPolicy ByPass -File dotnet-install.ps1 -Version 6.0.403 -InstallDir C:\dotnet
63+
- powershell -ExecutionPolicy ByPass -File dotnet-install.ps1 -Channel 8.0 -Quality GA -InstallDir C:\dotnet
5364
- $Env:Path += ";C:\dotnet"
5465
- dotnet restore
55-
- dotnet test -c ${CONFIGURATION} --no-build test/Confluent.Kafka.UnitTests/Confluent.Kafka.UnitTests.csproj
56-
- dotnet test -c ${CONFIGURATION} --no-build test/Confluent.SchemaRegistry.UnitTests/Confluent.SchemaRegistry.UnitTests.csproj
57-
- dotnet test -c ${CONFIGURATION} --no-build test/Confluent.SchemaRegistry.Serdes.UnitTests/Confluent.SchemaRegistry.Serdes.UnitTests.csproj
66+
- dotnet test -f net8.0 -c ${CONFIGURATION} --no-build test/Confluent.Kafka.UnitTests/Confluent.Kafka.UnitTests.csproj
67+
- dotnet test -f net8.0 -c ${CONFIGURATION} --no-build test/Confluent.SchemaRegistry.UnitTests/Confluent.SchemaRegistry.UnitTests.csproj
68+
- dotnet test -f net8.0 -c ${CONFIGURATION} --no-build test/Confluent.SchemaRegistry.Serdes.UnitTests/Confluent.SchemaRegistry.Serdes.UnitTests.csproj
5869
- name: 'Windows Artifacts on untagged commits'
5970
run:
6071
when: "tag !~ '.*'"
@@ -68,19 +79,22 @@ blocks:
6879
- name: 'Build and push artifacts'
6980
commands:
7081
- wget https://dot.net/v1/dotnet-install.ps1 -OutFile dotnet-install.ps1
71-
- powershell -ExecutionPolicy ByPass -File dotnet-install.ps1 -Version 6.0.403 -InstallDir C:\dotnet
82+
- powershell -ExecutionPolicy ByPass -File dotnet-install.ps1 -Channel 6.0 -Quality GA -InstallDir C:\dotnet
83+
- powershell -ExecutionPolicy ByPass -File dotnet-install.ps1 -Channel 8.0 -Quality GA -InstallDir C:\dotnet
7284
- $Env:Path += ";C:\dotnet"
73-
- dotnet tool update -g docfx
7485
- dotnet restore
7586
- dotnet build Confluent.Kafka.sln -c ${Env:CONFIGURATION}
7687
- dotnet pack src/Confluent.Kafka/Confluent.Kafka.csproj -c ${Env:CONFIGURATION} --version-suffix ci-${Env:SEMAPHORE_JOB_ID} --output artifacts
7788
- dotnet pack src/Confluent.SchemaRegistry/Confluent.SchemaRegistry.csproj -c ${Env:CONFIGURATION} --version-suffix ci-${Env:SEMAPHORE_JOB_ID} --output artifacts
89+
- dotnet pack src/Confluent.SchemaRegistry.Encryption/Confluent.SchemaRegistry.Encryption.csproj -c ${Env:CONFIGURATION} --version-suffix ci-${Env:SEMAPHORE_JOB_ID} --output artifacts
90+
- dotnet pack src/Confluent.SchemaRegistry.Encryption.Aws/Confluent.SchemaRegistry.Encryption.Aws.csproj -c ${Env:CONFIGURATION} --version-suffix ci-${Env:SEMAPHORE_JOB_ID} --output artifacts
91+
- dotnet pack src/Confluent.SchemaRegistry.Encryption.Azure/Confluent.SchemaRegistry.Encryption.Azure.csproj -c ${Env:CONFIGURATION} --version-suffix ci-${Env:SEMAPHORE_JOB_ID} --output artifacts
92+
- dotnet pack src/Confluent.SchemaRegistry.Encryption.Gcp/Confluent.SchemaRegistry.Encryption.Gcp.csproj -c ${Env:CONFIGURATION} --version-suffix ci-${Env:SEMAPHORE_JOB_ID} --output artifacts
93+
- dotnet pack src/Confluent.SchemaRegistry.Encryption.HcVault/Confluent.SchemaRegistry.Encryption.HcVault.csproj -c ${Env:CONFIGURATION} --version-suffix ci-${Env:SEMAPHORE_JOB_ID} --output artifacts
94+
- dotnet pack src/Confluent.SchemaRegistry.Rules/Confluent.SchemaRegistry.Rules.csproj -c ${Env:CONFIGURATION} --version-suffix ci-${Env:SEMAPHORE_JOB_ID} --output artifacts
7895
- dotnet pack src/Confluent.SchemaRegistry.Serdes.Avro/Confluent.SchemaRegistry.Serdes.Avro.csproj -c ${Env:CONFIGURATION} --version-suffix ci-${Env:SEMAPHORE_JOB_ID} --output artifacts
7996
- dotnet pack src/Confluent.SchemaRegistry.Serdes.Protobuf/Confluent.SchemaRegistry.Serdes.Protobuf.csproj -c ${Env:CONFIGURATION} --version-suffix ci-${Env:SEMAPHORE_JOB_ID} --output artifacts
8097
- dotnet pack src/Confluent.SchemaRegistry.Serdes.Json/Confluent.SchemaRegistry.Serdes.Json.csproj -c ${Env:CONFIGURATION} --version-suffix ci-${Env:SEMAPHORE_JOB_ID} --output artifacts
81-
- docfx doc/docfx.json
82-
- tar.exe -cvzf docs-${Env:SEMAPHORE_JOB_ID}.zip doc/_site/*
83-
- move docs-${Env:SEMAPHORE_JOB_ID}.zip artifacts
8498
- artifact push workflow artifacts
8599
- name: 'Windows Artifacts on tagged commits'
86100
run:
@@ -95,59 +109,60 @@ blocks:
95109
- name: 'Build and push artifacts'
96110
commands:
97111
- wget https://dot.net/v1/dotnet-install.ps1 -OutFile dotnet-install.ps1
98-
- powershell -ExecutionPolicy ByPass -File dotnet-install.ps1 -Version 6.0.403 -InstallDir C:\dotnet
112+
- powershell -ExecutionPolicy ByPass -File dotnet-install.ps1 -Channel 6.0 -Quality GA -InstallDir C:\dotnet
113+
- powershell -ExecutionPolicy ByPass -File dotnet-install.ps1 -Channel 8.0 -Quality GA -InstallDir C:\dotnet
99114
- $Env:Path += ";C:\dotnet"
100-
- dotnet tool update -g docfx
101115
- dotnet restore
102116
- dotnet build Confluent.Kafka.sln -c ${Env:CONFIGURATION}
103117
- dotnet pack src/Confluent.Kafka/Confluent.Kafka.csproj -c ${Env:CONFIGURATION} --output artifacts
104118
- dotnet pack src/Confluent.SchemaRegistry/Confluent.SchemaRegistry.csproj -c ${Env:CONFIGURATION} --output artifacts
119+
- dotnet pack src/Confluent.SchemaRegistry.Encryption/Confluent.SchemaRegistry.Encryption.csproj -c ${Env:CONFIGURATION} --output artifacts
120+
- dotnet pack src/Confluent.SchemaRegistry.Encryption.Aws/Confluent.SchemaRegistry.Encryption.Aws.csproj -c ${Env:CONFIGURATION} --output artifacts
121+
- dotnet pack src/Confluent.SchemaRegistry.Encryption.Azure/Confluent.SchemaRegistry.Encryption.Azure.csproj -c ${Env:CONFIGURATION} --output artifacts
122+
- dotnet pack src/Confluent.SchemaRegistry.Encryption.Gcp/Confluent.SchemaRegistry.Encryption.Gcp.csproj -c ${Env:CONFIGURATION} --output artifacts
123+
- dotnet pack src/Confluent.SchemaRegistry.Encryption.HcVault/Confluent.SchemaRegistry.Encryption.HcVault.csproj -c ${Env:CONFIGURATION} --output artifacts
124+
- dotnet pack src/Confluent.SchemaRegistry.Rules/Confluent.SchemaRegistry.Rules.csproj -c ${Env:CONFIGURATION} --output artifacts
105125
- dotnet pack src/Confluent.SchemaRegistry.Serdes.Avro/Confluent.SchemaRegistry.Serdes.Avro.csproj -c ${Env:CONFIGURATION} --output artifacts
106126
- dotnet pack src/Confluent.SchemaRegistry.Serdes.Protobuf/Confluent.SchemaRegistry.Serdes.Protobuf.csproj -c ${Env:CONFIGURATION} --output artifacts
107127
- dotnet pack src/Confluent.SchemaRegistry.Serdes.Json/Confluent.SchemaRegistry.Serdes.Json.csproj -c ${Env:CONFIGURATION} --output artifacts
108-
- docfx doc/docfx.json
109-
- tar.exe -cvzf docs-${Env:SEMAPHORE_JOB_ID}.zip doc/_site/*
110-
- move docs-${Env:SEMAPHORE_JOB_ID}.zip artifacts
111128
- artifact push workflow artifacts
112-
- name: 'Integration tests'
129+
- name: 'Integration tests and documentation'
113130
dependencies: [ ]
114131
task:
115132
agent:
116133
machine:
117-
type: s1-prod-ubuntu20-04-amd64-2
134+
type: s1-prod-ubuntu22-04-amd64-2
118135
prologue:
119136
commands:
120-
- docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY
137+
- '[[ -z $DOCKERHUB_APIKEY ]] || docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY'
138+
# TODO: install .NET 8.0 SDK in this agent image
139+
- sudo apt remove dotnet-sdk* dotnet-host* dotnet* aspnetcore* netstandard* -y
140+
# TODO: remove incorrect focal repos from this agent
141+
- sudo rm /etc/apt/sources.list.d/*microsoft*
142+
- sudo apt update && sudo apt install -y dotnet-sdk-8.0
121143
jobs:
144+
- name: 'Build documentation'
145+
commands:
146+
- dotnet tool update --tool-path $PWD docfx
147+
- ./docfx doc/docfx.json
122148
- name: 'Build and test with "classic" protocol'
123149
commands:
124150
- cd test/docker && docker-compose up -d && sleep 30 && cd ../..
125151
- export SEMAPHORE_SKIP_FLAKY_TESTS='true'
126-
- dotnet restore
127-
- cd test/Confluent.Kafka.IntegrationTests && dotnet test -l "console;verbosity=normal" && cd ../..
152+
- dotnet restore -p:TargetFramework=net8.0
153+
- cd test/Confluent.Kafka.IntegrationTests && dotnet test -f net8.0 -l "console;verbosity=normal" && cd ../..
128154
- name: 'Build and test with "consumer" protocol'
129155
commands:
130156
- cd test/docker && docker-compose -f docker-compose-kraft.yaml up -d && cd ../..
131157
- sleep 300
132158
- export SEMAPHORE_SKIP_FLAKY_TESTS='true'
133159
- export TEST_CONSUMER_GROUP_PROTOCOL=consumer
134-
- dotnet restore
135-
- cd test/Confluent.Kafka.IntegrationTests && dotnet test -l "console;verbosity=normal" && cd ../..
136-
137-
- name: 'Schema registry and serdes integration tests'
138-
dependencies: [ ]
139-
task:
140-
agent:
141-
machine:
142-
type: s1-prod-ubuntu20-04-amd64-2
143-
prologue:
144-
commands:
145-
- docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY
146-
jobs:
147-
- name: 'Build and test'
160+
- dotnet restore -p:TargetFramework=net8.0
161+
- cd test/Confluent.Kafka.IntegrationTests && dotnet test -f net8.0 -l "console;verbosity=normal" && cd ../..
162+
- name: 'Schema registry and serdes integration tests'
148163
commands:
149164
- cd test/docker && docker-compose up -d && cd ../..
150165
- export SEMAPHORE_SKIP_FLAKY_TESTS='true'
151-
- dotnet restore
152-
- cd test/Confluent.SchemaRegistry.Serdes.IntegrationTests && dotnet test -l "console;verbosity=normal" && cd ../..
153-
# - cd test/Confluent.SchemaRegistry.IntegrationTests && dotnet test -l "console;verbosity=normal" && cd ../..
166+
- dotnet restore -p:TargetFramework=net8.0
167+
- cd test/Confluent.SchemaRegistry.Serdes.IntegrationTests && dotnet test -f net8.0 -l "console;verbosity=normal" && cd ../..
168+
- cd test/Confluent.SchemaRegistry.IntegrationTests && dotnet test -f net8.0 -l "console;verbosity=normal" && cd ../..

3RD_PARTY.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,4 @@ To add your project, open a pull request!
99
- [Chr.Avro](https://github.com/ch-robinson/dotnet-avro) - A modern and flexible Avro implementation for .NET. Integrates seamlessly with Confluent.Kafka and Schema Registry.
1010
- [Multi Schema Avro Deserializer](https://github.com/ycherkes/multi-schema-avro-desrializer) - Avro deserializer for reading messages serialized with multiple schemas.
1111
- [OpenSleigh.Transport.Kafka](https://github.com/mizrael/OpenSleigh/tree/develop/src/OpenSleigh.Transport.Kafka) - A Kafka Transport for OpenSleigh, a distributed saga management library.
12+
- [SlimMessageBus.Host.Kafka](https://github.com/zarusz/SlimMessageBus) - Apache Kafka transport for SlimMessageBus (lightweight message bus for .NET)

CHANGELOG.md

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,90 @@
1+
# 2.6.1
2+
3+
## Enhancements
4+
5+
* References librdkafka.redist 2.6.1. Refer to the [librdkafka v2.6.1 release notes](https://github.com/confluentinc/librdkafka/releases/tag/v2.6.1) for more information.
6+
7+
## Fixes
8+
9+
* Fix to continue supporting .NET Framework 4.6.2+ in core client library (#2342).
10+
* Fix JSON Schema handling to not require use of `$id` (#2339).
11+
* Update Caching.Memory to 8.0.1 to address CVE (#23440.
12+
* Added Qualified and Custom reference name strategy approaches for protobuf references (#2345).
13+
* Fix validate of SSL CA certs in Schema Registry client (#2346).
14+
* Skip SSL certs validation when configured in Schema Registry client (#2347).
15+
* Allow proxy to be specified in Schema Registry client (#2348).
16+
17+
18+
# 2.6.0
19+
20+
## Enhancements
21+
22+
* References librdkafka.redist 2.6.0. Refer to the [librdkafka v2.6.0 release notes](https://github.com/confluentinc/librdkafka/releases/tag/v2.6.0) for more information.
23+
* [KIP-848 EA](https://cwiki.apache.org/confluence/display/KAFKA/KIP-848%3A+The+Next+Generation+of+the+Consumer+Rebalance+Protocol): Admin API for listing consumer groups now has an optional filter to return only groups of given types (#2323).
24+
* [KIP-460](https://cwiki.apache.org/confluence/display/KAFKA/KIP-460%3A+Admin+Leader+Election+RPC) Admin Leader Election RPC (#2320)
25+
* .NET 8 support with NJsonSchema 11 (#2314)
26+
27+
28+
# 2.5.3
29+
30+
## Enhancements
31+
32+
* References librdkafka.redist 2.5.3. Refer to the [librdkafka v2.5.3 release notes](https://github.com/confluentinc/librdkafka/releases/tag/v2.5.3) for more information.
33+
34+
## Fixes
35+
36+
* Properly handle messages with well-known types in Protobuf serializer
37+
* Use AES128_GCM in the Local KMS client, for consistency with Java/go
38+
* Include deleted schemas when getting schemas by subject and version
39+
* Handle signed ints when transforming Protobuf payloads
40+
* Allow null SchemaRegistryClient in AsyncSerde constructor
41+
42+
# 2.5.2
43+
44+
> [!WARNING]
45+
Versions 2.5.0, 2.5.1 and 2.5.2 have a regression in which an assert is triggered during **PushTelemetry** call. This happens when no metric is matched on the client side among those requested by broker subscription.
46+
>
47+
> You won't face any problem if:
48+
> * Broker doesn't support [KIP-714](https://cwiki.apache.org/confluence/display/KAFKA/KIP-714%3A+Client+metrics+and+observability).
49+
> * [KIP-714](https://cwiki.apache.org/confluence/display/KAFKA/KIP-714%3A+Client+metrics+and+observability) feature is disabled on the broker side.
50+
> * [KIP-714](https://cwiki.apache.org/confluence/display/KAFKA/KIP-714%3A+Client+metrics+and+observability) feature is disabled on the client side. This is enabled by default. Set configuration `enable.metrics.push` to `false`.
51+
> * If [KIP-714](https://cwiki.apache.org/confluence/display/KAFKA/KIP-714%3A+Client+metrics+and+observability) is enabled on the broker side and there is no subscription configured there.
52+
> * If [KIP-714](https://cwiki.apache.org/confluence/display/KAFKA/KIP-714%3A+Client+metrics+and+observability) is enabled on the broker side with subscriptions that match the [KIP-714](https://cwiki.apache.org/confluence/display/KAFKA/KIP-714%3A+Client+metrics+and+observability) metrics defined on the client.
53+
>
54+
> Having said this, we strongly recommend using `v2.5.3` and above to not face this regression at all.
55+
56+
## Fixes
57+
58+
- Fix CSFLE (client-side field-level encryption) to use the Google Tink format for DEKs for interoperability with clients in other languages (Java, go, etc.).
59+
- Improve error when specifying an invalid KMS type for CSFLE
60+
- Enhance CSFLE examples with KMS configuration settings
61+
62+
63+
# 2.5.1
64+
65+
## Fixes
66+
67+
- Fix CSFLE (client-side field-level encryption) when using Azure Key Vault by specifying RsaOaep256 (instead of RsaOaep) for interoperability with clients in other languages (Java, go, etc.).
68+
- Fix AvroSerializer configuration to allow using schema normalization.
69+
- Upgrade Azure Identity library to 1.11.4 to address a vulnerability in previous versions.
70+
71+
72+
# 2.5.0
73+
74+
## Enhancements
75+
76+
- References librdkafka.redist 2.5.0. Refer to the [librdkafka v2.5.0 release notes](https://github.com/confluentinc/librdkafka/releases/tag/v2.5.0) for more information.
77+
- Add support for metadata and ruleSet in the schema registry client, which together support data
78+
contracts.
79+
- Add support for CSFLE (client-side field-level encryption) for AWS, Azure, GCP, and HashiCorp
80+
Vault. See the encryption examples in the examples directory.
81+
- Add support for CEL, CEL_FIELD, and JSONata rules.
82+
83+
## Fixes
84+
85+
- Switch license expression and other repo information. (#2192, @thompson-tomo)
86+
87+
188
# 2.4.0
289

390
## Enhancements

0 commit comments

Comments
 (0)