Dev 20241212 ci2

.github/pull_request_template.md

@@ -0,0 +1,43 @@
+<!--
+Suggested PR template: Fill/delete/add sections as needed. Optionally delete any commented block.
+-->
+What
+----
+<!--
+Briefly describe **what** you have changed and **why**.
+Optionally include implementation strategy.
+-->
+
+Checklist
+------------------
+- [ ] Contains customer facing changes? Including API/behavior changes <!-- This can help identify if it has introduced any breaking changes -->
+- [ ] Did you add sufficient unit test and/or integration test coverage for this PR?
+  - If not, please explain why it is not required
+
+References
+----------
+JIRA: 
+<!--
+Copy&paste links: to Jira ticket, other PRs, issues, Slack conversations...
+For code bumps: link to PR, tag or GitHub `/compare/master...master`
+-->
+
+Test & Review
+------------
+<!--
+Has it been tested? how?
+Copy&paste any handy instructions, steps or requirements that can save time to the reviewer or any reader.
+-->
+
+Open questions / Follow-ups
+--------------------------
+<!--
+Optional: anything open to discussion for the reviewer, out of scope, or follow-ups.
+-->
+
+<!--
+Review stakeholders
+------------------
+<!--
+Optional: mention stakeholders or if special context that is required to review.
+-->

.github/workflows/build.yml

@@ -0,0 +1,98 @@
+name: 'confluent-kafka-dotnet build pipeline'
+
+env:
+  CONFIGURATION: Release
+  DOTNET_CLI_TELEMETRY_OPTOUT: 'true'
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  linux-build:
+    name: 'Linux x64'
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: '6.0.x'
+      - name: Build and test
+        run: |
+          dotnet restore
+          make build
+          make test
+
+  osx-build:
+    name: 'OSX x64'
+    runs-on: macos-13
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: '6.0.x'
+      - name: Set ulimit
+        run: ulimit -n 1024
+      - name: Build and test
+        run: |
+          dotnet restore
+          make build
+          make test
+
+  windows-build:
+    name: 'Windows x64'
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: '6.0.403'
+      - name: Build and test
+        run: |
+          dotnet restore
+          dotnet test -c $env:CONFIGURATION test/Confluent.Kafka.UnitTests/Confluent.Kafka.UnitTests.csproj
+          # dotnet test -c $env:CONFIGURATION test/Confluent.SchemaRegistry.UnitTests/Confluent.SchemaRegistry.UnitTests.csproj
+          # dotnet test -c $env:CONFIGURATION test/Confluent.SchemaRegistry.Serdes.UnitTests/Confluent.SchemaRegistry.Serdes.UnitTests.csproj
+
+  windows-artifacts:
+    name: 'Windows Artifacts'
+    needs: windows-build
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: '6.0.403'
+      - name: Install DocFX
+        run: dotnet tool update -g docfx
+      - name: Build and create packages
+        run: |
+          dotnet restore
+          dotnet build Confluent.Kafka.sln -c $env:CONFIGURATION
+
+          # Different packaging for tagged vs untagged builds
+          if ($env:GITHUB_REF -match '^refs/tags/') {
+            $suffix = ""
+            $vsuffix = ""
+          } else {
+            $suffix = "ci-$env:GITHUB_RUN_ID"
+            $vsuffix = "--version-suffix"
+          }
+
+          dotnet pack src/Confluent.Kafka/Confluent.Kafka.csproj --output artifacts -c $env:CONFIGURATION $vsuffix $suffix 
+          # dotnet pack src/Confluent.SchemaRegistry/Confluent.SchemaRegistry.csproj -c $env:CONFIGURATION $suffix --output artifacts
+          # dotnet pack src/Confluent.SchemaRegistry.Serdes.Avro/Confluent.SchemaRegistry.Serdes.Avro.csproj -c $env:CONFIGURATION $suffix --output artifacts
+          # dotnet pack src/Confluent.SchemaRegistry.Serdes.Protobuf/Confluent.SchemaRegistry.Serdes.Protobuf.csproj -c $env:CONFIGURATION $suffix --output artifacts
+          # dotnet pack src/Confluent.SchemaRegistry.Serdes.Json/Confluent.SchemaRegistry.Serdes.Json.csproj -c $env:CONFIGURATION $suffix --output artifacts
+
+          # docfx doc/docfx.json
+          # tar -czf artifacts/docs-$env:GITHUB_RUN_ID.zip doc/_site/*
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: build-artifacts
+          path: artifacts/

.github/workflows/integration.yml

@@ -0,0 +1,60 @@
+name: 'Integration Tests'
+
+env:
+  CONFIGURATION: Release
+  DOTNET_CLI_TELEMETRY_OPTOUT: 'true'
+  SEMAPHORE_SKIP_FLAKY_TESTS: 'true'
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  integration-tests:
+    name: 'Integration tests'
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: '6.0.x'
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_APIKEY }}
+
+      - name: Classic Protocol Tests
+        run: |
+          cd test/docker && docker-compose up -d && sleep 30 && cd ../..
+          dotnet restore
+          cd test/Confluent.Kafka.IntegrationTests && dotnet test -l "console;verbosity=normal" && cd ../..
+
+      - name: Consumer Protocol Tests
+        run: |
+          cd test/docker && docker-compose -f docker-compose-kraft.yaml up -d && cd ../..
+          sleep 300
+          export TEST_CONSUMER_GROUP_PROTOCOL=consumer
+          dotnet restore
+          cd test/Confluent.Kafka.IntegrationTests && dotnet test -l "console;verbosity=normal" && cd ../..
+
+  schema-registry-tests:
+    name: 'Schema registry and serdes integration tests'
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: '6.0.x'
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_APIKEY }}
+      - name: Run Tests
+        run: |
+          cd test/docker && docker-compose up -d && cd ../..
+          dotnet restore
+          cd test/Confluent.SchemaRegistry.Serdes.IntegrationTests && dotnet test -l "console;verbosity=normal" && cd ../..

3RD_PARTY.md

@@ -10,3 +10,4 @@ To add your project, open a pull request!
 - [Multi Schema Avro Deserializer](https://github.com/ycherkes/multi-schema-avro-desrializer) - Avro deserializer for reading messages serialized with multiple schemas.
 - [OpenSleigh.Transport.Kafka](https://github.com/mizrael/OpenSleigh/tree/develop/src/OpenSleigh.Transport.Kafka) - A Kafka Transport for OpenSleigh, a distributed saga management library.
 - [SlimMessageBus.Host.Kafka](https://github.com/zarusz/SlimMessageBus) - Apache Kafka transport for SlimMessageBus (lightweight message bus for .NET)
+- [Kafka Core](https://github.com/ffernandolima/confluent-kafka-core-dotnet) - Kafka Core empowers developers to build robust .NET applications on top of Confluent Kafka, focusing on simplicity, maintainability, and extensibility with intuitive abstractions and builders.

CHANGELOG.md

@@ -2,7 +2,17 @@
 
 ## Enhancements
 
+* References librdkafka.redist 2.6.1. Refer to the [librdkafka v2.6.1 release notes](https://github.com/confluentinc/librdkafka/releases/tag/v2.6.1) for more information.
+
+## Fixes
+
 * Fix to continue supporting .NET Framework 4.6.2+ in core client library (#2342).
+* Fix JSON Schema handling to not require use of `$id` (#2339).
+* Update Caching.Memory to 8.0.1 to address CVE (#23440.
+* Added Qualified and Custom reference name strategy approaches for protobuf references (#2345).
+* Fix validate of SSL CA certs in Schema Registry client (#2346).
+* Skip SSL certs validation when configured in Schema Registry client (#2347).
+* Allow proxy to be specified in Schema Registry client (#2348).
 
 
 # 2.6.0

src/Confluent.Kafka/Confluent.Kafka.csproj

@@ -12,16 +12,14 @@
     <PackageIconUrl>https://raw.githubusercontent.com/confluentinc/confluent-kafka-dotnet/master/confluent-logo.png</PackageIconUrl>
     <PackageReleaseNotes>https://github.com/confluentinc/confluent-kafka-dotnet/releases</PackageReleaseNotes>
     <PackageTags>Kafka;Confluent;librdkafka</PackageTags>
-    <PackageId>Confluent.Kafka</PackageId>
+    <PackageId>Confluent.Kafka.MK</PackageId>
     <PackageReadmeFile>README.md</PackageReadmeFile>
     <Title>Confluent.Kafka</Title>
-    <AssemblyName>Confluent.Kafka</AssemblyName>
+    <AssemblyName>Confluent.Kafka.MK</AssemblyName>
     <VersionPrefix>2.6.1</VersionPrefix>
     <TargetFrameworks>netstandard2.0;net462;net6.0;net8.0</TargetFrameworks>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <GenerateDocumentationFile>true</GenerateDocumentationFile>
-    <SignAssembly>true</SignAssembly>
-    <AssemblyOriginatorKeyFile>Confluent.Kafka.snk</AssemblyOriginatorKeyFile>
   </PropertyGroup>
 
   <ItemGroup>

src/Confluent.SchemaRegistry.Encryption.Aws/AwsKmsClient.cs

@@ -12,7 +12,7 @@ public class AwsKmsClient : IKmsClient
     {
         private AmazonKeyManagementServiceClient kmsClient;
         private string keyId;
-        
+
         public string KekId { get; }
 
         public AwsKmsClient(string kekId, AWSCredentials credentials)
@@ -36,7 +36,7 @@ public AwsKmsClient(string kekId, AWSCredentials credentials)
 
         public bool DoesSupport(string uri)
         {
-            return uri.StartsWith(AwsKmsDriver.Prefix); 
+            return KekId == uri;
         }
 
         public async Task<byte[]> Encrypt(byte[] plaintext)

src/Confluent.SchemaRegistry.Encryption.Azure/AzureKmsClient.cs

@@ -25,7 +25,7 @@ public AzureKmsClient(string kekId, TokenCredential tokenCredential)
 
         public bool DoesSupport(string uri)
         {
-            return uri.StartsWith(AzureKmsDriver.Prefix); 
+            return KekId == uri;
         }
 
         public async Task<byte[]> Encrypt(byte[] plaintext)

src/Confluent.SchemaRegistry.Encryption.Gcp/GcpKmsClient.cs

@@ -36,7 +36,7 @@ public GcpKmsClient(string kekId, GoogleCredential credential)
 
         public bool DoesSupport(string uri)
         {
-            return uri.StartsWith(GcpKmsDriver.Prefix);
+            return KekId == uri;
         }
 
         public async Task<byte[]> Encrypt(byte[] plaintext)

src/Confluent.SchemaRegistry.Encryption.HcVault/HcVaultKmsClient.cs

@@ -21,11 +21,6 @@ public class HcVaultKmsClient : IKmsClient
 
         public HcVaultKmsClient(string kekId, string ns, string tokenId)
         {
-            if (tokenId == null)
-            {
-                tokenId = Environment.GetEnvironmentVariable("VAULT_TOKEN");
-                ns = Environment.GetEnvironmentVariable("VAULT_NAMESPACE");
-            }
             KekId = kekId;
             Namespace = ns;
             TokenId = tokenId;
@@ -53,7 +48,7 @@ public HcVaultKmsClient(string kekId, string ns, string tokenId)
 
         public bool DoesSupport(string uri)
         {
-            return uri.StartsWith(HcVaultKmsDriver.Prefix);
+            return KekId == uri;
         }
 
         public async Task<byte[]> Encrypt(byte[] plaintext)

src/Confluent.SchemaRegistry.Encryption.HcVault/HcVaultKmsDriver.cs

@@ -23,6 +23,11 @@ public IKmsClient NewKmsClient(IDictionary<string, string> config, string keyUrl
         {
             config.TryGetValue(TokenId, out string tokenId);
             config.TryGetValue(Namespace, out string ns);
+            if (tokenId == null)
+            {
+                tokenId = Environment.GetEnvironmentVariable("VAULT_TOKEN");
+                ns = Environment.GetEnvironmentVariable("VAULT_NAMESPACE");
+            }
             return new HcVaultKmsClient(keyUrl, ns, tokenId);
         }
     }