Skip to content

Commit 749c4d3

Browse files
mccullyamccullya
committed
Merge branch 'quotas' into schema-linking
2 parents 487e433 + 4d081c6 commit 749c4d3

File tree

10 files changed

+173
-0
lines changed

10 files changed

+173
-0
lines changed

‎stable/quotas/confluent/kafka.yaml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
apiVersion: platform.confluent.io/v1beta1
2+
kind: Kafka
3+
metadata:
4+
name: kafka
5+
spec:
6+
configOverrides:
7+
server:
8+
- confluent.schema.registry.url=https://schemaregistry:8081
9+
- listener.name.internal.plain.sasl.server.callback.handler.class=io.confluent.security.auth.provider.ldap.LdapAuthenticateCallbackHandler
10+
- listener.name.external.plain.sasl.server.callback.handler.class=io.confluent.security.auth.provider.ldap.LdapAuthenticateCallbackHandler
11+
- listener.name.replication.plain.sasl.server.callback.handler.class=io.confluent.security.auth.provider.ldap.LdapAuthenticateCallbackHandler
12+
- authorizer.class.name=io.confluent.kafka.security.authorizer.ConfluentServerAuthorizer
13+
- confluent.authorizer.access.rule.providers=ZK_ACL,CONFLUENT
14+
- quota.consumer.default = 1024
15+
- quota.producer.default = 1024
16+
- quota.window.num = 11
17+
- quota.window.size.seconds = 1
18+
19+

‎stable/quotas/docker/Dockerfile‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
FROM confluentinc/cp-kafka:latest
2+
USER root
3+
RUN yum install -y jq vim vi
4+
RUN curl -L -o /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/`curl -s https://api.github.com/repos/mikefarah/yq/releases/latest | jq .tag_name | sed 's/"//g'`/yq_linux_amd64
5+
RUN chmod +x /usr/local/bin/yq
6+
USER appuser

‎stable/quotas/docker/build-inside.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/bash
2+
eval $(minikube docker-env)
3+
docker build -t quota-manager .

‎stable/quotas/kustomization.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
apiVersion: kustomize.config.k8s.io/v1beta1
2+
kind: Kustomization
3+
namespace: sandbox
4+
resources:
5+
- namespace.yaml
6+
- ../../base/cfk-components/confluent-rbac
7+
- quota-manager
8+
patchesStrategicMerge:
9+
- confluent/kafka.yaml

‎stable/quotas/namespace.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
apiVersion: v1
2+
kind: Namespace
3+
metadata:
4+
name: sandbox

‎stable/quotas/quota-manager/default/apply-quotas-script.sh‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
#!/bin/bash
2+
quota_definition="/tmp/quota-definition/quotas.yaml"
3+
for i in $(yq '.platform.quotas | keys' < $quota_definition); do
4+
principal="${i:0}"
5+
if [[ "$principal" != "-" ]]; then
6+
producer_byte_rate=$(yq ".platform.quotas.$principal.producer_byte_rate" < $quota_definition)
7+
consumer_byte_rate=$(yq ".platform.quotas.$principal.consumer_byte_rate" < $quota_definition)
8+
request_percentage=$(yq ".platform.quotas.$principal.request_percentage" < $quota_definition)
9+
config_string=""
10+
if [[ "$producer_byte_rate" != "null" ]]; then
11+
config_string="$config_string producer_byte_rate=$producer_byte_rate,"
12+
fi
13+
if [[ "$consumer_byte_rate" != "null" ]]; then
14+
config_string="$config_string consumer_byte_rate=$consumer_byte_rate,"
15+
fi
16+
if [[ "$request_percentage" != "null" ]]; then
17+
config_string="$config_string request_percentage=$request_percentage,"
18+
fi
19+
# Removes white space and the last trailing comma
20+
config=$(echo $config_string | tr -d " \t\n\r" | rev | cut -c 2- | rev)
21+
echo $config
22+
set -x
23+
if [[ "$principal" == "global" ]]; then
24+
kafka-configs \
25+
--bootstrap-server kafka:9071 \
26+
--alter \
27+
--entity-type users \
28+
--entity-default \
29+
--command-config /tmp/config-properties/kafka.properties \
30+
--add-config "$config"
31+
else
32+
kafka-configs \
33+
--bootstrap-server kafka:9071 \
34+
--alter \
35+
--entity-type users \
36+
--entity-name $principal \
37+
--command-config /tmp/config-properties/kafka.properties \
38+
--add-config "$config"
39+
fi
40+
set +x
41+
fi
42+
done
43+

‎stable/quotas/quota-manager/default/kafka.properties‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
bootstrap.servers=kafka:9071
2+
sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username=kafka password=kafka-secret;
3+
sasl.mechanism=PLAIN
4+
security.protocol=SASL_SSL
5+
ssl.truststore.location=/mnt/sslcerts/truststore.jks
6+
ssl.truststore.password=mystorepassword

‎stable/quotas/quota-manager/default/quotas.yaml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
---
2+
platform:
3+
quotas:
4+
global:
5+
producer_byte_rate: 1024
6+
consumer_byte_rate: 1024
7+
request_percentage: 50.0
8+
app-0:
9+
producer_byte_rate: 4096
10+
consumer_byte_rate: 2048
11+
request_percentage: 50.0
12+
app-2:
13+
producer_byte_rate: 2048
14+
consumer_byte_rate: 2048
15+
app-3:
16+
request_percentage: 80.0

‎stable/quotas/quota-manager/kustomization.yaml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
namespace: sandbox
2+
resources:
3+
- quota-manager.yaml
4+
configMapGenerator:
5+
- name: oso-quota-example
6+
files:
7+
- ./default/quotas.yaml
8+
- name: config-properties
9+
files:
10+
- ./default/kafka.properties
11+
- name: apply-quota-script
12+
files:
13+
- ./default/apply-quotas-script.sh
14+
generatorOptions:
15+
disableNameSuffixHash: true

‎stable/quotas/quota-manager/quota-manager.yaml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
---
2+
apiVersion: batch/v1
3+
kind: Job
4+
metadata:
5+
name: quota-manager
6+
namespace: sandbox
7+
spec:
8+
9+
template:
10+
metadata:
11+
labels:
12+
app: quota-manager
13+
spec:
14+
restartPolicy: OnFailure
15+
containers:
16+
- name: quota-manager
17+
image: quota-manager:latest
18+
imagePullPolicy: Never
19+
command:
20+
- /tmp/quotas/apply-quotas-script.sh
21+
# - "sleep"
22+
# - "10000000"
23+
24+
resources:
25+
requests:
26+
memory: 512Mi # 768Mi
27+
cpu: 500m # 1000m
28+
volumeMounts:
29+
- mountPath: /tmp/quotas
30+
name: apply-quota-script
31+
- mountPath: /tmp/quota-definition
32+
name: oso-quota-example
33+
- mountPath: /tmp/config-properties
34+
name: config-properties
35+
- mountPath: /mnt/sslcerts
36+
name: sslcerts
37+
volumes:
38+
- name: config-properties
39+
configMap:
40+
defaultMode: 0777
41+
name: config-properties
42+
- name: oso-quota-example
43+
configMap:
44+
defaultMode: 0777
45+
name: oso-quota-example
46+
- name: apply-quota-script
47+
configMap:
48+
defaultMode: 0777
49+
name: apply-quota-script
50+
- name: sslcerts
51+
secret:
52+
secretName: kafka-generated-jks

0 commit comments

Comments
 (0)