adjust Glue Schema Registry sample (logs, run script)

Author: alexrashed

glue-msk-schema-registry/consumer-2/src/main/resources/log4j.properties

@@ -1,5 +1,5 @@
-log4j.rootLogger=INFO, CONSOLE
+log4j.rootLogger=DEBUG, CONSOLE
 
 log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
 log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=[CONSUMER-2][%-5p] %m%n
+log4j.appender.CONSOLE.layout.ConversionPattern=[%c{1}][%-5p] %m%n

glue-msk-schema-registry/consumer/src/main/resources/log4j.properties

@@ -1,5 +1,5 @@
-log4j.rootLogger=INFO, CONSOLE
+log4j.rootLogger=DEBUG, CONSOLE
 
 log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
 log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=[CONSUMER-1][%-5p] %m%n
+log4j.appender.CONSOLE.layout.ConversionPattern=[%c{1}][%-5p] %m%n

glue-msk-schema-registry/producer-2/src/main/resources/log4j.properties

@@ -1,5 +1,5 @@
-log4j.rootLogger=INFO, CONSOLE
+log4j.rootLogger=DEBUG, CONSOLE
 
 log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
 log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=[PRODUCER-2][%-5p] %m%n
+log4j.appender.CONSOLE.layout.ConversionPattern=[%c{1}][%-5p] %m%n

glue-msk-schema-registry/producer-3/src/main/resources/log4j.properties

@@ -1,5 +1,5 @@
-log4j.rootLogger=INFO, CONSOLE
+log4j.rootLogger=DEBUG, CONSOLE
 
 log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
 log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=[PRODUCER-3][%-5p] %m%n
+log4j.appender.CONSOLE.layout.ConversionPattern=[%c{1}][%-5p] %m%n

glue-msk-schema-registry/producer/src/main/resources/log4j.properties

@@ -1,5 +1,5 @@
-log4j.rootLogger=INFO, CONSOLE
+log4j.rootLogger=DEBUG, CONSOLE
 
 log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
 log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=[PRODUCER-1][%-5p] %m%n
+log4j.appender.CONSOLE.layout.ConversionPattern=[%c{1}][%-5p] %m%n

glue-msk-schema-registry/run.sh

@@ -5,7 +5,7 @@ function finish {
   echo ""
   # Delete the cluster (if available)
   echo "(Cleanup) Deleting Kafka cluster."
-  awslocal kafka delete-cluster --cluster-arn  "arn:aws:kafka:us-east-1:000000000000:cluster/unicorn-ride-cluster" 2> /dev/null || true
+  awslocal kafka delete-cluster --cluster-arn  $cluster_arn 2> /dev/null || true
   # Delete the schema registry
   echo "(Cleanup) Deleting registry."
   awslocal glue delete-registry --registry-id RegistryName=unicorn-ride-request-registry 2> /dev/null || true
@@ -33,40 +33,40 @@ else
 fi
 
 step "Start with creating a Kafka cluster..."
-(set -x; awslocal kafka create-cluster \
+cluster_arn=$(set -x; awslocal kafka create-cluster \
   --cluster-name "unicorn-ride-cluster" \
   --kafka-version "2.2.1" \
   --number-of-broker-nodes 1 \
-  --broker-node-group-info "{\"ClientSubnets\": [], \"InstanceType\":\"kafka.m5.xlarge\"}")
+  --broker-node-group-info "{\"ClientSubnets\": [], \"InstanceType\":\"kafka.m5.xlarge\"}" | jq -r .ClusterArn)
 
-state=$(set -x; awslocal kafka describe-cluster --cluster-arn "arn:aws:kafka:us-east-1:000000000000:cluster/unicorn-ride-cluster" | jq -r .ClusterInfo.State)
+state=$(set -x; awslocal kafka describe-cluster --cluster-arn $cluster_arn | jq -r .ClusterInfo.State)
 while [ "$state" != ACTIVE ]; do
   echo "Waiting for Kafka cluster to become ACTIVE (current status: $state)..."
   sleep 4
-  state=$(awslocal kafka describe-cluster --cluster-arn "arn:aws:kafka:us-east-1:000000000000:cluster/unicorn-ride-cluster" | jq -r .ClusterInfo.State)
+  state=$(awslocal kafka describe-cluster --cluster-arn $cluster_arn | jq -r .ClusterInfo.State)
 done
 
-bootstrap_broker=$(set -x; awslocal kafka get-bootstrap-brokers --cluster-arn  "arn:aws:kafka:us-east-1:000000000000:cluster/unicorn-ride-cluster" | jq -r .BootstrapBrokerString)
+bootstrap_broker=$(set -x; awslocal kafka get-bootstrap-brokers --cluster-arn  $cluster_arn | jq -r .BootstrapBrokerString)
 echo "Kafka Bootstrap Broker: $bootstrap_broker"
 
 step "The Kafka cluster is ready. Let's create a Glue Schema Registry..."
 (set -x; awslocal glue create-registry --registry-name unicorn-ride-request-registry)
 
 step "Create the AVRO schema in the new registry (with compatibility BACKWARD)..."
-(set -x; awslocal glue create-schema \
+schema_arn=$(set -x; awslocal glue create-schema \
   --registry-id RegistryName="unicorn-ride-request-registry" \
   --schema-name unicorn-ride-request-schema-avro \
   --compatibility BACKWARD \
   --data-format AVRO \
-  --schema-definition "file://producer/src/main/resources/avro/unicorn_ride_request_v1.avsc")
+  --schema-definition "file://producer/src/main/resources/avro/unicorn_ride_request_v1.avsc" | jq -r .SchemaArn)
 
 step "Check if the schema has been created successfully..."
 (set -x; awslocal glue get-schema \
-  --schema-id SchemaArn="arn:aws:glue:us-east-1:000000000000:schema/unicorn-ride-request-registry/unicorn-ride-request-schema-avro")
+  --schema-id SchemaArn=$schema_arn)
 
 step "Check if the schema version has been created correctly..."
 (set -x; awslocal glue get-schema-version \
-  --schema-id SchemaArn="arn:aws:glue:us-east-1:000000000000:schema/unicorn-ride-request-registry/unicorn-ride-request-schema-avro" \
+  --schema-id SchemaArn=$schema_arn \
   --schema-version-number LatestVersion=True)
 
 step "Prepare the Java clients (clean modules, generate AVRO model classes, compile the modules)..."
@@ -83,22 +83,22 @@ step "Run a new producer (automatically registers a new schema which removes a r
 
 step "Get a diff between the initial version and the version registered by the new producer..."
 (set -x; awslocal glue get-schema-versions-diff \
-  --schema-id SchemaArn="arn:aws:glue:us-east-1:000000000000:schema/unicorn-ride-request-registry/unicorn-ride-request-schema-avro" \
+  --schema-id SchemaArn=$schema_arn \
   --schema-diff-type SYNTAX_DIFF \
   --first-schema-version-number VersionNumber=1 \
   --second-schema-version-number LatestVersion=True | jq -r)
 
-step "Expected failure: Execute an incompatible (outdated) consumer..."
-(set -x; mvn -pl consumer exec:java -Dexec.args="--bootstrap-servers $bootstrap_broker")
-
 step "Expected failure: Execute a producer which tries to register an incompatible schema..."
 (set -x; mvn -pl producer-3 exec:java -Dexec.args="--bootstrap-servers $bootstrap_broker")
 
 step "Check that the newly registered schema is in state 'FAILED'..."
 awslocal glue get-schema-version \
-  --schema-id SchemaArn="arn:aws:glue:us-east-1:000000000000:schema/unicorn-ride-request-registry/unicorn-ride-request-schema-avro" \
+  --schema-id SchemaArn=$schema_arn \
   --schema-version-number VersionNumber=3
 
+step "Expected failure: Execute an incompatible (outdated) consumer..."
+(set -x; mvn -pl consumer exec:java -Dexec.args="--bootstrap-servers $bootstrap_broker")
+
 step "Execute a compatible (updated) consumer..."
 (set -x; mvn -pl consumer-2 exec:java -Dexec.args="--bootstrap-servers $bootstrap_broker")