Fix splunk cisco asa source

.gitignore

@@ -1,3 +1,5 @@
 
 .DS_Store
 /.idea/
+pcaps/edited_syslog.pcap
+pcaps/zeek*

.gitpod.yml

@@ -16,10 +16,17 @@ tasks:
     init: python3 scripts/get_pcap.py
 
   - name: download docker images
-    init: docker-compose pull 
+    init: docker-compose build && docker-compose pull
 
   - name: start services
-    command: docker-compose up -d
+    command: |
+      if [[ -z "$SUBMIT_CONNECTORS" ]]; then
+        docker-compose up -d
+        echo "Started without connectors"
+      else
+        docker-compose -f docker-compose.yml -f kafka-connect/submit-connectors.yml up -d
+        echo "Started with connectors"
+      fi
 
 ports:
 

README.md

@@ -11,9 +11,11 @@ The examples in this repository give you hands-on experience optimizing Security
 This demo runs best using Gitpod. Gitpod uses your existing git service account (GitHub, Gitlab, or BitBucket) for authentication. See the [gitpod tips](./instructions/gitpod-tips.md) to get acquainted with gitpod.
 
 **Launch a workspace** to get hands-on with the labs:
-- (if demo not in confluentinc yet) https://gitpod.io/#https://github.com/chuck-confluent/demo-siem-optimization
 - https://gitpod.io/#https://github.com/confluentinc/demo-siem-optimization
 
+If you want to launch a workspace that **automatically submits all connectors**, use this link instead:
+- https://gitpod.io/#SUBMIT_CONNECTORS=true/https://github.com/confluentinc/demo-siem-optimization
+
 If you want to run locally or in a different environment, see the [appendix](./instructions/appendix.md).
 
 ### Hands-On Lab Instructions
@@ -43,4 +45,6 @@ Run through entire end-to-end demo to get the big picture. Zoom in on the indivi
 
 ### Confluent Sigma
 
-- https://github.com/michaelpeacock/kafka-sigma-streams
+
+- https://github.com/confluentinc/cyber/tree/master/confluent-sigma
+

docker-compose.yml

@@ -25,18 +25,16 @@ services:
       KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
       KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
-      KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
       KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
       KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_CONFLUENT_REPORTERS_TELEMETRY_AUTO_ENABLE: 'false'
+      KAFKA_CONFLUENT_BALANCER_ENABLE: 'false'
+      KAFKA_CONFLUENT_SCHEMA_REGISTRY_URL: http://schema-registry:8081
       KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
       KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
       KAFKA_JMX_PORT: 9101
-      KAFKA_CONFLUENT_SCHEMA_REGISTRY_URL: http://schema-registry:8081
-      CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker:29092
-      CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
-      CONFLUENT_METRICS_ENABLE: 'true'
       CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous'
 
   schema-registry:
@@ -52,7 +50,7 @@ services:
       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'broker:29092'
 
   connect:
-    image: confluentinc/cp-server-connect-base:latest
+    build: kafka-connect
     hostname: connect
     container_name: connect
     user: root
@@ -81,22 +79,14 @@ services:
       CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
       CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
       # CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
-      # ^^ From Johnny's docker-compose.yml file
       CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
       CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
       CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
       CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
-      # CLASSPATH required due to CC-2422
-      CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-latest.jar
-      CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor"
-      CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor"
       CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components"
       CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR
     volumes:
       - ./spooldir/:/var/spooldir/
-      - ./scripts/:/tmp/scripts/
-    command:
-      - /tmp/scripts/startKafkaConnectComponents.sh
 
   control-center:
     image: confluentinc/cp-enterprise-control-center:latest
@@ -113,14 +103,16 @@ services:
     environment:
       CONTROL_CENTER_BOOTSTRAP_SERVERS: 'broker:29092'
       CONTROL_CENTER_CONNECT_CONNECT-DEFAULT_CLUSTER: 'connect:8083'
+      # The control center server connects to ksqlDB through the docker network
       CONTROL_CENTER_KSQL_KSQLDB1_URL: "http://ksqldb-server:8088"
+      # If running in Gitpod, your browser must connect to ksqlDB via Gitpod's proxy URL
       CONTROL_CENTER_KSQL_KSQLDB1_ADVERTISED_URL: https://8088-${GITPOD_WORKSPACE_ID}.${GITPOD_WORKSPACE_CLUSTER_HOST}
+      # If running locally, your browser must connect to ksqlDB through localhost 8088. Comment out the above line and uncomment the line below.
+      # CONTROL_CENTER_KSQL_KSQLDB1_ADVERTISED_URL: https://localhost:8088
       CONTROL_CENTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
       CONTROL_CENTER_REPLICATION_FACTOR: 1
       CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1
-      CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1
-      CONFLUENT_METRICS_TOPIC_REPLICATION: 1
-      PORT: 9021
+      CONTROL_CENTER_MODE_ENABLE: "management"
 
   ksqldb-server:
     image: confluentinc/cp-ksqldb-server:latest
@@ -132,7 +124,7 @@ services:
     ports:
       - "8088:8088"
     volumes:
-      - ./ksql-extension:/etc/ksql-extension/
+      - ./ksqlDB/ksql-extension:/etc/ksql-extension/
       - ./mmdb:/opt/mmdb/
     environment:
       KSQL_CONFIG_DIR: "/etc/ksql"
@@ -144,8 +136,6 @@ services:
       KSQL_LISTENERS: "http://0.0.0.0:8088"
       KSQL_CACHE_MAX_BYTES_BUFFERING: 0
       KSQL_KSQL_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
-      KSQL_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor"
-      KSQL_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor"
       KSQL_KSQL_CONNECT_URL: "http://connect:8083"
       KSQL_KSQL_HIDDEN_TOPICS: '^_.*'
       KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: "true"
@@ -161,7 +151,7 @@ services:
     entrypoint: /bin/sh
     tty: true
     volumes:
-      - ./ksqldb_scripts:/ksqldb_scripts
+      - ./ksqlDB/queries:/queries
 
   # See https://github.com/berthayes/zeek-tcpreplay-kafka
   zeek-streamer:
@@ -175,8 +165,7 @@ services:
     volumes:
       - ./scripts/init_dummy.sh:/init_dummy.sh
       - ./pcaps:/pcaps
-      - ./local.zeek:/usr/local/zeek/share/zeek/site/local.zeek
-      - ./send-to-kafka.zeek:/usr/local/zeek/share/zeek/site/send-to-kafka.zeek
+      - ./zeek:/usr/local/zeek/share/zeek/site
     cap_add:
       - NET_ADMIN
 
@@ -221,30 +210,31 @@ services:
       - 8000:8000
       - 8090:8090
     volumes:
-      - ./default.yml:/tmp/defaults/default.yml
+      - ./splunk/default.yml:/tmp/defaults/default.yml
+      - ./splunk/splunk-search/:/opt/splunk/etc/apps/splunk-search
 
   splunk_uf1:
-      image: splunk/universalforwarder:latest
+      image: splunk/universalforwarder:8.2.1
       hostname: splunk_uf1
       container_name: splunk_uf1
       depends_on:
         - connect
       environment:
-        - SPLUNK_START_ARGS=--accept-license
+        - SPLUNK_START_ARGS=--accept-license --answer-yes --no-prompt
         - SPLUNK_PASSWORD=dingdong
         - SPLUNK_APPS_URL=https://raw.githubusercontent.com/JohnnyMirza/confluent_splunk_demo/main/splunk-add-on-for-cisco-asa_410.tgz
       volumes:
-        - $PWD/splunk-uf1/:/opt/splunkforwarder/etc/apps/splunk-uf1/
+        - ./splunk/splunk-uf1/:/opt/splunkforwarder/etc/apps/splunk-uf1/
       ports:
-        - 3333:3333
+        - "3333"
 
   splunk_eventgen:
     image: guilhemmarchand/splunk-eventgen:latest
     container_name: splunk_eventgen
     restart: unless-stopped
     user: 'root'
     volumes:
-      - $PWD/splunk-eventgen/:/opt/splunk-eventgen
+      - ./splunk/splunk-eventgen/:/opt/splunk-eventgen
     ports:
       - 6379:6379
       - 9500:9500
@@ -262,7 +252,7 @@ services:
       - ksqldb-server
     hostname: cyber-sigma-streams
     volumes:
-      - $PWD/scripts/:/tmp/config
+      - ./sigma:/tmp/config
     command:
       - bash
       - -c
@@ -273,7 +263,7 @@ services:
         sleep infinity
 
   cyber-sigma-regex-ui:
-    image: michaelpeacock/confluent-sigma-regex-ui:v2_1
+    image: michaelpeacock/confluent-sigma-regex-ui:latest
     container_name: cyber-sigma-regex-ui
     depends_on:
       - broker

instructions/00-executive-demo.md

@@ -107,7 +107,7 @@ Back in Gitpod, open Confluent Control Center by launching a new tab for port `9
 
 ### Filter and Enrich the DNS Stream
 
-> Let's use Confluent to optimize your data and look for threats upstream of your SIEM. We're going to use Confluent's stream processing database ksqlDB to filter, enrich, and aggregate these data streams in real-time. 
+> Let's use Confluent to optimize your data and look for threats upstream of your SIEM. We're going to use Confluent's stream processing capability ksqlDB to filter, enrich, and aggregate these data streams in real-time. 
 
 2. Create the `conn_stream` in the KSQL editor.
 
@@ -520,7 +520,7 @@ EMIT CHANGES;
 
 5. In the terminal, execute
     ```bash
-    ./scripts/submit_splunk_sink.sh
+    ./scripts/submit-connector.sh kafka-connect/connectors/splunk-sink.json
     ```
 
 6. Go to the Connect cluster in Control Center.
@@ -547,7 +547,7 @@ EMIT CHANGES;
 
 1. In the terminal, submit the connector and then go to Connect -> connectors in Control Center:
     ```bash
-    ./scripts/submit_elastic_sink.sh
+    ./scripts/submit-connector.sh kafka-connect/connectors/elastic-sink.json
     ```
 
 > You can now see we have a connector sending data to Elastic. Lets head over to Elastic to verify that its getting in.

instructions/01-introduction.md

@@ -21,4 +21,9 @@ This lab environment is a network of Docker containers. There is a Splunk event
 
 ## What next?
 
-Go to 
+- For a comprehensive, end-to-end demo, go to the [Executive Demonstration](./00-executive-demo.ms)
+- To explore different use cases, see the other small hands-on labs:
+  - [Analyze Syslog Data in Real Time with ksqlDB](./instructions/02-syslog.md)
+  - [Calculate Hourly Bandwidth Usage By Host with ksqlDB](./instructions/03-bandwidth.md)
+  - [Match Hostnames in a Watchlist Against Streaming DNS Data](./instructions/04-watchlist.md)
+  - [Filter SSL Transactions and Enrich with Geospatial Data](./instructions/05-ssl.md)

instructions/02-syslog.md

@@ -20,6 +20,13 @@ The Syslog connector is listening on port 5140/UDP.
 
 0. Open Confluent Control Center by launching a new tab for port `9021` (see [Gitpod tips](./gitpod-tips.md) if running in Gitpod).
 
+1. If it's not running already, create the syslog connector:
+   - Navigate to the connect cluster in Confluent Control Center.
+   - Select "add connector"
+   - Select "SyslogSourceConnector"
+   - Set `syslog.listener` to `UDP` and `syslog.port` to `5140`.
+   - Submit the connector.
+
 1.  Go to the ksqlDB editor in Create a stream from the syslog data with the following ksqlDB query:
 
     ```sql

instructions/04-watchlist.md

@@ -20,9 +20,9 @@ id,dateadded,domain,source
 ```
 To ingest this CSV file into a new topic and automatically create a schema for that topic, start a new Spooldir connector to watch for this source.  If you have CLI access, you can run:
 ```
-./scripts/submit_adhosts_spooldir.sh
+./scripts/submit-connector.sh kafka-connect/connectors/spooldir-source-adhosts.json
 ```
-Or you can upload the ```./scripts/adhosts_spooldir.json``` file by clicking "Upload connector config file" from within the Confluent Control Center UI.
+Or you can upload the ```kafka-connect/connectors/spooldir-source-adhosts.json``` file by clicking "Upload connector config file" from within the Confluent Control Center UI.
 
 Once this is started, or if it had already been started, the `ad_hosts.csv` file moves to:
 ```

instructions/appendix.md

@@ -10,16 +10,26 @@
 
 ### Configuring the demo environment
 
-- Running a really big pcap [optional]
-  - The packet capture file included in this repository features DNS exfiltration (among other things), but will repeat itself after a few minutes.  This can be tiresome during a live demo or workshop.
-  - Run ```python3 scripts/get_pcap.py``` script to download a 1GB/1hr playback pcap.
+1. Run ```python3 scripts/get_pcap.py``` script to download a 1GB/1hr playback pcap.
 
 
-- Configure Control Center's ksqlDB advertised listener
-  - You need to advertise the correct hostname for the ksqlDB server to ensure that the ksqlDB editor in Confluent Control Center can communicate with the ksqlDB server. 
-  - In the `docker-compose.yml` file, change the value of  `CONTROL_CENTER_KSQL_KSQLDB1_ADVERTISED_URL` to `http://localhost:8088` if running locally, or to whatever the public DNS hostname is for your EC2 instance.
+1. Configure Control Center's ksqlDB advertised listener
+    - You need to advertise the correct hostname for the ksqlDB server to ensure that the ksqlDB editor in Confluent Control Center can communicate with the ksqlDB server. 
+    - In the `docker-compose.yml` file, change the value of  `CONTROL_CENTER_KSQL_KSQLDB1_ADVERTISED_URL` to `http://localhost:8088` if running locally, or to whatever the public DNS hostname is for your VM instance.
 
 ### Starting the demo
-- Run ```docker-compose up -d```
+
+To run the demo without automatically submitting connectors, run
+```bash
+docker-compose up -d
+```
+
+If you would like to run it with connectors submitted automatically, run
+```bash
+docker-compose \
+  -f docker-compose.yml \
+  -f kafka-connect/submit-connectors.yml \
+    up -d
+```
 
 If you are using sudo with docker-compose then you will likely need to use the -E option to sudo so it inherits your environmental variables so the last command will become ```sudo -E docker-compose up -d```

kafka-connect/Dockerfile

@@ -0,0 +1,8 @@
+FROM confluentinc/cp-server-connect-base:latest
+
+# Install connectors
+RUN confluent-hub install --no-prompt confluentinc/kafka-connect-elasticsearch:latest && \
+    confluent-hub install --no-prompt splunk/kafka-connect-splunk:latest && \
+    confluent-hub install --no-prompt confluentinc/kafka-connect-splunk-s2s:latest && \
+    confluent-hub install --no-prompt jcustenborder/kafka-connect-spooldir:latest && \
+    confluent-hub install --no-prompt confluentinc/kafka-connect-syslog:latest

kafka-connect/connectors/elastic-sink.json

@@ -0,0 +1,17 @@
+{
+    "name": "ElasticsearchSinkConnectorConnector_0",
+    "config": {
+      "name": "ElasticsearchSinkConnectorConnector_0",
+      "connector.class": "io.confluent.connect.elasticsearch.ElasticsearchSinkConnector",
+      "key.converter": "org.apache.kafka.connect.storage.StringConverter",
+      "value.converter": "org.apache.kafka.connect.json.JsonConverter",
+      "key.converter.schemas.enable": "false",
+      "value.converter.schemas.enable": "false",
+      "tasks.max": "1",
+      "topics": "RICH_DNS",
+      "connection.url": "http://elasticsearch:9200",
+      "key.ignore": "true",
+      "schema.ignore": "true",
+      "type.name": "_doc"
+    }
+  }

kafka-connect/connectors/splunk-s2s-source.json

@@ -0,0 +1,17 @@
+{
+    "name": "splunk-s2s-source",
+    "config": {
+      "connector.class": "io.confluent.connect.splunk.s2s.SplunkS2SSourceConnector",
+      "topics": "splunk-s2s-events",
+      "splunk.s2s.port":"9997",
+      "kafka.topic":"splunk-s2s-events",
+      "key.converter":"org.apache.kafka.connect.storage.StringConverter",
+      "value.converter":"org.apache.kafka.connect.json.JsonConverter",
+      "key.converter.schemas.enable":"false",
+      "value.converter.schemas.enable":"false",
+      "errors.log.enable": false,
+      "errors.log.include.messages": false,
+      "confluent.topic.bootstrap.servers":"broker:29092",
+      "confluent.topic.replication.factor":"1"
+    }
+  }

kafka-connect/connectors/splunk-sink.json

@@ -0,0 +1,16 @@
+{
+    "name": "SPLUNKSINK",
+    "config": {
+      "confluent.topic.bootstrap.servers": "broker:29092",
+      "name": "SPLUNKSINK",
+      "connector.class": "com.splunk.kafka.connect.SplunkSinkConnector",
+      "tasks.max": "1",
+      "key.converter": "org.apache.kafka.connect.storage.StringConverter",
+      "value.converter": "org.apache.kafka.connect.storage.StringConverter",
+      "topics": "CISCO_ASA",
+      "splunk.hec.token": "ef16f05f-40e0-4108-a644-5323e02aaa44",
+      "splunk.hec.uri": "https://splunk:8090",
+      "splunk.hec.ssl.validate.certs": "false",
+      "splunk.hec.json.event.formatted": "true"
+    }
+  }