diff --git a/.gitignore b/.gitignore index fc04d59..93aee7d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ .DS_Store /.idea/ +pcaps/edited_syslog.pcap +pcaps/zeek* diff --git a/.gitpod.yml b/.gitpod.yml index fecb6c9..93e775e 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -16,10 +16,17 @@ tasks: init: python3 scripts/get_pcap.py - name: download docker images - init: docker-compose pull + init: docker-compose build && docker-compose pull - name: start services - command: docker-compose up -d + command: | + if [[ -z "$SUBMIT_CONNECTORS" ]]; then + docker-compose up -d + echo "Started without connectors" + else + docker-compose -f docker-compose.yml -f kafka-connect/submit-connectors.yml up -d + echo "Started with connectors" + fi ports: diff --git a/README.md b/README.md index 1c6f56d..c241d97 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,11 @@ The examples in this repository give you hands-on experience optimizing Security This demo runs best using Gitpod. Gitpod uses your existing git service account (GitHub, Gitlab, or BitBucket) for authentication. See the [gitpod tips](./instructions/gitpod-tips.md) to get acquainted with gitpod. **Launch a workspace** to get hands-on with the labs: -- (if demo not in confluentinc yet) https://gitpod.io/#https://github.com/chuck-confluent/demo-siem-optimization - https://gitpod.io/#https://github.com/confluentinc/demo-siem-optimization +If you want to launch a workspace that **automatically submits all connectors**, use this link instead: +- https://gitpod.io/#SUBMIT_CONNECTORS=true/https://github.com/confluentinc/demo-siem-optimization + If you want to run locally or in a different environment, see the [appendix](./instructions/appendix.md). ### Hands-On Lab Instructions @@ -43,4 +45,6 @@ Run through entire end-to-end demo to get the big picture. Zoom in on the indivi ### Confluent Sigma -- https://github.com/michaelpeacock/kafka-sigma-streams + +- https://github.com/confluentinc/cyber/tree/master/confluent-sigma + diff --git a/docker-compose.yml b/docker-compose.yml index eeae036..6fe6ccc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,18 +25,16 @@ services: KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092 - KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_CONFLUENT_REPORTERS_TELEMETRY_AUTO_ENABLE: 'false' + KAFKA_CONFLUENT_BALANCER_ENABLE: 'false' + KAFKA_CONFLUENT_SCHEMA_REGISTRY_URL: http://schema-registry:8081 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_JMX_PORT: 9101 - KAFKA_CONFLUENT_SCHEMA_REGISTRY_URL: http://schema-registry:8081 - CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker:29092 - CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1 - CONFLUENT_METRICS_ENABLE: 'true' CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous' schema-registry: @@ -52,7 +50,7 @@ services: SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'broker:29092' connect: - image: confluentinc/cp-server-connect-base:latest + build: kafka-connect hostname: connect container_name: connect user: root @@ -81,22 +79,14 @@ services: CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000 CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter # CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter - # ^^ From Johnny's docker-compose.yml file CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081 CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" - # CLASSPATH required due to CC-2422 - CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-latest.jar - CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor" - CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor" CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components" CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR volumes: - ./spooldir/:/var/spooldir/ - - ./scripts/:/tmp/scripts/ - command: - - /tmp/scripts/startKafkaConnectComponents.sh control-center: image: confluentinc/cp-enterprise-control-center:latest @@ -113,14 +103,16 @@ services: environment: CONTROL_CENTER_BOOTSTRAP_SERVERS: 'broker:29092' CONTROL_CENTER_CONNECT_CONNECT-DEFAULT_CLUSTER: 'connect:8083' + # The control center server connects to ksqlDB through the docker network CONTROL_CENTER_KSQL_KSQLDB1_URL: "http://ksqldb-server:8088" + # If running in Gitpod, your browser must connect to ksqlDB via Gitpod's proxy URL CONTROL_CENTER_KSQL_KSQLDB1_ADVERTISED_URL: https://8088-${GITPOD_WORKSPACE_ID}.${GITPOD_WORKSPACE_CLUSTER_HOST} + # If running locally, your browser must connect to ksqlDB through localhost 8088. Comment out the above line and uncomment the line below. + # CONTROL_CENTER_KSQL_KSQLDB1_ADVERTISED_URL: https://localhost:8088 CONTROL_CENTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081" CONTROL_CENTER_REPLICATION_FACTOR: 1 CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1 - CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1 - CONFLUENT_METRICS_TOPIC_REPLICATION: 1 - PORT: 9021 + CONTROL_CENTER_MODE_ENABLE: "management" ksqldb-server: image: confluentinc/cp-ksqldb-server:latest @@ -132,7 +124,7 @@ services: ports: - "8088:8088" volumes: - - ./ksql-extension:/etc/ksql-extension/ + - ./ksqlDB/ksql-extension:/etc/ksql-extension/ - ./mmdb:/opt/mmdb/ environment: KSQL_CONFIG_DIR: "/etc/ksql" @@ -144,8 +136,6 @@ services: KSQL_LISTENERS: "http://0.0.0.0:8088" KSQL_CACHE_MAX_BYTES_BUFFERING: 0 KSQL_KSQL_SCHEMA_REGISTRY_URL: "http://schema-registry:8081" - KSQL_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor" - KSQL_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor" KSQL_KSQL_CONNECT_URL: "http://connect:8083" KSQL_KSQL_HIDDEN_TOPICS: '^_.*' KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: "true" @@ -161,7 +151,7 @@ services: entrypoint: /bin/sh tty: true volumes: - - ./ksqldb_scripts:/ksqldb_scripts + - ./ksqlDB/queries:/queries # See https://github.com/berthayes/zeek-tcpreplay-kafka zeek-streamer: @@ -175,8 +165,7 @@ services: volumes: - ./scripts/init_dummy.sh:/init_dummy.sh - ./pcaps:/pcaps - - ./local.zeek:/usr/local/zeek/share/zeek/site/local.zeek - - ./send-to-kafka.zeek:/usr/local/zeek/share/zeek/site/send-to-kafka.zeek + - ./zeek:/usr/local/zeek/share/zeek/site cap_add: - NET_ADMIN @@ -221,22 +210,23 @@ services: - 8000:8000 - 8090:8090 volumes: - - ./default.yml:/tmp/defaults/default.yml + - ./splunk/default.yml:/tmp/defaults/default.yml + - ./splunk/splunk-search/:/opt/splunk/etc/apps/splunk-search splunk_uf1: - image: splunk/universalforwarder:latest + image: splunk/universalforwarder:8.2.1 hostname: splunk_uf1 container_name: splunk_uf1 depends_on: - connect environment: - - SPLUNK_START_ARGS=--accept-license + - SPLUNK_START_ARGS=--accept-license --answer-yes --no-prompt - SPLUNK_PASSWORD=dingdong - SPLUNK_APPS_URL=https://raw.githubusercontent.com/JohnnyMirza/confluent_splunk_demo/main/splunk-add-on-for-cisco-asa_410.tgz volumes: - - $PWD/splunk-uf1/:/opt/splunkforwarder/etc/apps/splunk-uf1/ + - ./splunk/splunk-uf1/:/opt/splunkforwarder/etc/apps/splunk-uf1/ ports: - - 3333:3333 + - "3333" splunk_eventgen: image: guilhemmarchand/splunk-eventgen:latest @@ -244,7 +234,7 @@ services: restart: unless-stopped user: 'root' volumes: - - $PWD/splunk-eventgen/:/opt/splunk-eventgen + - ./splunk/splunk-eventgen/:/opt/splunk-eventgen ports: - 6379:6379 - 9500:9500 @@ -262,7 +252,7 @@ services: - ksqldb-server hostname: cyber-sigma-streams volumes: - - $PWD/scripts/:/tmp/config + - ./sigma:/tmp/config command: - bash - -c @@ -273,7 +263,7 @@ services: sleep infinity cyber-sigma-regex-ui: - image: michaelpeacock/confluent-sigma-regex-ui:v2_1 + image: michaelpeacock/confluent-sigma-regex-ui:latest container_name: cyber-sigma-regex-ui depends_on: - broker diff --git a/elastic-connect.json b/elastic-connect.json deleted file mode 100644 index 0e5da4a..0000000 --- a/elastic-connect.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "ElasticsearchSinkConnectorConnector_0", - "config": { - "name": "ElasticsearchSinkConnectorConnector_0", - "connector.class": "io.confluent.connect.elasticsearch.ElasticsearchSinkConnector", - "tasks.max": "1", - "topics": "DNS_AVRO", - "connection.url": "http://elasticsearch:9200", - "key.ignore": "true" - } -} \ No newline at end of file diff --git a/instructions/00-executive-demo.md b/instructions/00-executive-demo.md index 200a1ed..c9037da 100644 --- a/instructions/00-executive-demo.md +++ b/instructions/00-executive-demo.md @@ -107,7 +107,7 @@ Back in Gitpod, open Confluent Control Center by launching a new tab for port `9 ### Filter and Enrich the DNS Stream -> Let's use Confluent to optimize your data and look for threats upstream of your SIEM. We're going to use Confluent's stream processing database ksqlDB to filter, enrich, and aggregate these data streams in real-time. +> Let's use Confluent to optimize your data and look for threats upstream of your SIEM. We're going to use Confluent's stream processing capability ksqlDB to filter, enrich, and aggregate these data streams in real-time. 2. Create the `conn_stream` in the KSQL editor. @@ -520,7 +520,7 @@ EMIT CHANGES; 5. In the terminal, execute ```bash - ./scripts/submit_splunk_sink.sh + ./scripts/submit-connector.sh kafka-connect/connectors/splunk-sink.json ``` 6. Go to the Connect cluster in Control Center. @@ -547,7 +547,7 @@ EMIT CHANGES; 1. In the terminal, submit the connector and then go to Connect -> connectors in Control Center: ```bash - ./scripts/submit_elastic_sink.sh + ./scripts/submit-connector.sh kafka-connect/connectors/elastic-sink.json ``` > You can now see we have a connector sending data to Elastic. Lets head over to Elastic to verify that its getting in. diff --git a/instructions/01-introduction.md b/instructions/01-introduction.md index bd20ab3..38bd7f8 100644 --- a/instructions/01-introduction.md +++ b/instructions/01-introduction.md @@ -21,4 +21,9 @@ This lab environment is a network of Docker containers. There is a Splunk event ## What next? -Go to \ No newline at end of file +- For a comprehensive, end-to-end demo, go to the [Executive Demonstration](./00-executive-demo.ms) +- To explore different use cases, see the other small hands-on labs: + - [Analyze Syslog Data in Real Time with ksqlDB](./instructions/02-syslog.md) + - [Calculate Hourly Bandwidth Usage By Host with ksqlDB](./instructions/03-bandwidth.md) + - [Match Hostnames in a Watchlist Against Streaming DNS Data](./instructions/04-watchlist.md) + - [Filter SSL Transactions and Enrich with Geospatial Data](./instructions/05-ssl.md) \ No newline at end of file diff --git a/instructions/02-syslog.md b/instructions/02-syslog.md index a190a8d..92f6784 100644 --- a/instructions/02-syslog.md +++ b/instructions/02-syslog.md @@ -20,6 +20,13 @@ The Syslog connector is listening on port 5140/UDP. 0. Open Confluent Control Center by launching a new tab for port `9021` (see [Gitpod tips](./gitpod-tips.md) if running in Gitpod). +1. If it's not running already, create the syslog connector: + - Navigate to the connect cluster in Confluent Control Center. + - Select "add connector" + - Select "SyslogSourceConnector" + - Set `syslog.listener` to `UDP` and `syslog.port` to `5140`. + - Submit the connector. + 1. Go to the ksqlDB editor in Create a stream from the syslog data with the following ksqlDB query: ```sql diff --git a/instructions/04-watchlist.md b/instructions/04-watchlist.md index f93b2f6..bc69ac2 100644 --- a/instructions/04-watchlist.md +++ b/instructions/04-watchlist.md @@ -20,9 +20,9 @@ id,dateadded,domain,source ``` To ingest this CSV file into a new topic and automatically create a schema for that topic, start a new Spooldir connector to watch for this source. If you have CLI access, you can run: ``` -./scripts/submit_adhosts_spooldir.sh +./scripts/submit-connector.sh kafka-connect/connectors/spooldir-source-adhosts.json ``` -Or you can upload the ```./scripts/adhosts_spooldir.json``` file by clicking "Upload connector config file" from within the Confluent Control Center UI. +Or you can upload the ```kafka-connect/connectors/spooldir-source-adhosts.json``` file by clicking "Upload connector config file" from within the Confluent Control Center UI. Once this is started, or if it had already been started, the `ad_hosts.csv` file moves to: ``` diff --git a/instructions/appendix.md b/instructions/appendix.md index 9d16965..444d1d8 100644 --- a/instructions/appendix.md +++ b/instructions/appendix.md @@ -10,16 +10,26 @@ ### Configuring the demo environment -- Running a really big pcap [optional] - - The packet capture file included in this repository features DNS exfiltration (among other things), but will repeat itself after a few minutes. This can be tiresome during a live demo or workshop. - - Run ```python3 scripts/get_pcap.py``` script to download a 1GB/1hr playback pcap. +1. Run ```python3 scripts/get_pcap.py``` script to download a 1GB/1hr playback pcap. -- Configure Control Center's ksqlDB advertised listener - - You need to advertise the correct hostname for the ksqlDB server to ensure that the ksqlDB editor in Confluent Control Center can communicate with the ksqlDB server. - - In the `docker-compose.yml` file, change the value of `CONTROL_CENTER_KSQL_KSQLDB1_ADVERTISED_URL` to `http://localhost:8088` if running locally, or to whatever the public DNS hostname is for your EC2 instance. +1. Configure Control Center's ksqlDB advertised listener + - You need to advertise the correct hostname for the ksqlDB server to ensure that the ksqlDB editor in Confluent Control Center can communicate with the ksqlDB server. + - In the `docker-compose.yml` file, change the value of `CONTROL_CENTER_KSQL_KSQLDB1_ADVERTISED_URL` to `http://localhost:8088` if running locally, or to whatever the public DNS hostname is for your VM instance. ### Starting the demo -- Run ```docker-compose up -d``` + +To run the demo without automatically submitting connectors, run +```bash +docker-compose up -d +``` + +If you would like to run it with connectors submitted automatically, run +```bash +docker-compose \ + -f docker-compose.yml \ + -f kafka-connect/submit-connectors.yml \ + up -d +``` If you are using sudo with docker-compose then you will likely need to use the -E option to sudo so it inherits your environmental variables so the last command will become ```sudo -E docker-compose up -d``` diff --git a/kafka-connect/Dockerfile b/kafka-connect/Dockerfile new file mode 100644 index 0000000..658a671 --- /dev/null +++ b/kafka-connect/Dockerfile @@ -0,0 +1,8 @@ +FROM confluentinc/cp-server-connect-base:latest + +# Install connectors +RUN confluent-hub install --no-prompt confluentinc/kafka-connect-elasticsearch:latest && \ + confluent-hub install --no-prompt splunk/kafka-connect-splunk:latest && \ + confluent-hub install --no-prompt confluentinc/kafka-connect-splunk-s2s:latest && \ + confluent-hub install --no-prompt jcustenborder/kafka-connect-spooldir:latest && \ + confluent-hub install --no-prompt confluentinc/kafka-connect-syslog:latest \ No newline at end of file diff --git a/kafka-connect/connectors/elastic-sink.json b/kafka-connect/connectors/elastic-sink.json new file mode 100644 index 0000000..856c61d --- /dev/null +++ b/kafka-connect/connectors/elastic-sink.json @@ -0,0 +1,17 @@ +{ + "name": "ElasticsearchSinkConnectorConnector_0", + "config": { + "name": "ElasticsearchSinkConnectorConnector_0", + "connector.class": "io.confluent.connect.elasticsearch.ElasticsearchSinkConnector", + "key.converter": "org.apache.kafka.connect.storage.StringConverter", + "value.converter": "org.apache.kafka.connect.json.JsonConverter", + "key.converter.schemas.enable": "false", + "value.converter.schemas.enable": "false", + "tasks.max": "1", + "topics": "RICH_DNS", + "connection.url": "http://elasticsearch:9200", + "key.ignore": "true", + "schema.ignore": "true", + "type.name": "_doc" + } + } \ No newline at end of file diff --git a/kafka-connect/connectors/splunk-s2s-source.json b/kafka-connect/connectors/splunk-s2s-source.json new file mode 100644 index 0000000..99eaf2e --- /dev/null +++ b/kafka-connect/connectors/splunk-s2s-source.json @@ -0,0 +1,17 @@ +{ + "name": "splunk-s2s-source", + "config": { + "connector.class": "io.confluent.connect.splunk.s2s.SplunkS2SSourceConnector", + "topics": "splunk-s2s-events", + "splunk.s2s.port":"9997", + "kafka.topic":"splunk-s2s-events", + "key.converter":"org.apache.kafka.connect.storage.StringConverter", + "value.converter":"org.apache.kafka.connect.json.JsonConverter", + "key.converter.schemas.enable":"false", + "value.converter.schemas.enable":"false", + "errors.log.enable": false, + "errors.log.include.messages": false, + "confluent.topic.bootstrap.servers":"broker:29092", + "confluent.topic.replication.factor":"1" + } + } \ No newline at end of file diff --git a/kafka-connect/connectors/splunk-sink.json b/kafka-connect/connectors/splunk-sink.json new file mode 100644 index 0000000..5353419 --- /dev/null +++ b/kafka-connect/connectors/splunk-sink.json @@ -0,0 +1,16 @@ +{ + "name": "SPLUNKSINK", + "config": { + "confluent.topic.bootstrap.servers": "broker:29092", + "name": "SPLUNKSINK", + "connector.class": "com.splunk.kafka.connect.SplunkSinkConnector", + "tasks.max": "1", + "key.converter": "org.apache.kafka.connect.storage.StringConverter", + "value.converter": "org.apache.kafka.connect.storage.StringConverter", + "topics": "CISCO_ASA", + "splunk.hec.token": "ef16f05f-40e0-4108-a644-5323e02aaa44", + "splunk.hec.uri": "https://splunk:8090", + "splunk.hec.ssl.validate.certs": "false", + "splunk.hec.json.event.formatted": "true" + } + } \ No newline at end of file diff --git a/scripts/adhosts_spooldir.json b/kafka-connect/connectors/spooldir-source-adhosts.json similarity index 100% rename from scripts/adhosts_spooldir.json rename to kafka-connect/connectors/spooldir-source-adhosts.json diff --git a/kafka-connect/connectors/spooldir-source-csv.json b/kafka-connect/connectors/spooldir-source-csv.json new file mode 100644 index 0000000..681b2df --- /dev/null +++ b/kafka-connect/connectors/spooldir-source-csv.json @@ -0,0 +1,15 @@ +{ + "name": "csv_spooldir", + "config": { + "name": "csv_spooldir", + "connector.class": "com.github.jcustenborder.kafka.connect.spooldir.SpoolDirCsvSourceConnector", + "tasks.max": "1", + "topic": "urlhaus", + "input.path": "/var/spooldir/urlhaus/csv_input", + "finished.path": "/var/spooldir/urlhaus/csv_finished", + "error.path": "/var/spooldir/urlhaus/csv_errors", + "input.file.pattern": ".*\\.csv$", + "schema.generation.enabled": true, + "csv.first.row.as.header": true + } +} \ No newline at end of file diff --git a/scripts/syslog_source.sh b/kafka-connect/connectors/syslog-source.json old mode 100755 new mode 100644 similarity index 64% rename from scripts/syslog_source.sh rename to kafka-connect/connectors/syslog-source.json index a727c28..4405fc6 --- a/scripts/syslog_source.sh +++ b/kafka-connect/connectors/syslog-source.json @@ -1,7 +1,3 @@ -#!/bin/bash - -HEADER="Content-Type: application/json" -DATA=$( cat << EOF { "name": "syslog", "config": { @@ -12,8 +8,4 @@ DATA=$( cat << EOF "confluent.topic.bootstrap.servers": "broker:29092", "confluent.topic.replication.factor": "1" } -} -EOF -) - -curl -X POST -H "${HEADER}" --data "${DATA}" http://localhost:8083/connectors +} \ No newline at end of file diff --git a/kafka-connect/submit-connectors.yml b/kafka-connect/submit-connectors.yml new file mode 100644 index 0000000..5eb480b --- /dev/null +++ b/kafka-connect/submit-connectors.yml @@ -0,0 +1,20 @@ +--- +# Add this as an override to submit connectors. +# ex: docker-compose -f docker-compose.yml -f kafka-connect/sumbit-connectors.yml up -d +version: '3' +services: + submit-connectors: + image: confluentinc/cp-server-connect-base:latest + hostname: submit-connectors + container_name: submit-connectors + depends_on: + connect: + condition: service_healthy + volumes: + - ./scripts/submit-connector.sh:/usr/bin/submit-connector.sh + - ./kafka-connect/connectors:/connectors + command: + - bash + - -c + - | + for connector in /connectors/*.json; do submit-connector.sh $${connector} connect; done \ No newline at end of file diff --git a/ksql-extension/bert-ksql-udf-asn.jar b/ksqlDB/ksql-extension/bert-ksql-udf-asn.jar similarity index 100% rename from ksql-extension/bert-ksql-udf-asn.jar rename to ksqlDB/ksql-extension/bert-ksql-udf-asn.jar diff --git a/ksql-extension/ksql-udf-geoip.jar b/ksqlDB/ksql-extension/ksql-udf-geoip.jar similarity index 100% rename from ksql-extension/ksql-udf-geoip.jar rename to ksqlDB/ksql-extension/ksql-udf-geoip.jar diff --git a/ksqldb_scripts/create_zeek_conn_stream.sql b/ksqlDB/queries/create_zeek_conn_stream.sql similarity index 100% rename from ksqldb_scripts/create_zeek_conn_stream.sql rename to ksqlDB/queries/create_zeek_conn_stream.sql diff --git a/ksqldb_scripts/create_zeek_dhcp_stream.sql b/ksqlDB/queries/create_zeek_dhcp_stream.sql similarity index 100% rename from ksqldb_scripts/create_zeek_dhcp_stream.sql rename to ksqlDB/queries/create_zeek_dhcp_stream.sql diff --git a/ksqldb_scripts/create_zeek_dns_stream.sql b/ksqlDB/queries/create_zeek_dns_stream.sql similarity index 100% rename from ksqldb_scripts/create_zeek_dns_stream.sql rename to ksqlDB/queries/create_zeek_dns_stream.sql diff --git a/ksqldb_scripts/create_zeek_http_stream.sql b/ksqlDB/queries/create_zeek_http_stream.sql similarity index 100% rename from ksqldb_scripts/create_zeek_http_stream.sql rename to ksqlDB/queries/create_zeek_http_stream.sql diff --git a/ksqldb_scripts/create_zeek_ssl_stream.sql b/ksqlDB/queries/create_zeek_ssl_stream.sql similarity index 100% rename from ksqldb_scripts/create_zeek_ssl_stream.sql rename to ksqlDB/queries/create_zeek_ssl_stream.sql diff --git a/ksqldb_scripts/create_zeek_x509_stream.sql b/ksqlDB/queries/create_zeek_x509_stream.sql similarity index 100% rename from ksqldb_scripts/create_zeek_x509_stream.sql rename to ksqlDB/queries/create_zeek_x509_stream.sql diff --git a/ksqldb_scripts/select_rich_ssl_stream.sql b/ksqlDB/queries/select_rich_ssl_stream.sql similarity index 100% rename from ksqldb_scripts/select_rich_ssl_stream.sql rename to ksqlDB/queries/select_rich_ssl_stream.sql diff --git a/pcaps/zeek_streamer.pcap b/pcaps/zeek_streamer.pcap deleted file mode 100644 index 5189ae8..0000000 Binary files a/pcaps/zeek_streamer.pcap and /dev/null differ diff --git a/scripts/edit-docker-compose.sh b/scripts/edit-docker-compose.sh deleted file mode 100755 index 68fa854..0000000 --- a/scripts/edit-docker-compose.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -# Dumb script to edit a template file and replace the ksqldb-server line - -# This is a neat trick to find out the public hostname of your EC2 host -PUBLIC_HOSTNAME=$(curl http://169.254.169.254/latest/meta-data/public-hostname) - - -DEFAULT_COMPOSE="docker-compose.yml" -DOCKER_COMPOSE_FILE="${1:-$DEFAULT_COMPOSE}" - - -# Run this if you want to (have to) run Confluent Control Center on port 80 and ksqldb server (REST API) on port 443 with no encryption -#/bin/sed -i -e 's/ CONTROL_CENTER_KSQL_KSQLDB1_ADVERTISED_URL: \"http:\/\/localhost:8088\"/ CONTROL_CENTER_KSQL_KSQLDB1_ADVERTISED_URL: \"http:\/\/'$PUBLIC_HOSTNAME':443\"/' -e 's/ - "9021:9021"/ - "80:9021"/' -e 's/ - "8088:8088"/ - "443:8088"/' $DOCKER_COMPOSE_FILE > /home/ubuntu/cp-siem/docker-compose.yml - - -# Run this if you are sane and can run these services on their default ports (9021 for Confluent Control Center and 8088 for ksqlDB) -/bin/sed -i -e 's/ CONTROL_CENTER_KSQL_KSQLDB1_ADVERTISED_URL: .*$/ CONTROL_CENTER_KSQL_KSQLDB1_ADVERTISED_URL: \"http:\/\/'$PUBLIC_HOSTNAME':8088\"/' $DOCKER_COMPOSE_FILE diff --git a/scripts/get_pcap.py b/scripts/get_pcap.py index eb4ea85..50ce63d 100644 --- a/scripts/get_pcap.py +++ b/scripts/get_pcap.py @@ -5,7 +5,8 @@ # TODO: pull the bucket/object values from a config file print("Renaming existing zeek_streamer.pcap file") -os.rename('./pcaps/zeek_streamer.pcap', './pcaps/zeek_streamer.pcap.bak') +if os.path.exists('./pcaps/zeek_streamer.pcap'): + os.rename('./pcaps/zeek_streamer.pcap', './pcaps/zeek_streamer.pcap.bak') url = 'https://bhayes-pcaps.s3.us-east-2.amazonaws.com/garage-2020-10-18.pcap' headers = {'Host': 'bhayes-pcaps.s3.us-east-2.amazonaws.com'} diff --git a/scripts/run-ksql-query.sh b/scripts/run-ksql-query.sh index 95f5ce9..ec1f41d 100644 --- a/scripts/run-ksql-query.sh +++ b/scripts/run-ksql-query.sh @@ -7,10 +7,11 @@ # This script takes a ksqlDB query file and executes it within # a ksqldb-cli container run with docker-compose, -# assuming the query file has been mounted in the /tmp directory of the container. +# assuming the query file has been mounted in the /queries directory of the container. +QUERY=$(basename $1) docker-compose exec ksqldb-cli bash -c " cat <