Updates for 0.7.2

finance-credit-card-chatbot/credit-card-views/creditcard_views_package_snowflake.json

@@ -13,15 +13,15 @@
     "snowflake" : {
       "catalog-name": "MyCatalog",
       "external-volume": "MyNewVolume",
-      "url": "jdbc:snowflake://${SNOWFLAKE_ID}.snowflakecomputing.com/?user=${SNOWFLAKE_USER}&password=${SNOWFLAKE_PASSWORD}&warehouse=COMPUTE_WH&db=MYSNOWFLAKEDB&schema=public&disableSslHostnameVerification=true"
+      "url": "${SNOWFLAKE_JDBC_URL}"
     }
   },
   "connectors" : {
     "iceberg" : {
       "warehouse":"s3://my-iceberg-table-test",
       "catalog-impl":"org.apache.iceberg.aws.glue.GlueCatalog",
       "io-impl":"org.apache.iceberg.aws.s3.S3FileIO",
-      "catalog-name": "mydatabase",
+      "catalog-name": "mycatalog",
       "catalog-database": "mydatabase"
     }
   },

getting-started-examples/01_kafka_to_console/README.md

@@ -1,16 +1,16 @@
-# Kafka-to-Kafka with Avro using DataSQRL
+# Kafka to Console with Avro using DataSQRL
 
 This project demonstrates how to use [DataSQRL](https://datasqrl.com) to build a streaming pipeline that:
 
-- Reads data from a kafka topic and prints output to console
+- Reads data from a Kafka topic and prints output to console
 - Kafka is part of the DataSQRL package.
 
 ## 🐳 Running DataSQRL
 
 Run the following command from the project root where your `package.json` and SQRL scripts reside:
 
 ```bash
-docker run -it --rm -p 8888:8888 -p 9092:9092 -v $PWD:/build datasqrl/cmd:0.7.1 run -c package.json
+docker run -it --rm -p 8888:8888 -p 9092:9092 -v $PWD:/build datasqrl/cmd:latest run -c package.json
 ```
 
 ## Generate Data

getting-started-examples/02_kafka_to_kafka/README.md

@@ -1,16 +1,16 @@
-# Kafka-to-Kafka with Avro using DataSQRL
+# Kafka to Kafka with Avro using DataSQRL
 
 This project demonstrates how to use [DataSQRL](https://datasqrl.com) to build a streaming pipeline that:
 
-- Reads data from a kafka topic and writes to another kafka topic
+- Reads data from a Kafka topic and writes to another Kafka topic
 - Kafka is part of the DataSQRL package.
 
 ## 🐳 Running DataSQRL
 
 Run the following command from the project root where your `package.json` and SQRL scripts reside:
 
 ```bash
-docker run -it --rm -p 8888:8888 -p 9092:9092  -v $PWD:/build datasqrl/cmd:0.7.1 run -c package.json
+docker run -it --rm -p 8888:8888 -p 9092:9092  -v $PWD:/build datasqrl/cmd:latest run -c package.json
 ```
 
 ## Generate Data

getting-started-examples/03_two_streams_kafka_to_kafka/README.md → getting-started-examples/03_kafka_join/README.md

@@ -1,8 +1,8 @@
-# Kafka-to-Kafka with Avro using DataSQRL
+# Kafka Join using DataSQRL
 
 This project demonstrates how to use [DataSQRL](https://datasqrl.com) to build a streaming pipeline that:
 
-- Reads data from two kafka topics and combines the data from two streams using temporal join
+- Reads data from two Kafka topics and combines the data from two streams using temporal join
 - writes output to another kafka topic
 - Kafka is part of datasqrl package.
 
@@ -13,7 +13,7 @@ This project demonstrates how to use [DataSQRL](https://datasqrl.com) to build a
 Run the following command from the project root where your `package.json` and SQRL scripts reside:
 
 ```bash
-docker run -it --rm -p 8888:8888 -p 9092:9092 -v $PWD:/build datasqrl/cmd:0.7.1 run -c package.json
+docker run -it --rm -p 8888:8888 -p 9092:9092 -v $PWD:/build datasqrl/cmd:latest run -c package.json
 ```
 
 ## Generate Data

getting-started-examples/05_file_iceberg_test/README.md → getting-started-examples/04_file_to_iceberg_test/README.md

@@ -9,13 +9,13 @@ This project demonstrates how to use [DataSQRL](https://datasqrl.com) to build a
 Run the following command from the project root where your `package.json` and SQRL scripts reside:
 
 ```bash
-docker run -it --rm -p 8888:8888 -p 8081:8081 -v $PWD:/build -v $PWD/data:/data datasqrl/cmd:0.7.1 run -c package.json
+docker run -it --rm -p 8888:8888 -p 8081:8081 -v $PWD:/build datasqrl/cmd:latest run -c package.json
 ```
 
 > [!NOTE]
 > Iceberg files will be stored in the `warehouse` directory set by `package.json`
 
 ## Output
 
-* There should be iceberg files and folders generated in `$PWD/data/iceberg` directory 
+* There should be iceberg files and folders generated in `$PWD/warehouse` directory 
 * Data for the output table will reside in `ProcessedData` (as defined in the sqrl script)

getting-started-examples/05_file_iceberg_test/package.json → getting-started-examples/04_file_to_iceberg_test/package.json

@@ -14,7 +14,7 @@
   },
   "connectors": {
     "iceberg": {
-      "warehouse": "/data/iceberg",
+      "warehouse": "warehouse",
       "catalog-type": "hadoop",
       "catalog-name": "mycatalog"
     }

getting-started-examples/05_kafka_to_iceberg_local_test/README.md

@@ -0,0 +1,27 @@
+# Kafka to Local Iceberg Warehouse Using DataSQRL
+
+This project demonstrates how to use [DataSQRL](https://datasqrl.com) to build a streaming pipeline that:
+
+- Reads data from a Kafka topic
+- Writes data to an Iceberg table locally
+
+## 🐳 Running DataSQRL
+
+Run the following command from the project root where your `package.json` and SQRL scripts reside:
+```bash
+docker run -it --rm -p 8888:8888 -p 8081:8081 -p 9092:9092 -v $PWD:/build datasqrl/cmd:latest run -c package.json
+```
+
+## Generate Data
+
+* Go to `data-generator` folder
+   * `python3 load_data.py <jsonl_file> <topic_name>`
+* To send Contact data
+```bash
+python3 load_data.py contacts.jsonl contacts
+```
+
+## Output
+
+* There should be iceberg files and folders generated in `$PWD/warehouse` directory
+* Data for the output table will reside in `MyContacts` (as defined in the sqrl script)

getting-started-examples/07_external_kafka_iceberg_glue_test/kafka-source/Contact.table.sql → getting-started-examples/05_kafka_to_iceberg_local_test/kafka-source/Contact.table.sql

@@ -6,9 +6,9 @@ CREATE TABLE Contact (
   WATERMARK FOR last_updated AS last_updated - INTERVAL '1' SECOND
 ) WITH (
   'connector' = 'kafka',
-  'topic' = 'contact',
-  'properties.bootstrap.servers' = 'host.docker.internal:9092',
-  'properties.group.id' = 'group1_contacts',
+  'topic' = 'contacts',
+  'properties.bootstrap.servers' = '${KAFKA_BOOTSTRAP_SERVERS}',
+  'properties.group.id' = '${KAFKA_GROUP_ID}',
   'scan.startup.mode' = 'earliest-offset',
   'format' = 'flexible-json'
 );

getting-started-examples/05_kafka_to_iceberg_local_test/kafka-to-iceberg.sqrl

@@ -0,0 +1,3 @@
+IMPORT kafka-source.Contact AS _Contacts;
+
+MyContacts := SELECT id, firstname, lastname, last_updated FROM _Contacts;

getting-started-examples/06_external_kafka_iceberg_test/package.json → getting-started-examples/05_kafka_to_iceberg_local_test/package.json

@@ -1,6 +1,6 @@
 {
   "version": "1",
-  "enabled-engines": ["flink", "iceberg"],
+  "enabled-engines": ["flink", "kafka", "iceberg"],
   "script": {
     "main": "kafka-to-iceberg.sqrl"
   },
@@ -14,9 +14,12 @@
   },
   "connectors": {
     "iceberg": {
-      "warehouse": "/data/iceberg",
+      "warehouse": "warehouse",
       "catalog-type": "hadoop",
       "catalog-name": "mycatalog"
     }
+  },
+  "test-runner": {
+    "create-topics": ["contacts"]
   }
 }