Migrate Examples to DataSQRL 0.6

.github/workflows/build.yml

@@ -24,9 +24,10 @@ jobs:
             path: healthcare-study-monitoring
             test_commands: |
               test -c study_analytics_package.json study_analytics.sqrl --snapshot snapshots-study-analytics
-              test -c study_api_test_package.json --tests tests-api --snapshot snapshots-study-api
+              test -c study_api_test_package.json --tests tests-study-api --snapshot snapshots-study-api
               test -c study_stream_local_package.json study_stream.sqrl --snapshot snapshots-study-stream
-              compile study_create_api.sqrl
+              compile -c study_stream_kafka_package.json
+              compile -c study_analytics_snowflake_package.json
 
           - example: logistics
             path: logistics-shipping-geodata
@@ -36,17 +37,8 @@ jobs:
           - example: iot-sensor
             path: iot-sensor-metrics
             test_commands: |
-              test sensors.sqrl --snapshot snapshots
-
-          - example: retail
-            path: retail-customer360-nutshop
-            test_commands: |
-              test customer360.sqrl --snapshot snapshots
-
-          - example: recommendation
-            path: clickstream-ai-recommendation
-            test_commands: |
-              test -c package.json --snapshot snapshots
+              test -c sensor-static.json --snapshot snapshots-static
+              test -c sensor-api.json --tests test-api --snapshot snapshots-api
 
           - example: law
             path: law-enforcement
@@ -60,7 +52,7 @@ jobs:
 
     env:
       TZ: 'America/Los_Angeles'
-      SQRL_VERSION: 'v0.5.10'
+      SQRL_VERSION: 'ee61b3f'
 
     steps:
     - uses: actions/checkout@v4

.gitignore

@@ -4,6 +4,7 @@ target
 h2.db.mv.db
 cache
 .project
+*/myenv
 
 
 # Created by https://www.toptal.com/developers/gitignore/api/python

clickstream-ai-recommendation/README.md

@@ -1,5 +1,7 @@
 # Clickstream Recommendation
 
+> ⚠️ **Warning:** This example has not yet been updated to 0.6 and uses an older version of DataSQRL
+
 This example demonstrates DataSQRL's capabilities by creating personalized content recommendations
 based on clickstream data and content vector embeddings. The pipeline ingests data, processes it,
 and serves recommendations in real time, highlighting the power of DataSQRL in handling streaming

finance-credit-card-chatbot/creditcard-analytics.graphqls

@@ -1,14 +1,15 @@
 scalar DateTime
+scalar GraphQLBigInteger
 
 type Query {
     """
     Returns all credit card transactions within a specified time period ordered by time (most recent first).
     """
     Transactions(
         """
-        customerid: The ID of the customer whose transactions to query.
+        customerId: The ID of the customer whose transactions to query.
         """
-        customerid: Int!,
+        customerId: GraphQLBigInteger!,
         """
         fromTime: RFC-3339 compliant date time scalar. Returns transactions after this time. Use the start of the day only, e.g. `2024-01-19T00:00:00-00:00`.
         """
@@ -24,9 +25,9 @@ type Query {
     """
     SpendingByDay(
         """
-        customerid: The ID of the customer whose spending to query.
+        customerId: The ID of the customer whose spending to query.
         """
-        customerid: Int!,
+        customerId: GraphQLBigInteger!,
         """
         fromTime: RFC-3339 compliant date time scalar. Returns spending from this time. Use the start of the day only, e.g. `2024-01-19T00:00:00-00:00`.
         """
@@ -42,9 +43,9 @@ type Query {
     """
     SpendingByCategory(
         """
-        customerid: The ID of the customer whose spending to query.
+        customerId: The ID of the customer whose spending to query.
         """
-        customerid: Int!,
+        customerId: GraphQLBigInteger!,
         """
         limit: The number of spending records to return. Each week has 12 records - one for each spending category. To return the spending records for multiple weeks, multiple the number of weeks by 12. For example, to return spending records for 5 week, set the limit to 60.
         """
@@ -56,14 +57,14 @@ type Query {
     Retrieves User Chat Messages
     """
     InternalGetChatMessages(
-        customerid: Int!,
+        customerId: Int!,
         limit: Int = 10,
         offset: Int = 0
     ): [CustomerChatMessage!]
 }
 
 type CustomerTransaction {
-    transactionId: Int!
+    transactionId: GraphQLBigInteger!
     cardNo: String!
     time: String!
     amount: Float!
@@ -87,40 +88,27 @@ type CustomerChatMessage {
     content: String!
     name: String
     functionCall: String
-    customerid: Int!
+    customerId: Int!
     timestamp: String!
     uuid: String!
 }
 
 type Subscription {
-    CustomerTransaction(customerid: Int!): CustomerTransaction
+    NewCustomerTransaction(customerId: GraphQLBigInteger!): CustomerTransaction
 }
 
 type Mutation {
     InternalSaveChatMessage(message: ChatMessageInput!): CreatedChatMessage
-    AddTransaction(tx: TransactionInput!): CreatedTransaction
 }
 
 input ChatMessageInput {
     role: String!
     content: String!
     name: String
     functionCall: String
-    customerid: Int
+    customerId: Int
 }
 
 type CreatedChatMessage {
     event_time: String!
 }
-
-input TransactionInput {
-    transactionId: Int!
-    cardNo: String!
-    amount: Float!
-    merchantId: Int!
-}
-
-type CreatedTransaction {
-    transactionId: Int!
-    event_time: String!
-}

finance-credit-card-chatbot/creditcard-analytics.sqrl

@@ -1,61 +1,75 @@
 /* Import Data */
-IMPORT creditcard-data.Merchant;
-IMPORT creditcard-data.CardAssignment;
-IMPORT creditcard-data.Transaction;
-/* Import Functions */
-IMPORT time.*;
+IMPORT creditcard-data.Merchant AS _MerchantStream;
+IMPORT creditcard-data.CardAssignment AS _CardAssignmentStream;
+IMPORT creditcard-data.Transaction AS _Transaction;
 
 /* Deduplicate CDC Streams */
-Merchant :=       DISTINCT Merchant ON merchantId ORDER BY updatedTime DESC;
-CardAssignment := DISTINCT CardAssignment ON cardNo ORDER BY timestamp DESC;
+_Merchant :=       DISTINCT _MerchantStream ON merchantId ORDER BY updatedTime DESC;
+_CardAssignment := DISTINCT _CardAssignmentStream ON cardNo ORDER BY `timestamp` DESC;
 
 /* Enrich credit card transactions with customer and merchant information */
-CustomerTransaction := SELECT t.transactionId, t.cardNo, t.time, t.amount, m.name AS merchantName,
-                              m.category, c.customerid
-                       FROM Transaction t
-                       TEMPORAL JOIN CardAssignment c ON t.cardNo = c.cardNo
-                       TEMPORAL JOIN Merchant m ON t.merchantId = m.merchantid ORDER BY t.time DESC;
-
-SpendingByCategory := SELECT customerid, endOfWeek(time) as timeWeek, category, SUM(amount) as spending
-                          FROM CustomerTransaction
-                          GROUP BY customerid, timeWeek, category
-                          ORDER BY timeWeek DESC, category ASC;
-
-_SpendingByDay := SELECT customerid, endOfDay(time) as timeDay, SUM(amount) as spending
-                         FROM CustomerTransaction
-                         GROUP BY customerid, timeDay
-                         ORDER BY timeDay DESC;
+CustomerTransaction := SELECT t.transactionId, t.cardNo, t.`time`, t.amount, m.name AS merchantName,
+                              m.category, c.customerId
+                       FROM _Transaction t
+                        JOIN _CardAssignment FOR SYSTEM_TIME AS OF t.`time` c ON t.cardNo = c.cardNo
+                        JOIN _Merchant FOR SYSTEM_TIME AS OF t.`time` m ON t.merchantId = m.merchantId;
+
+/*+query_by_all(customerId) */
+SpendingByCategory := SELECT customerId, window_time as timeWeek, category, SUM(amount) as spending
+                      FROM TABLE(TUMBLE(TABLE CustomerTransaction, DESCRIPTOR(`time`), INTERVAL '7' DAY))
+                      GROUP BY customerId, window_start, window_end, window_time, category
+                      ORDER BY window_time DESC, category ASC;
+
+_SpendingByDay := SELECT customerId, window_time as timeDay, SUM(amount) as spending
+                 FROM TABLE(TUMBLE(TABLE CustomerTransaction, DESCRIPTOR(`time`), INTERVAL '1' DAY))
+                 GROUP BY customerId, window_start, window_end, window_time
+                 ORDER BY window_time DESC;
 
 /* Query Endpoints */
-Transactions(@customerid: BIGINT, @fromTime: TIMESTAMP, @toTime: TIMESTAMP) :=
-    SELECT * FROM CustomerTransaction WHERE customerid = @customerid AND @fromTime <= time AND @toTime > time
-    ORDER BY time DESC LIMIT 10000;
+Transactions(customerId BIGINT NOT NULL, fromTime TIMESTAMP NOT NULL, toTime TIMESTAMP NOT NULL) :=
+    SELECT * FROM CustomerTransaction WHERE customerId = :customerId AND :fromTime <= `time` AND :toTime > `time`
+    ORDER BY `time` DESC LIMIT 10000;
 
-SpendingByDay(@customerid: BIGINT, @fromTime: TIMESTAMP, @toTime: TIMESTAMP) :=
+SpendingByDay(customerId BIGINT NOT NULL, fromTime TIMESTAMP NOT NULL, toTime TIMESTAMP NOT NULL) :=
     SELECT timeDay, spending
-    FROM _SpendingByDay WHERE customerid = @customerid AND @fromTime <= timeDay AND @toTime > timeDay
+    FROM _SpendingByDay WHERE customerId = :customerId AND :fromTime <= timeDay AND :toTime > timeDay
     ORDER BY timeDay DESC;
 
-IMPORT creditcard-analytics.InternalSaveChatMessage;
+NewCustomerTransaction(customerId BIGINT) :=
+    SUBSCRIBE SELECT * FROM CustomerTransaction WHERE customerId = :customerId;
+
+/* ==== Agent message history === */
+
+CREATE TABLE InternalSaveChatMessage (
+    uuid STRING NOT NULL METADATA FROM 'uuid',
+    role STRING NOT NULL,
+    content STRING NOT NULL,
+    name STRING,
+    functionCall STRING,
+    customerId INT,
+    event_time TIMESTAMP_LTZ(3) NOT NULL METADATA FROM 'timestamp'
+);
 
-InternalGetChatMessages := SELECT c.role, c.content, c.name, c.functionCall, c.customerid, c.event_time AS timestamp,
-                        c._uuid AS uuid FROM InternalSaveChatMessage c ORDER BY timestamp DESC;
+/*+query_by_all(customerId) */
+InternalGetChatMessages := SELECT c.role, c.content, c.name, c.functionCall, c.customerId,
+                                  c.event_time AS `timestamp`, uuid
+                           FROM InternalSaveChatMessage c ORDER BY event_time DESC;
 
 /* =======TEST CASES======== */
 
 /*+test */
-CustomerTransactionTest := SELECT * FROM CustomerTransaction ORDER BY time DESC limit 5;
+CustomerTransactionTest := SELECT * FROM CustomerTransaction ORDER BY `time` DESC limit 5;
 
 /*+test */
-SpendingByCategoryTest := SELECT * FROM SpendingByCategory ORDER BY customerid DESC, timeWeek DESC limit 5;
+SpendingByCategoryTest := SELECT * FROM SpendingByCategory ORDER BY customerId DESC, timeWeek DESC, category ASC limit 5;
 
 /*+test */
 InternalGetChatMessagesTest := SELECT role, 
                                       content,
                                       name,
                                       functionCall, 
-                                      customerid,
-                                      timestamp, 
+                                      customerId,
+                                      `timestamp`,
                                       uuid 
                                       FROM InternalGetChatMessages
                                       LIMIT 5;

finance-credit-card-chatbot/creditcard-kafka/cardassignment.table.sql

@@ -0,0 +1,12 @@
+CREATE TABLE CardAssignment (
+     PRIMARY KEY (`customerId`, `cardNo`, `timestamp`) NOT ENFORCED,
+     WATERMARK FOR `timestamp` AS `timestamp` - INTERVAL '1' SECOND
+) WITH (
+      'connector' = 'kafka',
+      'properties.bootstrap.servers' = '${PROPERTIES_BOOTSTRAP_SERVERS}',
+      'properties.group.id' = 'mygroupid',
+      'scan.startup.mode' = 'group-offsets',
+      'properties.auto.offset.reset' = 'earliest',
+      'value.format' = 'flexible-json',
+      'topic' = 'cardassignment'
+      );

finance-credit-card-chatbot/creditcard-kafka/merchant.table.sql

@@ -0,0 +1,9 @@
+CREATE TABLE Merchant (
+     PRIMARY KEY (`merchantId`, `updatedTime`) NOT ENFORCED,
+     WATERMARK FOR `updatedTime` AS `updatedTime` - INTERVAL '1' SECOND
+) WITH (
+     'format' = 'flexible-json',
+     'path' = '${DATA_PATH}/merchant.jsonl',
+     'source.monitor-interval' = '10 min',
+     'connector' = 'filesystem'
+     );

finance-credit-card-chatbot/creditcard-kafka/merchantreward.table.sql

@@ -0,0 +1,9 @@
+CREATE TABLE MerchantReward (
+    PRIMARY KEY (`merchantId`, `updatedTime`) NOT ENFORCED,
+    WATERMARK FOR `updatedTime` AS `updatedTime` - INTERVAL '1' SECOND
+) WITH (
+    'format' = 'flexible-json',
+    'path' = '${DATA_PATH}/merchantReward.jsonl',
+    'source.monitor-interval' = '10 min',
+    'connector' = 'filesystem'
+);

finance-credit-card-chatbot/creditcard-kafka/rewardsink.table.sql

@@ -0,0 +1,13 @@
+CREATE TABLE CustomerReward (
+     PRIMARY KEY (`customerId`) NOT ENFORCED
+) WITH (
+      'connector' = 'kafka',
+      'properties.bootstrap.servers' = '${PROPERTIES_BOOTSTRAP_SERVERS}',
+      'properties.group.id' = 'mygroupid',
+      'scan.startup.mode' = 'group-offsets',
+      'properties.auto.offset.reset' = 'earliest',
+      'key.format' = 'raw',
+      'key.fields' = 'customerId',
+      'value.format' = 'flexible-json',
+      'topic' = 'customerreward'
+      );