Add simple batch test job

.github/workflows/build.yml

@@ -113,17 +113,28 @@ jobs:
               compile usertokens_package_test.json
               test usertokens_package_test.json
 
-          - example: canary
-            path: test-jobs
+          - example: canary-aggregation
+            path: test-jobs/aggregation-tests
             tag: canary
             test_commands: |
               compile aggregation-test-package.json
               compile aggregation-query-test-package.json
               compile aggregation-query-test-iceberg-package.json
               compile aggregation-query-subscription-test-package.json
+
+          - example: canary-failures
+            path: test-jobs/failure-tests
+            tag: canary
+            test_commands: |
               compile runtime-failing-connector-package.json
               compile runtime-failing-udf-package.json
 
+          - example: canary-batch
+            path: test-jobs/batch-test
+            tag: canary
+            test_commands: |
+              compile simple-batch-test-package.json
+
           - example: udf-jbang
             path: user-defined-function/jbang
             tag: udf-jbang

test-jobs/batch-test/connectors/addresses.jsonl

@@ -0,0 +1,100 @@
+{"id":1,"name":"John Doe","street":"123 Maple Street","city":"Springfield","zip":"12345","country":"USA"}
+{"id":2,"name":"Jane Smith","street":"48 Oak Avenue","city":"Fairview","zip":"90210","country":"USA"}
+{"id":3,"name":"Michael Johnson","street":"77 Pine Road","city":"Riverside","zip":"33101","country":"USA"}
+{"id":4,"name":"Emily Brown","street":"9 Cedar Street","city":"Lakeside","zip":"60601","country":"USA"}
+{"id":5,"name":"Daniel Wilson","street":"642 Elm Drive","city":"Hillcrest","zip":"75201","country":"USA"}
+{"id":6,"name":"Sarah Miller","street":"89 Birch Lane","city":"Greenville","zip":"85001","country":"USA"}
+{"id":7,"name":"Matthew Davis","street":"501 Walnut Court","city":"Brookfield","zip":"07001","country":"USA"}
+{"id":8,"name":"Olivia Garcia","street":"32 Poplar Street","city":"Kingston","zip":"30301","country":"USA"}
+{"id":9,"name":"Christopher Martinez","street":"214 Aspen Drive","city":"Mount Vernon","zip":"55401","country":"USA"}
+{"id":10,"name":"Sophia Rodriguez","street":"56 Sycamore Avenue","city":"Clifton","zip":"11201","country":"USA"}
+{"id":11,"name":"Ethan Hernandez","street":"777 Chestnut Boulevard","city":"Oakridge","zip":"92101","country":"USA"}
+{"id":12,"name":"Ava Lee","street":"14 Dogwood Circle","city":"Mapleton","zip":"63101","country":"USA"}
+{"id":13,"name":"Isabella Walker","street":"205 Willow Way","city":"Brentwood","zip":"32801","country":"USA"}
+{"id":14,"name":"James Hall","street":"88 Hickory Road","city":"Monroe","zip":"44101","country":"USA"}
+{"id":15,"name":"Mia Allen","street":"473 Redwood Drive","city":"Pleasantville","zip":"14604","country":"USA"}
+{"id":16,"name":"Alexander Young","street":"39 Palm Street","city":"Somerset","zip":"45202","country":"USA"}
+{"id":17,"name":"Charlotte King","street":"120 Magnolia Lane","city":"Arlington","zip":"73101","country":"USA"}
+{"id":18,"name":"Benjamin Wright","street":"68 Alder Road","city":"Ashford","zip":"53703","country":"USA"}
+{"id":19,"name":"Amelia Scott","street":"311 Cypress Avenue","city":"Hamilton","zip":"27401","country":"USA"}
+{"id":20,"name":"Jacob Green","street":"52 Fir Street","city":"Glenwood","zip":"97201","country":"USA"}
+{"id":21,"name":"Harper Adams","street":"95 Laurel Drive","city":"Newport","zip":"84101","country":"USA"}
+{"id":22,"name":"William Baker","street":"412 Spruce Court","city":"Madison","zip":"53711","country":"USA"}
+{"id":23,"name":"Evelyn Nelson","street":"207 Hemlock Street","city":"Georgetown","zip":"29601","country":"USA"}
+{"id":24,"name":"Logan Carter","street":"16 Juniper Road","city":"Fairmont","zip":"68502","country":"USA"}
+{"id":25,"name":"Abigail Mitchell","street":"778 Maplewood Lane","city":"Salem","zip":"97301","country":"USA"}
+{"id":26,"name":"Henry Perez","street":"541 Cottonwood Way","city":"Rockford","zip":"61101","country":"USA"}
+{"id":27,"name":"Ella Roberts","street":"103 Beech Street","city":"Dayton","zip":"45402","country":"USA"}
+{"id":28,"name":"Lucas Turner","street":"602 Aspen Court","city":"Richmond","zip":"23219","country":"USA"}
+{"id":29,"name":"Luna Phillips","street":"27 Ivy Road","city":"Lexington","zip":"40502","country":"USA"}
+{"id":30,"name":"Sebastian Campbell","street":"330 Grove Street","city":"Albany","zip":"12207","country":"USA"}
+{"id":31,"name":"Avery Parker","street":"44 Forest Drive","city":"Trenton","zip":"08608","country":"USA"}
+{"id":32,"name":"Jack Evans","street":"159 Meadow Lane","city":"Concord","zip":"03301","country":"USA"}
+{"id":33,"name":"Sofia Edwards","street":"820 Orchard Street","city":"Hartford","zip":"06103","country":"USA"}
+{"id":34,"name":"Levi Collins","street":"73 River Road","city":"Charleston","zip":"29401","country":"USA"}
+{"id":35,"name":"Chloe Stewart","street":"18 Hilltop Drive","city":"Burlington","zip":"05401","country":"USA"}
+{"id":36,"name":"Noah Sanchez","street":"926 Valley Street","city":"Aurora","zip":"80010","country":"USA"}
+{"id":37,"name":"Grace Morris","street":"57 Harbor Road","city":"Dover","zip":"03820","country":"USA"}
+{"id":38,"name":"Daniel Butler","street":"610 Brookside Avenue","city":"Harrisburg","zip":"17101","country":"USA"}
+{"id":39,"name":"Zoey Rivera","street":"22 Highland Road","city":"Raleigh","zip":"27601","country":"USA"}
+{"id":40,"name":"Wyatt Foster","street":"345 Canyon Drive","city":"Boise","zip":"83702","country":"USA"}
+{"id":41,"name":"Aria Morgan","street":"91 Woodside Street","city":"Helena","zip":"59601","country":"USA"}
+{"id":42,"name":"David Peterson","street":"284 Ridge Road","city":"Topeka","zip":"66603","country":"USA"}
+{"id":43,"name":"Lily Bailey","street":"77 Cliff Street","city":"Cheyenne","zip":"82001","country":"USA"}
+{"id":44,"name":"Joseph Ramirez","street":"402 Pinecrest Avenue","city":"Honolulu","zip":"96813","country":"USA"}
+{"id":45,"name":"Scarlett Cooper","street":"55 Evergreen Road","city":"Anchorage","zip":"99501","country":"USA"}
+{"id":46,"name":"Owen Reed","street":"703 Summit Avenue","city":"Billings","zip":"59101","country":"USA"}
+{"id":47,"name":"Victoria Cook","street":"29 Bayshore Drive","city":"Mobile","zip":"36602","country":"USA"}
+{"id":48,"name":"Samuel Rogers","street":"144 Prairie Street","city":"Tulsa","zip":"74103","country":"USA"}
+{"id":49,"name":"Madison Morgan","street":"501 Sunset Drive","city":"Shreveport","zip":"71101","country":"USA"}
+{"id":50,"name":"Leo Bell","street":"310 Crestview Lane","city":"Wichita","zip":"67202","country":"USA"}
+{"id":51,"name":"Hannah Murphy","street":"63 Woodland Court","city":"Fargo","zip":"58102","country":"USA"}
+{"id":52,"name":"Julian Murphy","street":"12 Meadowbrook Road","city":"Sioux Falls","zip":"57104","country":"USA"}
+{"id":53,"name":"Aubrey Rivera","street":"245 Stonegate Drive","city":"Des Moines","zip":"50309","country":"USA"}
+{"id":54,"name":"Gabriel Hughes","street":"880 Prairie View Road","city":"Omaha","zip":"68102","country":"USA"}
+{"id":55,"name":"Nora Russell","street":"34 Woodland Way","city":"Lincoln","zip":"68508","country":"USA"}
+{"id":56,"name":"Carter Griffin","street":"917 Maple Court","city":"Bismarck","zip":"58501","country":"USA"}
+{"id":57,"name":"Riley Diaz","street":"71 Creekside Lane","city":"Pierre","zip":"57501","country":"USA"}
+{"id":58,"name":"Isaac Hayes","street":"450 Silver Street","city":"Salinas","zip":"93901","country":"USA"}
+{"id":59,"name":"Layla Myers","street":"23 Sunrise Drive","city":"Santa Fe","zip":"87501","country":"USA"}
+{"id":60,"name":"Hudson Price","street":"566 Orchard Lane","city":"Phoenix","zip":"85004","country":"USA"}
+{"id":61,"name":"Penelope Barnes","street":"99 Willowbrook Drive","city":"Tallahassee","zip":"32301","country":"USA"}
+{"id":62,"name":"Anthony Ross","street":"812 Park Avenue","city":"Columbia","zip":"29201","country":"USA"}
+{"id":63,"name":"Ellie Henderson","street":"47 Glenview Drive","city":"Richmond","zip":"23220","country":"USA"}
+{"id":64,"name":"Dylan Coleman","street":"380 Crestwood Drive","city":"Roanoke","zip":"24011","country":"USA"}
+{"id":65,"name":"Addison Jenkins","street":"121 Orchard Hill Road","city":"Knoxville","zip":"37902","country":"USA"}
+{"id":66,"name":"Luke Perry","street":"604 Timber Lane","city":"Asheville","zip":"28801","country":"USA"}
+{"id":67,"name":"Hazel Powell","street":"75 Ridgecrest Drive","city":"Savannah","zip":"31401","country":"USA"}
+{"id":68,"name":"Andrew Long","street":"262 Cedar Crest Lane","city":"Charlottesville","zip":"22902","country":"USA"}
+{"id":69,"name":"Violet Patterson","street":"81 Pine Valley Road","city":"Boulder","zip":"80302","country":"USA"}
+{"id":70,"name":"Grayson Hughes","street":"130 Riverbend Drive","city":"Eugene","zip":"97401","country":"USA"}
+{"id":71,"name":"Camila Flores","street":"55 Lakeview Drive","city":"Reno","zip":"89501","country":"USA"}
+{"id":72,"name":"Joshua Simmons","street":"402 Forest Hill Road","city":"Spokane","zip":"99201","country":"USA"}
+{"id":73,"name":"Natalie Bryant","street":"18 Creekwood Drive","city":"Tacoma","zip":"98402","country":"USA"}
+{"id":74,"name":"Eli Alexander","street":"570 Briarwood Lane","city":"Bozeman","zip":"59715","country":"USA"}
+{"id":75,"name":"Zoe West","street":"244 Sunset Boulevard","city":"Santa Barbara","zip":"93101","country":"USA"}
+{"id":76,"name":"Lincoln Jordan","street":"67 Maple Hollow Road","city":"Pasadena","zip":"91101","country":"USA"}
+{"id":77,"name":"Hannah Stone","street":"301 Highland Avenue","city":"Irvine","zip":"92614","country":"USA"}
+{"id":78,"name":"Christopher Warren","street":"92 Willow Ridge Road","city":"Fresno","zip":"93721","country":"USA"}
+{"id":79,"name":"Stella Boyd","street":"447 Summer Street","city":"Modesto","zip":"95354","country":"USA"}
+{"id":80,"name":"Evan Reynolds","street":"36 Cedar Hill Road","city":"Stockton","zip":"95202","country":"USA"}
+{"id":81,"name":"Lillian Carr","street":"710 Meadow View Lane","city":"Chandler","zip":"85225","country":"USA"}
+{"id":82,"name":"Jonathan Black","street":"51 Brookhaven Drive","city":"Henderson","zip":"89011","country":"USA"}
+{"id":83,"name":"Paisley Dean","street":"88 Orchard Grove Road","city":"Salt Lake City","zip":"84111","country":"USA"}
+{"id":84,"name":"Adam Crawford","street":"293 Evergreen Street","city":"Provo","zip":"84601","country":"USA"}
+{"id":85,"name":"Bella George","street":"137 Birchwood Lane","city":"Kansas City","zip":"64106","country":"USA"}
+{"id":86,"name":"Nathan Wallace","street":"799 Sycamore Ridge Road","city":"Oklahoma City","zip":"73102","country":"USA"}
+{"id":87,"name":"Skylar Cole","street":"24 Woodridge Circle","city":"New Orleans","zip":"70112","country":"USA"}
+{"id":88,"name":"Miles Weber","street":"560 Bayview Road","city":"Baton Rouge","zip":"70801","country":"USA"}
+{"id":89,"name":"Lucy Mills","street":"413 Valley Crest Drive","city":"Birmingham","zip":"35203","country":"USA"}
+{"id":90,"name":"Aaron Nichols","street":"27 Briar Lane","city":"Louisville","zip":"40202","country":"USA"}
+{"id":91,"name":"Savannah Hart","street":"962 Pine Brook Road","city":"Memphis","zip":"38103","country":"USA"}
+{"id":92,"name":"Hunter Bishop","street":"590 Meadow Brook Lane","city":"Nashville","zip":"37201","country":"USA"}
+{"id":93,"name":"Brooklyn Steele","street":"74 Riverwood Drive","city":"Milwaukee","zip":"53202","country":"USA"}
+{"id":94,"name":"Christian Barrett","street":"158 Woodland Terrace","city":"Cincinnati","zip":"45202","country":"USA"}
+{"id":95,"name":"Claire Graves","street":"803 Oak Ridge Lane","city":"Pittsburgh","zip":"15222","country":"USA"}
+{"id":96,"name":"Isaiah Brewer","street":"66 Maple Grove Court","city":"Cleveland","zip":"44114","country":"USA"}
+{"id":97,"name":"Audrey Walton","street":"402 Willow Creek Lane","city":"Detroit","zip":"48226","country":"USA"}
+{"id":98,"name":"Elianna Page","street":"125 Brookside Court","city":"St. Louis","zip":"63103","country":"USA"}
+{"id":99,"name":"Jordan Chandler","street":"380 Cedar View Road","city":"Minneapolis","zip":"55415","country":"USA"}
+{"id":100,"name":"Aiden Blake","street":"97 Forest Meadow Drive","city":"Denver","zip":"80202","country":"USA"}

test-jobs/batch-test/connectors/tables.sqrl

@@ -0,0 +1,11 @@
+CREATE TABLE Addresses WITH (
+    'connector' = 'filesystem',
+    'format' = 'flexible-json',
+    'path' = '${DATA_PATH}/addresses.jsonl'
+) LIKE `addresses.jsonl`;
+
+CREATE TABLE SinkTable WITH (
+    'connector' = 'filesystem',
+    'format' = 'flexible-json',
+    'path' = '/tmp/sink/'
+) LIKE `addresses.jsonl`;

test-jobs/batch-test/simple-batch-test-package.json

@@ -0,0 +1,18 @@
+{
+  "version": "1",
+  "enabled-engines": ["flink", "kafka", "postgres", "vertx"],
+  "script": {
+    "main": "simple-batch.sqrl"
+  },
+  "engines": {
+    "flink": {
+      "config": {
+        "execution.runtime-mode": "BATCH",
+        "table.exec.source.idle-timeout": "1 s"
+      }
+    }
+  },
+  "test-runner": {
+    "delay-sec": -1
+  }
+}

test-jobs/batch-test/simple-batch.sqrl

@@ -0,0 +1,3 @@
+IMPORT connectors.tables.*;
+
+EXPORT Addresses TO SinkTable;