Restart testing and general pubber stability (#427)

Author: grafnu

.gencode_hash.txt

@@ -1,4 +1,4 @@
-026b146c355ab6d2a6767e25d8a5c382f49c22f05289abd5eb6b6ca941e1ac47  gencode/docs/config.html
+a9cf25fa459ab0122be8df276ab4d95c6b1f2ed2256e1cf202f18c178e6906c0  gencode/docs/config.html
 90679d3d866579501e7aa00b515af05d42fc9fe399eafacaacf297d1e4a22884  gencode/docs/envelope.html
 80bbcb648a4ebe8f1b5dbb363e1654dd6db0cec2d67ac2c642f2bddc69d5a6ff  gencode/docs/event_discovery.html
 8133e380e40f27c56accbffc665b2eeb56ec84a4da3b52ba7aa5e439c9c40572  gencode/docs/event_pointset.html
@@ -9,10 +9,10 @@ f48026471ae3cd7867bce416dc21c2fb728f48d8476a8d6e95f6acaf1d8b6cf3  gencode/docs/i
 741b880216be3743f6747800a042f2dbd89f3b0344c6b0a965f4bc010f03a930  gencode/docs/schema_doc.css
 878ea88206c974f40643c3cc430875f9c4e8c5e3fd6bcd6358bd3eb6d48699a9  gencode/docs/schema_doc.min.js
 7ed934930aee763e0beebc349725ba3909115e8d346bb762f28bcbe745bb163a  gencode/docs/schema_extras.js
-46a606b867b60c1c7249612283be534ea31c95c11b04532f880bf600668124c6  gencode/docs/state.html
+9b1f7c10c78d58acf650d8f500e8d972af17fce409189652cb62a1774760cd27  gencode/docs/state.html
 d39d7fe37a41c74a40080af7b0a429d201ab1fdff7444428c4b98eb7b38c332b  gencode/java/udmi/schema/Asset.java
 0825a5cec83003bb0a6488c4ed7010a04ae0d3848ef36fe01bb4e6718ba7b96d  gencode/java/udmi/schema/Aux.java
-902704240cee3e70c0549aed1608ba8f3c2626f719483ac06631c3811d1e0330  gencode/java/udmi/schema/BlobBlobsetConfig.java
+1f2757c67215cf657297a009a01288108a04daeea919713871a2d34ea903b5f9  gencode/java/udmi/schema/BlobBlobsetConfig.java
 a7c57d119adcd0cf6363cc5301ce562004222522242e8ffd1d0cd7010f235ae1  gencode/java/udmi/schema/BlobBlobsetState.java
 b9f903444ab08907e41eb123286434ff3207b1edd01397af3ddefb8475bbdadc  gencode/java/udmi/schema/BlobsetConfig.java
 fcbed49f1af8b791d8c52bcbe18f65521a79d9ac3eb33ec3afd9b342ab2bfc56  gencode/java/udmi/schema/BlobsetState.java
@@ -64,12 +64,12 @@ abe99dd74122c186403baa6982300a9d5968f8bbb7a67b1689104111b98f32fb  gencode/java/u
 5a44075bc03f2b9b2cc090f007fd1692832871f0981dcb02579d8dda96a96206  gencode/java/udmi/schema/SetupReflectorConfig.java
 649c0291ad81421fc51da0c2f7da3286628127157a5a6eef77610e8c37c14941  gencode/java/udmi/schema/SetupReflectorState.java
 580df660dad1b97399002271716d597f72aa1a6110a49de9e162104c231752f4  gencode/java/udmi/schema/State.java
-a21ef284dcc82b0addbb10fdc429d1ef543f2353d2374a4b23c0d01e6ee2c2f9  gencode/java/udmi/schema/SystemConfig.java
+b2274107ce0e4e663b2a35e2ca284e492ec683bd687f6b8d8b014d5f8be67233  gencode/java/udmi/schema/SystemConfig.java
 7de481ba531ea915a57cb6c5b23278db4fac6588354683cf63613b20c522af24  gencode/java/udmi/schema/SystemDiscoveryEvent.java
 247652dd11714452adc27ffa542e3d5915f8e9b7b255d181723d6fe1e897b565  gencode/java/udmi/schema/SystemEvent.java
 2cf23174ef4e2876511fb471d3f9fcb5cefe2fde324db844c2d0d505fd2c8844  gencode/java/udmi/schema/SystemHardware.java
 1c79bdb3da4c9127c75aa6660be0e62bfab1d1ee47c289637662fee2e37f1491  gencode/java/udmi/schema/SystemModel.java
-127cd1f0f137cd79c8f42e289bd80e8509b7cc269f69d9ac10874706a7a1a0b6  gencode/java/udmi/schema/SystemState.java
+64e5b9a2c5699a3ca3b13dc6cb0243ff5b4c90908a14e3f11c5298ae89c8b261  gencode/java/udmi/schema/SystemState.java
 7d6dd13e368e7f073738fee69c15e18652a9b7d7ac63bde0a200f747e3aa1b1d  gencode/java/udmi/schema/TargetTestingModel.java
 d3968b92497e83a63f18cc0e74484a9807f1bb92db0c92d556ec2caaa143d645  gencode/java/udmi/schema/TestingModel.java
 8a7dd590b84944de25392b1437184608edd2f8305a183eced8b1b40eae27c6d2  gencode/java/udmi/schema/TestingSystemConfig.java
@@ -90,7 +90,7 @@ b461bdc24310ef972faf579b5be577b5af67fb0977d6afb4c42955211b26e3d5  gencode/python
 8b2bf3e753c09e375f0fd59dcf1f0be61205ed247160b7a8718cb3b9ebef2c30  gencode/python/udmi/schema/config_localnet.py
 9eab64849e04b25203d5da47856c3f8dda2b96903e4dc43ab932ee35014700bd  gencode/python/udmi/schema/config_pointset.py
 607c5047df878a1333df3ce88dcce34668959b0b315f6954bf1a4963dcf7839e  gencode/python/udmi/schema/config_pointset_point.py
-2a016b8d45868db8a146a6b84badc13c0668b94d55ab2647630645e71c4e4419  gencode/python/udmi/schema/config_system.py
+ba37d2d54df565aba42ef055a0ec961175d180c2e8092a914e62029bc5388857  gencode/python/udmi/schema/config_system.py
 97c2e5fadc6da0d84660f3296de885ab59f4b04154179b6717f77ec366f1544b  gencode/python/udmi/schema/config_system_testing.py
 5781474ae451777f65dbb13eb0a4707845551da9cfde8bf214f192e7849cd2ed  gencode/python/udmi/schema/configuration_endpoint.py
 3814c88403934dbd3fce77d92a8ad45c68dc7e07c319ba9b4e8f1b1ac7518c07  gencode/python/udmi/schema/configuration_pubber.py
@@ -129,5 +129,5 @@ a58f8c98e837a5b56126ca0f410e02f1e9cfcd80a8cb429e0ef522defab1f690  gencode/python
 05e82aa15c64842e206ae8ce3d5810d115bb890d009ea5d657822fad0e0d2165  gencode/python/udmi/schema/state_gateway.py
 3520ad936af70b414d9e7f90e606a011768bc4ee3bf1248714acc517ee9b393d  gencode/python/udmi/schema/state_pointset.py
 837ecc89c477abe3a1faf837733ca05475774891b55353d84ca231d90a1fbf31  gencode/python/udmi/schema/state_pointset_point.py
-c3d8f40e6651ca02740312066d417e59721da0ecb1860d922d53e68b650c5d64  gencode/python/udmi/schema/state_system.py
+8184e783f3e2da5a6d3fb7e7fac105aaeb28106a7c146e82b610049554abf184  gencode/python/udmi/schema/state_system.py
 791006619518fd7b38adb532879bcacd6f3f8795026cd75e10d3434a177757ac  gencode/python/udmi/schema/state_system_hardware.py

.github/workflows/testing.yml

@@ -68,9 +68,9 @@ jobs:
           echo :::::::
           echo Simple checks that a redirect happened and failed
           fgrep registries/ZZ-TRI-FECTA/devices pubber.out.1
-          fgrep 'system.config.parse success' pubber.out.1
           fgrep registries/missing/devices pubber.out.1
-          fgrep 'Not authorized to connect' pubber.out.1
+          fgrep 'While waiting for connection start: Connection error' pubber.out.1
+          fgrep 'Endpoint connection restored to last working endpoint' pubber.out.1
       - name: pubber config checks
         env:
           GCP_TARGET_PROJECT: ${{ secrets.GCP_TARGET_PROJECT }}
@@ -81,9 +81,12 @@ jobs:
           echo :::::::
           echo Check the redirect-by-config setup
           fgrep registries/ZZ-TRI-FECTA/devices pubber.out.2
-          fgrep 'system.config.parse success' pubber.out.2
           fgrep registries/reconfigure/devices pubber.out.2
-          fgrep 'Not authorized to connect' pubber.out.2
+          fgrep 'While waiting for connection start: Connection error' pubber.out.2
+          fgrep 'Endpoint connection restored to last working endpoint' pubber.out.2
+          fgrep 'Stopping system with extreme prejudice, restart true' pubber.out.2  # restart config
+          fgrep 'Stopping system with extreme prejudice, restart false' pubber.out.2  # auto-kill
+          fgrep 'Done with pubber run, exit code 193' pubber.out.2  # last_start auto-kill check
     
   udmi:
     name: Sequence tests
@@ -129,4 +132,4 @@ jobs:
         if: ${{ always() }}
         run: |
           cat pubber.out || true
-          more pubber/out/*.json
+          more pubber/out/*/*.json

.gitignore

@@ -25,6 +25,7 @@ credentials.json
 /venv/
 /local/
 .pubber.pid
+pubber.out.*
 __pycache__/
 /tests/downgrade.site/devices/*/out/
 

bin/pubber

@@ -18,8 +18,8 @@ if [ ! -f $site_path/cloud_iot_config.json ]; then
     false
 fi
 
-echo Cleaning output direction $ROOT_DIR/pubber/out
-rm -rf $ROOT_DIR/pubber/out
+echo Cleaning output directory $ROOT_DIR/pubber/out/$serial_no
+rm -rf $ROOT_DIR/pubber/out/$serial_no
 
 echo Building pubber...
 $ROOT_DIR/pubber/bin/build

bin/reset_config

@@ -25,7 +25,7 @@ src_config=${site_dir}/devices/${device_id}/out/$config_file
 now_date=$(python3 -c 'import datetime; print(datetime.datetime.utcnow().isoformat() + "Z")')
 echo Setting config timestamp ${now_date}
 jq < ${src_config} .timestamp=\"${now_date}\" |\
-    jq .system.testing.sqeuence_name=\"${config_file%.json}\" > ${dst_config}
+    jq .system.testing.sequence_name=\"${config_file%.json}\" > ${dst_config}
 
 echo Resetting device ${device_id} config...
 validator/bin/reflector ${site_dir} ${project_id} ${device_id} update/config:${dst_config}

bin/test_redirect

@@ -30,6 +30,7 @@ cloud_region=$(jq -r .cloud_region $site_config)
 registry_id=$(jq -r .registry_id $site_config)
 
 PUBBER_OUT=pubber.out
+rm -f $PUBBER_OUT.*
 
 echo Killing running pubber instances...
 ps ax | fgrep pubber | fgrep java | awk '{print $1}' | xargs kill || true
@@ -57,6 +58,9 @@ base64=$(base64 -w 0 out/endpoint.json)
 
 cat <<EOF > out/blobs.json
 {
+  "system": {
+    "mode": "active"
+  },
   "blobset": {
     "blobs": {
       "_iot_endpoint_config": {
@@ -71,6 +75,7 @@ EOF
 out_base=$site_path/devices/$device_id/out
 # Merge JSON files together into new redirect config
 jq -s '.[0] * .[1]' $out_base/generated_config.json out/blobs.json > $out_base/redirect_config.json
+jq '.system.mode = "restart"' $out_base/redirect_config.json > $out_base/restart_config.json
 
 echo New redirection config:
 cat /tmp/${device_id}_config.json
@@ -81,8 +86,21 @@ bin/reset_config $site_path $project_id $device_id redirect_config.json
 echo Let pubber react to the new configuration...
 sleep 20
 
-# Ideally use lock files, not grep-and-kill...
-echo Killing running pubber instances...
-ps ax | fgrep pubber | fgrep java | awk '{print $1}' | xargs kill || true
+echo Restart the system...
+echo bin/reset_config $site_path $project_id $device_id restart_config.json
+bin/reset_config $site_path $project_id $device_id restart_config.json
+
+echo And let it settle for restart...
+sleep 20
+
+# Now test that pubber responds to an later last_start config (to trigger automatic shutdown)
+now_date=$(python3 -c 'import datetime; print(datetime.datetime.utcnow().isoformat() + "Z")')
+echo Setting last_start time to $now_date for pubber shutdown...
+jq ".system.last_start = \"$now_date\"" $out_base/redirect_config.json > $out_base/shutdown_config.json
+echo bin/reset_config $site_path $project_id $device_id shutdown_config.json
+bin/reset_config $site_path $project_id $device_id shutdown_config.json
+
+echo And let it settle for last start...
+sleep 20
 
 echo Done with redirect test.

dashboard/functions/index.js

@@ -266,6 +266,10 @@ function process_state_update(attributes, msgObject) {
   attributes.subType = STATE_TYPE;
   promises.push(publishPubsubMessage('udmi_target', attributes, msgObject));
 
+  const stateStart = msgObject.system && msgObject.system.last_start;
+  stateStart && promises.push(modify_device_config(registryId, deviceId, 'last_start',
+                                                   stateStart, currentTimestamp()));
+
   for (var block in msgObject) {
     let subMsg = msgObject[block];
     if (typeof subMsg === 'object') {
@@ -325,11 +329,24 @@ function parse_old_config(oldConfig, resetConfig) {
   }
 }
 
+function update_last_start(config, stateStart) {
+  const configStart = config.system && config.system.last_start;
+  const shouldUpdate = stateStart && (!configStart || (stateStart > configStart));
+  console.log('State update last state/config', stateStart, configStart, shouldUpdate);
+  config.system.last_start = stateStart;
+  return shouldUpdate;
+}
+
 async function modify_device_config(registryId, deviceId, subFolder, subContents, startTime) {
   const [oldConfig, version] = await get_device_config(registryId, deviceId);
   var newConfig;
 
-  if (subFolder == 'update') {
+  if (subFolder == 'last_start') {
+    newConfig = parse_old_config(oldConfig, false);
+    if (!newConfig || !update_last_start(newConfig, subContents)) {
+      return;
+    }
+  } else if (subFolder == 'update') {
     console.log('Config replace version', version, startTime);
     newConfig = subContents;
   } else {
@@ -440,7 +457,7 @@ function consolidate_config(registryId, deviceId, subFolder) {
   if (subFolder == UPDATE_FOLDER) {
     return;
   }
-  
+
   console.log('consolidating config for', registryId, deviceId);
 
   const new_config = {