[TWAP] Update Hermes binary response format, add TwapMessage parsing to price-service-sdk (#2158)

* feat: change hermes twap binary response, add twap data model and parsing to price-service-sdk

* feat: reexport parseTwapMessage

* test: add parse twap msg test

* test: update parse twap test

* refactor: pack both start & end updatedatas into a single binaryupdate

* refactor: snake -> camel case

Author: tejasbadadare

apps/hermes/server/Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name        = "hermes"
-version     = "0.8.0"
+version     = "0.8.1"
 description = "Hermes is an agent that provides Verified Prices from the Pythnet Pyth Oracle."
 edition     = "2021"
 

apps/hermes/server/Cargo.toml

@@ -131,22 +131,17 @@ where
     })?;
 
     let twap_update_data = twaps_with_update_data.update_data;
-    let binary: Vec<BinaryUpdate> = twap_update_data
+    let encoded_data = twap_update_data
         .into_iter()
-        .map(|data_vec| {
-            let encoded_data = data_vec
-                .into_iter()
-                .map(|data| match params.encoding {
-                    EncodingType::Base64 => base64_standard_engine.encode(data),
-                    EncodingType::Hex => hex::encode(data),
-                })
-                .collect();
-            BinaryUpdate {
-                encoding: params.encoding,
-                data: encoded_data,
-            }
+        .map(|data| match params.encoding {
+            EncodingType::Base64 => base64_standard_engine.encode(data),
+            EncodingType::Hex => hex::encode(data),
         })
         .collect();
+    let binary = BinaryUpdate {
+        encoding: params.encoding,
+        data: encoded_data,
+    };
 
     let parsed: Option<Vec<ParsedPriceFeedTwap>> = if params.parsed {
         Some(

apps/hermes/server/src/api/rest/v2/latest_twaps.rs

@@ -281,9 +281,9 @@ impl From<PriceFeedTwap> for ParsedPriceFeedTwap {
 
 #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
 pub struct TwapsResponse {
-    /// Each BinaryUpdate contains the start & end cumulative price updates used to
+    /// Contains the start & end cumulative price updates used to
     /// calculate a given price feed's TWAP.
-    pub binary: Vec<BinaryUpdate>,
+    pub binary: BinaryUpdate,
 
     /// The calculated TWAPs for each price ID
     #[serde(skip_serializing_if = "Option::is_none")]

apps/hermes/server/src/api/types.rs

@@ -206,7 +206,7 @@ pub struct PublisherStakeCapsWithUpdateData {
 #[derive(Debug)]
 pub struct TwapsWithUpdateData {
     pub twaps: Vec<PriceFeedTwap>,
-    pub update_data: Vec<Vec<Vec<u8>>>,
+    pub update_data: Vec<Vec<u8>>,
 }
 
 #[derive(Debug, Serialize)]
@@ -652,7 +652,6 @@ where
     }
 
     let mut twaps = Vec::new();
-    let mut update_data = Vec::new();
 
     // Iterate through start and end messages together
     for (start_message, end_message) in start_messages.iter().zip(end_messages.iter()) {
@@ -676,34 +675,27 @@ where
                         end_timestamp: end_twap.publish_time,
                         down_slots_ratio,
                     });
-
-                    // Combine messages for update data
-                    let mut messages = Vec::new();
-                    messages.push(start_message.clone().into());
-                    messages.push(end_message.clone().into());
-
-                    if let Ok(update) = construct_update_data(messages) {
-                        update_data.push(update);
-                    } else {
-                        tracing::warn!(
-                            "Failed to construct update data for price feed {:?}",
-                            start_twap.feed_id
-                        );
-                        continue;
-                    }
                 }
                 Err(e) => {
-                    tracing::warn!(
+                    return Err(anyhow!(
                         "Failed to calculate TWAP for price feed {:?}: {}",
                         start_twap.feed_id,
                         e
-                    );
-                    continue;
+                    ));
                 }
             }
         }
     }
 
+    // Construct update data.
+    // update_data[0] contains the start VAA and merkle proofs
+    // update_data[1] contains the end VAA and merkle proofs
+    let mut update_data =
+        construct_update_data(start_messages.into_iter().map(Into::into).collect())?;
+    update_data.extend(construct_update_data(
+        end_messages.into_iter().map(Into::into).collect(),
+    )?);
+
     Ok(TwapsWithUpdateData { twaps, update_data })
 }
 
@@ -1316,10 +1308,8 @@ mod test {
         assert_eq!(twap_2.start_timestamp, 100);
         assert_eq!(twap_2.end_timestamp, 200);
 
-        // Verify update data contains both start and end messages for both feeds
+        // update_data should have 2 elements, one for the start block and one for the end block.
         assert_eq!(result.update_data.len(), 2);
-        assert_eq!(result.update_data[0].len(), 2); // Should contain 2 messages
-        assert_eq!(result.update_data[1].len(), 2); // Should contain 2 messages
     }
     #[tokio::test]
 

apps/hermes/server/src/state/aggregate.rs

@@ -1,6 +1,6 @@
 {
   "name": "@pythnetwork/price-service-sdk",
-  "version": "1.7.1",
+  "version": "1.8.0",
   "description": "Pyth price service SDK",
   "homepage": "https://pyth.network",
   "main": "lib/index.js",

price_service/sdk/js/package.json

@@ -5,12 +5,12 @@ const MAJOR_VERSION = 1;
 const MINOR_VERSION = 0;
 const KECCAK160_HASH_SIZE = 20;
 const PRICE_FEED_MESSAGE_VARIANT = 0;
+const TWAP_MESSAGE_VARIANT = 1;
 
 export type AccumulatorUpdateData = {
   vaa: Buffer;
   updates: { message: Buffer; proof: number[][] }[];
 };
-
 export type PriceFeedMessage = {
   feedId: Buffer;
   price: BN;
@@ -22,13 +22,25 @@ export type PriceFeedMessage = {
   emaConf: BN;
 };
 
+export type TwapMessage = {
+  feedId: Buffer;
+  cumulativePrice: BN;
+  cumulativeConf: BN;
+  numDownSlots: BN;
+  exponent: number;
+  publishTime: BN;
+  prevPublishTime: BN;
+  publishSlot: BN;
+};
+
 export function isAccumulatorUpdateData(updateBytes: Buffer): boolean {
   return (
     updateBytes.toString("hex").slice(0, 8) === ACCUMULATOR_MAGIC &&
     updateBytes[4] === MAJOR_VERSION &&
     updateBytes[5] === MINOR_VERSION
   );
 }
+
 export function parsePriceFeedMessage(message: Buffer): PriceFeedMessage {
   let cursor = 0;
   const variant = message.readUInt8(cursor);
@@ -64,6 +76,41 @@ export function parsePriceFeedMessage(message: Buffer): PriceFeedMessage {
   };
 }
 
+export function parseTwapMessage(message: Buffer): TwapMessage {
+  let cursor = 0;
+  const variant = message.readUInt8(cursor);
+  if (variant !== TWAP_MESSAGE_VARIANT) {
+    throw new Error("Not a twap message");
+  }
+  cursor += 1;
+  const feedId = message.subarray(cursor, cursor + 32);
+  cursor += 32;
+  const cumulativePrice = new BN(message.subarray(cursor, cursor + 16), "be");
+  cursor += 16;
+  const cumulativeConf = new BN(message.subarray(cursor, cursor + 16), "be");
+  cursor += 16;
+  const numDownSlots = new BN(message.subarray(cursor, cursor + 8), "be");
+  cursor += 8;
+  const exponent = message.readInt32BE(cursor);
+  cursor += 4;
+  const publishTime = new BN(message.subarray(cursor, cursor + 8), "be");
+  cursor += 8;
+  const prevPublishTime = new BN(message.subarray(cursor, cursor + 8), "be");
+  cursor += 8;
+  const publishSlot = new BN(message.subarray(cursor, cursor + 8), "be");
+  cursor += 8;
+  return {
+    feedId,
+    cumulativePrice,
+    cumulativeConf,
+    numDownSlots,
+    exponent,
+    publishTime,
+    prevPublishTime,
+    publishSlot,
+  };
+}
+
 /**
  * An AccumulatorUpdateData contains a VAA and a list of updates. This function returns a new serialized AccumulatorUpdateData with only the updates in the range [start, end).
  */

price_service/sdk/js/src/AccumulatorUpdateData.ts

@@ -1,6 +1,7 @@
 import {
   parseAccumulatorUpdateData,
   parsePriceFeedMessage,
+  parseTwapMessage,
   sliceAccumulatorUpdateData,
 } from "../AccumulatorUpdateData";
 
@@ -113,4 +114,38 @@ describe("Test parse accumulator update", () => {
       "Invalid accumulator message"
     );
   });
+
+  test("Parse TWAP message", () => {
+    // Sample data from the Hermes latest TWAP endpoint.
+    const testAccumulatorDataTwap =
+      "UE5BVQEAAAADuAEAAAAEDQB0NFyANOScwaiDg0Z/8auG9F+gU98tL7TkAP7Oh5T6phJ1ztvkN/C+2vyPwzuYsY2qtW81C/TsmDISW4jprp7/AAOrwFH1EEaS7yDJ36Leva1xYh+iMITR6iQitFceC0+oPgIa24JOBZkhVn+2QU92LG5fQ7Qaigm1+SeeB5X1A8XJAQRrrQ5UwkYGFtE2XNU+pdYuSxUUaF7AbLAYu0tQ0UZEmFFRxYEhOM5dI+CmER4iXcXnbJY6vds6B4lCBGMu7dq1AAa0mOMBi3R2jUReD5fn0doFzGm7B8BD51CJYa7JL1th1g3KsgJUafvGVxRW8pVvMKGxJVnTEAty4073n0Yso72qAAgSZI1VGEhfft2ZRSbFNigZtqULTAHUs1Z/jEY1H9/VhgCOrkcX4537ypQag0782/8NOWMzyx/MIcC2TO1paC0FAApLUa4AH2mRbh9UBeMZrHhq8pqp8NiZkU91J4c97x2HpXOBuqbD+Um/zEhpBMWT2ew+5i5c2znOynCBRKmfVfX9AQvfJRz5/U2/ym9YVL2Cliq5eg7CyItz54tAoRaYr0N0RUP/S0w4o+3Vedcik1r7kE0rtulxy8GkCTmQMIhQ3zDTAA3Rug0WuQLb+ozeXprjwx/IrTY2pCo0hqOTTtYY/RqRDAnlxMWXnfFAADa2AkrPIdkrc9rcY7Vk7Q3OA2A2UDk7AQ6oE+H8iwtc6vuGgqSlPezdQwV+utfqsAtBEu4peTGYwGzgRQT6HAu3KA73IF9bS+JdDnffRIyaaSmAtgqKDc1yAQ8h92AsTgpNY+fKFwbFJKuyp92M9zVzoe8I+CNx1Mp59El/ScLRYYWfaYh3bOiJ7FLk5sWp8vKKuTv0CTNxtND5ABAKJqOrb7LSJZDP89VR7WszEW3y2ldxbWgzPcooMxczsXqFGdgKoj5puH6gNnU7tF3WDBaT2znkkQgZIE1fVGdtABEYOz3yXevBkKcPRY7Frn9RgLujva9qCJA75QTdor7w2XIhNFs8dTraTGdDE53s2syYIhh47MPYRfbrDJvJIZJ3ABJSt1XkGdeGsEA4S/78vJbmmcRndrJM5MDl1S3ChJ2iRVQgZxe0dxOHxWbwX4z5yDExkY0lfTTK3fQF2H0KQs6/AWdN2T8AAAAAABrhAfrtrFhR4yubI7X5QRqMK6xKrj7U3XuBHdGnLqSqcQAAAAAFykghAUFVV1YAAAAAAArXIu8AACcQCNiVurGRlVTMB0BmraQJiubDgKEDAGUBSfa2XLHeaxDq9158A8oCnDBtA1fpG1MRsXUISlrVVogAAAAAAAAAAAAGQO17DQ6NAAAAAAAAAAAAAASmkl6YWgAAAAAESzQb////+wAAAABnTdk/AAAAAGdN2T4AAAAACtci7wsj6vNMqJrG2JNfJY5yygVRvYFPfqEccSfDTemrudDuCgdhZucSwdNcVF/3QkxaBwCdfedAX7wyPoSu6LJJa57CwK41xm+wQUxF+sQXHePp4CsWWFrlzQNVzU4XsKhrTEdfjsRJslSTLbZpdRfIlxmaUtbr8xBKcpEQzfZjnCntTVTIQYeFvSqAdbz2Re5sjGLGnfQ8B46ZYgBeIeVUs2rIOK1rSE1ObprtZdkb4PUTqfqt96YTtAsUPMq1uVjpQu+8HtYt/BZr3A60bXnxyUxc06SJLdpmwgCZUZcTAGUBK5qx6XKigVhQhBSLoTiYAHmb1L5juVdQfbE0kxTkdEUAAAAAAAAAAA0ueWD9HZgqAAAAAAAAAAAAA3UA2y4cRwAAAAAAAGoE////+AAAAABnTdk/AAAAAGdN2T4AAAAACtci7wvdelw0MqOTe1cEWlMuAQOb+g+aOjj25mEaG17nGLUt6R+fbQmWnpeAMBY2iyR21sQh/HkkPVZ7WUvi8LIDs0l6CxKFlqBJ/GpO27lLI1ua4pgCTInm3pR6PSha3omIpRyBLlDCi+TdAW4pHS03DJ5HfzKsxxTLTsQLf+ToMwDmEQ7oOuukWrswx6YE5+5sjGLGnfQ8B46ZYgBeIeVUs2rIOK1rSE1ObprtZdkb4PUTqfqt96YTtAsUPMq1uVjpQu+8HtYt/BZr3A60bXnxyUxc06SJLdpmwgCZUZcTAGUBKgHersnlGleSd7NLEiOZmE0Lv1fiRYp+Qv7NKCmGeg0AAAAAAAAAAAAN5aKJ8+yVAAAAAAAAAAAAAAOCrlpWWgAAAAAAAGoI////+AAAAABnTdk/AAAAAGdN2T4AAAAACtci7wuKT84vWz8EFU5vAJ7UMs01HF1LnfUK2NS0SoHjdzdaIE3KToeRn1qn+JgVyownBm5NO6eveTckccp2xHbt9YeiASNxDuEx6AM7TbDcQBtoTj2s3Pk3icB5ivrH9sSOohCUJPoyi+TdAW4pHS03DJ5HfzKsxxTLTsQLf+ToMwDmEQ7oOuukWrswx6YE5+5sjGLGnfQ8B46ZYgBeIeVUs2rIOK1rSE1ObprtZdkb4PUTqfqt96YTtAsUPMq1uVjpQu+8HtYt/BZr3A60bXnxyUxc06SJLdpmwgCZUZcT";
+    const { updates } = parseAccumulatorUpdateData(
+      Buffer.from(testAccumulatorDataTwap, "base64")
+    );
+
+    // Test that both messages are parsed successfully
+    const twapMessage1 = parseTwapMessage(updates[0].message);
+    expect(twapMessage1.feedId.toString("hex")).toBe(
+      "49f6b65cb1de6b10eaf75e7c03ca029c306d0357e91b5311b175084a5ad55688"
+    );
+    expect(twapMessage1.cumulativePrice.toString()).toBe("1760238576144013");
+    expect(twapMessage1.cumulativeConf.toString()).toBe("5113466755162");
+    expect(twapMessage1.numDownSlots.toString()).toBe("72037403");
+    expect(twapMessage1.exponent).toBe(-5);
+    expect(twapMessage1.publishTime.toString()).toBe("1733155135");
+    expect(twapMessage1.prevPublishTime.toString()).toBe("1733155134");
+    expect(twapMessage1.publishSlot.toString()).toBe("181871343");
+
+    const twapMessage2 = parseTwapMessage(updates[1].message);
+    expect(twapMessage2.feedId.toString("hex")).toBe(
+      "2b9ab1e972a281585084148ba1389800799bd4be63b957507db1349314e47445"
+    );
+    expect(twapMessage2.cumulativePrice.toString()).toBe("949830028892149802");
+    expect(twapMessage2.cumulativeConf.toString()).toBe("973071467813959");
+    expect(twapMessage2.numDownSlots.toString()).toBe("27140");
+    expect(twapMessage2.exponent).toBe(-8);
+    expect(twapMessage2.publishTime.toString()).toBe("1733155135");
+    expect(twapMessage2.prevPublishTime.toString()).toBe("1733155134");
+    expect(twapMessage2.publishSlot.toString()).toBe("181871343");
+  });
 });

price_service/sdk/js/src/__tests__/AccumulatorUpdateData.test.ts

@@ -15,6 +15,7 @@ export {
   parseAccumulatorUpdateData,
   AccumulatorUpdateData,
   parsePriceFeedMessage,
+  parseTwapMessage,
 } from "./AccumulatorUpdateData";
 
 /**