[TWAP] Implement new twap (#359)

* checkpoint

* Remove all old sma files

* Bring back warnings denial

* Restore PRICE_ACCOUNT_SIZE

* Checkpoint

* Add some tests

* First test

* Cleanup

* Cleanup

* Add test sizes

* Add another test

* Add another test

* More comments

* Rename

* Finish test

* Finish test

* Delete weird files

* Delete resize_price_account

* Remove test gating

* Rename PRICE_ACCOUNT_SIZE

* Renames

* Fix tests

* Update size check to pythnet

* Add comments

* set initial size

* Refactor update

* More tests

* Delete outdate comment

* Set initial size to the actual size for price accounts

* Add size setter for backward compatibility

Author: guibescos

.github/workflows/docker.yaml

@@ -2,6 +2,7 @@ name: Docker
 
 on:
   push:
+    branches: [ main ]
   pull_request:
     branches: [ main ]
 

pc/request.cpp

@@ -354,7 +354,7 @@ void product::update( T *res )
     return;
   }
   pc_prod_t *prod;
-  size_t plen = std::max( PRICE_ACCOUNT_SIZE, (size_t)PC_PROD_ACC_SIZE );
+  size_t plen = std::max( ZSTD_UPPER_BOUND, (size_t)PC_PROD_ACC_SIZE );
   if ( sizeof( pc_prod_t ) > res->get_data_ref( prod, plen ) ||
        prod->magic_ != PC_MAGIC ||
        !init_from_account( prod ) ) {
@@ -465,7 +465,7 @@ price::price( const pub_key& acc, product *prod )
   preq_->set_account( &apub_ );
   areq_->set_sub( this );
   preq_->set_sub( this );
-  size_t tlen = ZSTD_compressBound( PRICE_ACCOUNT_SIZE );
+  size_t tlen = ZSTD_compressBound( ZSTD_UPPER_BOUND );
   pptr_ = (pc_price_t*)new char[tlen];
   __builtin_memset( pptr_, 0, tlen );
 }
@@ -964,7 +964,7 @@ void price::update( T *res )
   }
 
   // get account data
-  size_t tlen = ZSTD_compressBound( PRICE_ACCOUNT_SIZE );
+  size_t tlen = ZSTD_compressBound( ZSTD_UPPER_BOUND );
   res->get_data_val( pptr_, tlen );
   if ( PC_UNLIKELY( pptr_->magic_ != PC_MAGIC ) ) {
     on_error_sub( "bad price account header", this );

program/c/src/oracle/oracle.h

@@ -21,6 +21,7 @@ extern "C" {
 #define PC_PUBKEY_SIZE_64   (PC_PUBKEY_SIZE/sizeof(uint64_t))
 #define PC_MAP_TABLE_SIZE   640
 #define PC_COMP_SIZE         32
+#define PC_COMP_SIZE_V2     128
 
 #define PC_PROD_ACC_SIZE    512
 #define PC_EXP_DECAY         -9
@@ -200,7 +201,7 @@ static_assert( sizeof( pc_price_t ) == 3312, "" );
 
 // This constant needs to be an upper bound of the price account size, it is used within pythd for ztsd.
 // It is set tighly to the current price account + 96 component prices + 48 bytes for cumulative sums
-const uint64_t PRICE_ACCOUNT_SIZE = 3312 + 96 * sizeof( pc_price_comp_t) + 48;
+const uint64_t ZSTD_UPPER_BOUND = 3312 + 96 * sizeof( pc_price_comp_t) + 48;
 
 
 // command enumeration

program/rust/src/accounts.rs

@@ -46,7 +46,9 @@ pub use {
     permission::PermissionAccount,
     price::{
         PriceAccount,
+        PriceAccountV2,
         PriceComponent,
+        PriceCumulative,
         PriceEma,
         PriceFeedMessage,
         PriceInfo,

program/rust/src/accounts/price.rs

@@ -3,17 +3,22 @@ use {
         AccountHeader,
         PythAccount,
     },
-    crate::c_oracle_header::{
-        PC_ACCTYPE_PRICE,
-        PC_COMP_SIZE,
-        PC_PRICE_T_COMP_OFFSET,
-        PC_STATUS_TRADING,
+    crate::{
+        c_oracle_header::{
+            PC_ACCTYPE_PRICE,
+            PC_COMP_SIZE,
+            PC_COMP_SIZE_V2,
+            PC_MAX_SEND_LATENCY,
+            PC_STATUS_TRADING,
+        },
+        error::OracleError,
     },
     bytemuck::{
         Pod,
         Zeroable,
     },
     solana_program::pubkey::Pubkey,
+    std::mem::size_of,
 };
 
 #[repr(C)]
@@ -63,6 +68,101 @@ pub struct PriceAccount {
     pub comp_:              [PriceComponent; PC_COMP_SIZE as usize],
 }
 
+/// We are currently out of space in our price accounts. We plan to resize them
+/// using the resize_price_account function. Until all the accounts have been resized, all instructions MUST work with either of the two versions.
+/// Operations may check the account size to determine whether the old or the new version has been passed.
+/// The new price accounts add more publishers and introduce PriceCumulative, a new struct designed to store cumulative sums for computing TWAPs.
+#[repr(C)]
+#[derive(Copy, Clone, Pod, Zeroable)]
+pub struct PriceAccountV2 {
+    pub header:             AccountHeader,
+    /// Type of the price account
+    pub price_type:         u32,
+    /// Exponent for the published prices
+    pub exponent:           i32,
+    /// Current number of authorized publishers
+    pub num_:               u32,
+    /// Number of valid quotes for the last aggregation
+    pub num_qt_:            u32,
+    /// Last slot with a succesful aggregation (status : TRADING)
+    pub last_slot_:         u64,
+    /// Second to last slot where aggregation was attempted
+    pub valid_slot_:        u64,
+    /// Ema for price
+    pub twap_:              PriceEma,
+    /// Ema for confidence
+    pub twac_:              PriceEma,
+    /// Last time aggregation was attempted
+    pub timestamp_:         i64,
+    /// Minimum valid publisher quotes for a succesful aggregation
+    pub min_pub_:           u8,
+    pub unused_1_:          i8,
+    pub unused_2_:          i16,
+    pub unused_3_:          i32,
+    /// Corresponding product account
+    pub product_account:    Pubkey,
+    /// Next price account in the list
+    pub next_price_account: Pubkey,
+    /// Second to last slot where aggregation was succesful (i.e. status : TRADING)
+    pub prev_slot_:         u64,
+    /// Aggregate price at prev_slot_
+    pub prev_price_:        i64,
+    /// Confidence interval at prev_slot_
+    pub prev_conf_:         u64,
+    /// Timestamp of prev_slot_
+    pub prev_timestamp_:    i64,
+    /// Last attempted aggregate results
+    pub agg_:               PriceInfo,
+    /// Publishers' price components
+    pub comp_:              [PriceComponent; PC_COMP_SIZE as usize],
+    pub extra_comp_:        [PriceComponent; (PC_COMP_SIZE_V2 - PC_COMP_SIZE) as usize], // This space is empty until we update the aggregation to support more pubs
+    /// Cumulative sums of aggregative price and confidence used to compute arithmetic moving averages
+    pub price_cumulative:   PriceCumulative,
+}
+
+impl PriceAccountV2 {
+    /// This function gets triggered when there's a succesful aggregation and updates the cumulative sums
+    pub fn update_price_cumulative(&mut self) -> Result<(), OracleError> {
+        if self.agg_.status_ == PC_STATUS_TRADING {
+            self.price_cumulative.update(
+                self.agg_.price_,
+                self.agg_.conf_,
+                self.agg_.pub_slot_.saturating_sub(self.prev_slot_),
+            ); // pub_slot should always be >= prev_slot, but we protect ourselves against underflow just in case
+            Ok(())
+        } else {
+            Err(OracleError::NeedsSuccesfulAggregation)
+        }
+    }
+}
+
+// This struct can't overflow since :
+// |sum(price * slotgap)| <= sum(|price * slotgap|) <= max(|price|) * sum(slotgap) <= i64::MAX * * current_slot <= i64::MAX * u64::MAX <= i128::MAX
+// |sum(conf * slotgap)| <= sum(|conf * slotgap|) <= max(|conf|) * sum(slotgap) <= u64::MAX * current_slot <= u64::MAX * u64::MAX <= u128::MAX
+// num_gaps <= current_slot <= u64::MAX
+/// Contains cumulative sums of aggregative price and confidence used to compute arithmetic moving averages.
+/// Informally the TWAP between time t and time T can be computed as :
+/// `(T.price_cumulative.price - t.price_cumulative.price) / (T.agg_.pub_slot_ - t.agg_.pub_slot_)`
+#[repr(C)]
+#[derive(Copy, Clone, Pod, Zeroable)]
+pub struct PriceCumulative {
+    pub price:    i128, // Cumulative sum of price * slot_gap
+    pub conf:     u128, // Cumulative sum of conf * slot_gap
+    pub num_gaps: u64,  // Cumulative number of gaps in the uptime
+    pub unused:   u64,  // Padding for alignment
+}
+
+impl PriceCumulative {
+    pub fn update(&mut self, price: i64, conf: u64, slot_gap: u64) {
+        self.price += i128::from(price) * i128::from(slot_gap);
+        self.conf += u128::from(conf) * u128::from(slot_gap);
+        if slot_gap > PC_MAX_SEND_LATENCY.into() {
+            self.num_gaps += 1;
+        }
+    }
+}
+
+
 #[repr(C)]
 #[derive(Copy, Clone, Pod, Zeroable)]
 pub struct PriceComponent {
@@ -91,8 +191,12 @@ pub struct PriceEma {
 
 impl PythAccount for PriceAccount {
     const ACCOUNT_TYPE: u32 = PC_ACCTYPE_PRICE;
-    /// Equal to the offset of `comp_` in `PriceAccount`, see the trait comment for more detail
-    const INITIAL_SIZE: u32 = PC_PRICE_T_COMP_OFFSET as u32;
+    const INITIAL_SIZE: u32 = size_of::<PriceAccount>() as u32;
+}
+
+impl PythAccount for PriceAccountV2 {
+    const ACCOUNT_TYPE: u32 = PC_ACCTYPE_PRICE;
+    const INITIAL_SIZE: u32 = size_of::<PriceAccountV2>() as u32;
 }
 
 /// Message format for sending data to other chains via the accumulator program

program/rust/src/error.rs

@@ -50,6 +50,8 @@ pub enum OracleError {
     InvalidReadableAccount         = 618,
     #[error("PermissionViolation")]
     PermissionViolation            = 619,
+    #[error("NeedsSuccesfulAggregation")]
+    NeedsSuccesfulAggregation      = 620,
 }
 
 impl From<OracleError> for ProgramError {

program/rust/src/processor/add_publisher.rs

@@ -3,6 +3,7 @@ use {
         accounts::{
             PriceAccount,
             PriceComponent,
+            PythAccount,
         },
         c_oracle_header::PC_COMP_SIZE,
         deserialize::{
@@ -26,10 +27,7 @@ use {
         program_memory::sol_memset,
         pubkey::Pubkey,
     },
-    std::mem::{
-        size_of,
-        size_of_val,
-    },
+    std::mem::size_of,
 };
 
 /// Add publisher to symbol account
@@ -84,8 +82,6 @@ pub fn add_publisher(
     );
     price_data.comp_[current_index].pub_ = cmd_args.publisher;
     price_data.num_ += 1;
-    price_data.header.size =
-        try_convert::<_, u32>(size_of::<PriceAccount>() - size_of_val(&price_data.comp_))?
-            + price_data.num_ * try_convert::<_, u32>(size_of::<PriceComponent>())?;
+    price_data.header.size = try_convert::<_, u32>(PriceAccount::INITIAL_SIZE)?;
     Ok(())
 }

program/rust/src/processor/del_publisher.rs

@@ -3,6 +3,7 @@ use {
         accounts::{
             PriceAccount,
             PriceComponent,
+            PythAccount,
         },
         deserialize::{
             load,
@@ -25,10 +26,7 @@ use {
         program_memory::sol_memset,
         pubkey::Pubkey,
     },
-    std::mem::{
-        size_of,
-        size_of_val,
-    },
+    std::mem::size_of,
 };
 
 /// Delete publisher from symbol account
@@ -76,9 +74,7 @@ pub fn del_publisher(
                 0,
                 size_of::<PriceComponent>(),
             );
-            price_data.header.size =
-                try_convert::<_, u32>(size_of::<PriceAccount>() - size_of_val(&price_data.comp_))?
-                    + price_data.num_ * try_convert::<_, u32>(size_of::<PriceComponent>())?;
+            price_data.header.size = try_convert::<_, u32>(PriceAccount::INITIAL_SIZE)?;
             return Ok(());
         }
     }

program/rust/src/processor/upd_price.rs

@@ -2,8 +2,10 @@ use {
     crate::{
         accounts::{
             PriceAccount,
+            PriceAccountV2,
             PriceFeedMessage,
             PriceInfo,
+            PythAccount,
             UPD_PRICE_WRITE_SEED,
         },
         deserialize::{
@@ -163,6 +165,15 @@ pub fn upd_price(
         }
     }
 
+    {
+        let account_len = price_account.try_data_len()?;
+        if aggregate_updated && account_len >= PriceAccountV2::MINIMUM_SIZE {
+            let mut price_data =
+                load_checked::<PriceAccountV2>(price_account, cmd_args.header.version)?;
+            price_data.update_price_cumulative()?;
+        }
+    }
+
     let mut price_data = load_checked::<PriceAccount>(price_account, cmd_args.header.version)?;
     // We want to send a message every time the aggregate price updates. However, during the migration,
     // not every publisher will necessarily provide the accumulator accounts. The message_sent_ flag

program/rust/src/tests/mod.rs

@@ -17,9 +17,11 @@ mod test_publish;
 mod test_publish_batch;
 mod test_set_min_pub;
 mod test_sizes;
+mod test_twap;
 mod test_upd_aggregate;
 mod test_upd_permissions;
 mod test_upd_price;
 mod test_upd_price_no_fail_on_error;
+mod test_upd_price_v2;
 mod test_upd_product;
 mod test_utils;