Add test utilities to compensate hiding the storage interface. (#1556)

### What

Add test utilities to compensate hiding the storage interface.

Also do a small passing-by cleanup for the storage function names.

### Why

There generally should not be a need to expose the Storage
implementation to the env users; the interface is pretty complex and we
want to be able to change it (as we did e.g. for the error handling).
The test utilities make sure the Storage remains mostly encapsulated in
host, while SDK gets the necessary tools to read/write the arbitrary
ledger entries (which is mostly useful for SAC setup - maybe eventually
this should be moved to host as well, as this is protocol level logic).

### Known limitations

N/A

Author: dmkozh

soroban-env-host/src/auth.rs

@@ -2226,22 +2226,19 @@ impl Host {
         let live_until_ledger = live_until_ledger
             .max(self.get_min_live_until_ledger(xdr::ContractDataDurability::Temporary)?);
         self.with_mut_storage(|storage| {
-            if storage
-                .has_with_host(&nonce_key, self, None)
-                .map_err(|err| {
-                    if err.error.is_type(ScErrorType::Storage)
-                        && err.error.is_code(ScErrorCode::ExceededLimit)
-                    {
-                        return self.err(
-                            ScErrorType::Storage,
-                            ScErrorCode::ExceededLimit,
-                            "trying to access nonce outside of footprint for address",
-                            &[address.to_val()],
-                        );
-                    }
-                    err
-                })?
-            {
+            if storage.has(&nonce_key, self, None).map_err(|err| {
+                if err.error.is_type(ScErrorType::Storage)
+                    && err.error.is_code(ScErrorCode::ExceededLimit)
+                {
+                    return self.err(
+                        ScErrorType::Storage,
+                        ScErrorCode::ExceededLimit,
+                        "trying to access nonce outside of footprint for address",
+                        &[address.to_val()],
+                    );
+                }
+                err
+            })? {
                 return Err(self.err(
                     ScErrorType::Auth,
                     ScErrorCode::ExistingValue,
@@ -2261,7 +2258,7 @@ impl Host {
                 data,
                 ext: LedgerEntryExt::V0,
             };
-            storage.put_with_host(
+            storage.put(
                 &nonce_key,
                 &Rc::metered_new(entry, self)?,
                 Some(live_until_ledger),

soroban-env-host/src/builtin_contracts/stellar_asset_contract/balance.rs

@@ -453,7 +453,7 @@ fn transfer_account_balance(
         if new_balance >= min_balance && new_balance <= max_balance {
             ae.balance = new_balance;
             le = Host::modify_ledger_entry_data(host, &le, LedgerEntryData::Account(ae))?;
-            storage.put_with_host(&lk, &le, None, &host, None)
+            storage.put(&lk, &le, None, &host, None)
         } else {
             Err(err!(
                 host,
@@ -535,7 +535,7 @@ fn transfer_trustline_balance(
         if new_balance >= min_balance && new_balance <= max_balance {
             tl.balance = new_balance;
             le = Host::modify_ledger_entry_data(host, &le, LedgerEntryData::Trustline(tl))?;
-            storage.put_with_host(&lk, &le, None, &host, None)
+            storage.put(&lk, &le, None, &host, None)
         } else {
             Err(err!(
                 host,
@@ -805,7 +805,7 @@ fn set_trustline_authorization(
             tl.flags |= TrustLineFlags::AuthorizedToMaintainLiabilitiesFlag as u32;
         }
         le = Host::modify_ledger_entry_data(host, &le, LedgerEntryData::Trustline(tl))?;
-        storage.put_with_host(&lk, &le, None, &host, None)
+        storage.put(&lk, &le, None, &host, None)
     })
 }
 

soroban-env-host/src/e2e_invoke.rs

@@ -652,7 +652,7 @@ pub fn invoke_host_function_in_recording_mode(
     base_prng_seed: [u8; 32],
     diagnostic_events: &mut Vec<DiagnosticEvent>,
 ) -> Result<InvokeHostFunctionRecordingModeResult, HostError> {
-    let storage = Storage::with_recording_footprint(ledger_snapshot.clone())?;
+    let storage = Storage::with_recording_footprint(ledger_snapshot.clone());
     let host = Host::with_storage_and_budget(storage, budget.clone());
     let is_recording_auth = matches!(auth_mode, RecordingInvocationAuthMode::Recording(_));
     let ledger_seq = ledger_info.sequence_number;

soroban-env-host/src/host.rs

@@ -2095,8 +2095,7 @@ impl VmCallerEnv for Host {
         let res = match t {
             StorageType::Temporary | StorageType::Persistent => {
                 let key = self.storage_key_from_val(k, t.try_into()?)?;
-                self.try_borrow_storage_mut()?
-                    .has_with_host(&key, self, Some(k))?
+                self.try_borrow_storage_mut()?.has(&key, self, Some(k))?
             }
             StorageType::Instance => {
                 self.with_instance_storage(|s| Ok(s.map.get(&k, self)?.is_some()))?
@@ -2116,9 +2115,7 @@ impl VmCallerEnv for Host {
         match t {
             StorageType::Temporary | StorageType::Persistent => {
                 let key = self.storage_key_from_val(k, t.try_into()?)?;
-                let entry = self
-                    .try_borrow_storage_mut()?
-                    .get_with_host(&key, self, Some(k))?;
+                let entry = self.try_borrow_storage_mut()?.get(&key, self, Some(k))?;
                 match &entry.data {
                     LedgerEntryData::ContractData(e) => Ok(self.to_valid_host_val(&e.val)?),
                     _ => Err(self.err(
@@ -2155,8 +2152,7 @@ impl VmCallerEnv for Host {
         match t {
             StorageType::Temporary | StorageType::Persistent => {
                 let key = self.storage_key_from_val(k, t.try_into()?)?;
-                self.try_borrow_storage_mut()?
-                    .del_with_host(&key, self, Some(k))?;
+                self.try_borrow_storage_mut()?.del(&key, self, Some(k))?;
             }
             StorageType::Instance => {
                 self.with_mut_instance_storage(|s| {
@@ -3367,44 +3363,40 @@ impl VmCallerEnv for Host {
         address: AddressObject,
     ) -> Result<Val, Self::Error> {
         let sc_address = self.scaddress_from_address(address)?;
-        let maybe_executable =
-            match sc_address {
-                ScAddress::Account(account_id) => {
-                    let key = self.to_account_key(account_id)?;
-                    if self
-                        .try_borrow_storage_mut()?
-                        .has_with_host(&key, &self, None)?
-                    {
-                        Some(AddressExecutable::Account)
-                    } else {
-                        None
-                    }
-                }
-                ScAddress::Contract(id) => {
-                    let storage_key = self.contract_instance_ledger_key(&id)?;
-                    let maybe_instance_entry = self
-                        .try_borrow_storage_mut()?
-                        .try_get_full_with_host(&storage_key, self, None)?;
-                    if let Some((instance_entry, _ttl)) = maybe_instance_entry {
-                        let instance =
-                            self.extract_contract_instance_from_ledger_entry(&instance_entry)?;
-                        Some(AddressExecutable::from_contract_executable_xdr(
-                            &self,
-                            &instance.executable,
-                        )?)
-                    } else {
-                        None
-                    }
+        let maybe_executable = match sc_address {
+            ScAddress::Account(account_id) => {
+                let key = self.to_account_key(account_id)?;
+                if self.try_borrow_storage_mut()?.has(&key, &self, None)? {
+                    Some(AddressExecutable::Account)
+                } else {
+                    None
                 }
-                _ => {
-                    return Err(self.err(
-                        ScErrorType::Object,
-                        ScErrorCode::InternalError,
-                        "Unexpected Address variant in get_address_executable",
-                        &[address.into()],
-                    ));
+            }
+            ScAddress::Contract(id) => {
+                let storage_key = self.contract_instance_ledger_key(&id)?;
+                let maybe_instance_entry =
+                    self.try_borrow_storage_mut()?
+                        .try_get_full(&storage_key, self, None)?;
+                if let Some((instance_entry, _ttl)) = maybe_instance_entry {
+                    let instance =
+                        self.extract_contract_instance_from_ledger_entry(&instance_entry)?;
+                    Some(AddressExecutable::from_contract_executable_xdr(
+                        &self,
+                        &instance.executable,
+                    )?)
+                } else {
+                    None
                 }
-            };
+            }
+            _ => {
+                return Err(self.err(
+                    ScErrorType::Object,
+                    ScErrorCode::InternalError,
+                    "Unexpected Address variant in get_address_executable",
+                    &[address.into()],
+                ));
+            }
+        };
         if let Some(exec) = maybe_executable {
             Val::try_from_val(self, &exec)
         } else {

soroban-env-host/src/host/data_helper.rs

@@ -19,7 +19,7 @@ use crate::{
 };
 
 impl Host {
-    pub fn with_mut_storage<F, U>(&self, f: F) -> Result<U, HostError>
+    pub(crate) fn with_mut_storage<F, U>(&self, f: F) -> Result<U, HostError>
     where
         F: FnOnce(&mut Storage) -> Result<U, HostError>,
     {
@@ -115,9 +115,7 @@ impl Host {
         &self,
         key: &Rc<LedgerKey>,
     ) -> Result<ScContractInstance, HostError> {
-        let entry = self
-            .try_borrow_storage_mut()?
-            .get_with_host(key, self, None)?;
+        let entry = self.try_borrow_storage_mut()?.get(key, self, None)?;
         self.extract_contract_instance_from_ledger_entry(&entry)
     }
 
@@ -137,11 +135,7 @@ impl Host {
         wasm_hash: &Hash,
     ) -> Result<(BytesM, VersionedContractCodeCostInputs), HostError> {
         let key = self.contract_code_ledger_key(wasm_hash)?;
-        match &self
-            .try_borrow_storage_mut()?
-            .get_with_host(&key, self, None)?
-            .data
-        {
+        match &self.try_borrow_storage_mut()?.get(&key, self, None)?.data {
             LedgerEntryData::ContractCode(e) => {
                 let code = e.code.metered_clone(self)?;
                 let costs = match &e.ext {
@@ -167,8 +161,7 @@ impl Host {
 
     pub(crate) fn wasm_exists(&self, wasm_hash: &Hash) -> Result<bool, HostError> {
         let key = self.contract_code_ledger_key(wasm_hash)?;
-        self.try_borrow_storage_mut()?
-            .has_with_host(&key, self, None)
+        self.try_borrow_storage_mut()?.has(&key, self, None)
     }
 
     // Stores the contract instance specified with its parts (executable and
@@ -184,10 +177,7 @@ impl Host {
         contract_id: ContractId,
         key: &Rc<LedgerKey>,
     ) -> Result<(), HostError> {
-        if self
-            .try_borrow_storage_mut()?
-            .has_with_host(key, self, None)?
-        {
+        if self.try_borrow_storage_mut()?.has(key, self, None)? {
             let (current, live_until_ledger) = self
                 .try_borrow_storage_mut()?
                 .get_with_live_until_ledger(key, self, None)?;
@@ -218,7 +208,7 @@ impl Host {
                 ));
             }
 
-            self.try_borrow_storage_mut()?.put_with_host(
+            self.try_borrow_storage_mut()?.put(
                 &key,
                 &Rc::metered_new(current, self)?,
                 live_until_ledger,
@@ -243,7 +233,7 @@ impl Host {
                 durability: ContractDataDurability::Persistent,
                 ext: ExtensionPoint::V0,
             };
-            self.try_borrow_storage_mut()?.put_with_host(
+            self.try_borrow_storage_mut()?.put(
                 key,
                 &Host::new_contract_data(self, data)?,
                 Some(self.get_min_live_until_ledger(ContractDataDurability::Persistent)?),
@@ -305,17 +295,15 @@ impl Host {
     // notes on metering: `get` from storage is covered. Rest are free.
     pub(crate) fn load_account(&self, account_id: AccountId) -> Result<AccountEntry, HostError> {
         let acc = self.to_account_key(account_id)?;
-        self.with_mut_storage(
-            |storage| match &storage.get_with_host(&acc, self, None)?.data {
-                LedgerEntryData::Account(ae) => ae.metered_clone(self),
-                e => Err(err!(
-                    self,
-                    (ScErrorType::Storage, ScErrorCode::InternalError),
-                    "ledger entry is not account",
-                    e.name()
-                )),
-            },
-        )
+        self.with_mut_storage(|storage| match &storage.get(&acc, self, None)?.data {
+            LedgerEntryData::Account(ae) => ae.metered_clone(self),
+            e => Err(err!(
+                self,
+                (ScErrorType::Storage, ScErrorCode::InternalError),
+                "ledger entry is not account",
+                e.name()
+            )),
+        })
     }
 
     pub(crate) fn to_account_key(&self, account_id: AccountId) -> Result<Rc<LedgerKey>, HostError> {
@@ -485,10 +473,7 @@ impl Host {
         // operation might only modify the internal `ScVal` value. Thus we
         // need to only overwrite the value in case if there is already an
         // existing ledger entry value for the key in the storage.
-        if self
-            .try_borrow_storage_mut()?
-            .has_with_host(&key, self, Some(k))?
-        {
+        if self.try_borrow_storage_mut()?.has(&key, self, Some(k))? {
             let (current, live_until_ledger) = self
                 .try_borrow_storage_mut()?
                 .get_with_live_until_ledger(&key, self, Some(k))?;
@@ -506,7 +491,7 @@ impl Host {
                     ));
                 }
             }
-            self.try_borrow_storage_mut()?.put_with_host(
+            self.try_borrow_storage_mut()?.put(
                 &key,
                 &Rc::metered_new(current, self)?,
                 live_until_ledger,
@@ -521,7 +506,7 @@ impl Host {
                 durability,
                 ext: ExtensionPoint::V0,
             };
-            self.try_borrow_storage_mut()?.put_with_host(
+            self.try_borrow_storage_mut()?.put(
                 &key,
                 &Host::new_contract_data(self, data)?,
                 Some(self.get_min_live_until_ledger(durability)?),
@@ -537,20 +522,36 @@ impl Host {
 #[cfg(any(test, feature = "testutils"))]
 use crate::crypto;
 #[cfg(any(test, feature = "testutils"))]
-use crate::storage::{AccessType, Footprint};
+use crate::storage::{AccessType, EntryWithLiveUntil, Footprint};
 
 #[cfg(any(test, feature = "testutils"))]
 impl Host {
-    // Writes an arbitrary ledger entry to storage.
+    /// Writes an arbitrary ledger entry to storage.
     pub fn add_ledger_entry(
         &self,
         key: &Rc<LedgerKey>,
         val: &Rc<soroban_env_common::xdr::LedgerEntry>,
         live_until_ledger: Option<u32>,
     ) -> Result<(), HostError> {
-        self.with_mut_storage(|storage| {
-            storage.put_with_host(key, val, live_until_ledger, self, None)
-        })
+        self.with_mut_storage(|storage| storage.put(key, val, live_until_ledger, self, None))
+    }
+
+    /// Reads an arbitrary ledger entry from the storage.
+    ///
+    /// Returns `None` if the entry does not exist.
+    pub fn get_ledger_entry(
+        &self,
+        key: &Rc<LedgerKey>,
+    ) -> Result<Option<EntryWithLiveUntil>, HostError> {
+        self.with_mut_storage(|storage| storage.try_get_full(key, self, None))
+    }
+
+    /// Returns all the ledger entries stored in the storage as key-value pairs.
+    #[allow(clippy::type_complexity)]
+    pub fn get_stored_entries(
+        &self,
+    ) -> Result<Vec<(Rc<LedgerKey>, Option<EntryWithLiveUntil>)>, HostError> {
+        self.with_mut_storage(|storage| Ok(storage.map.map.clone()))
     }
 
     // Performs the necessary setup to access the provided ledger key/entry in

soroban-env-host/src/host/frame.rs

@@ -697,10 +697,7 @@ impl Host {
             // checking the cache: this seems like overkill but it
             // provides some future-proofing, see below.
             let wasm_key = self.contract_code_ledger_key(wasm_hash)?;
-            if self
-                .try_borrow_storage_mut()?
-                .has_with_host(&wasm_key, self, None)?
-            {
+            if self.try_borrow_storage_mut()?.has(&wasm_key, self, None)? {
                 if let Some(parsed_module) = cache.get_module(wasm_hash)? {
                     return Vm::from_parsed_module_and_wasmi_linker(
                         self,

soroban-env-host/src/host/invocation_metering.rs

@@ -323,7 +323,7 @@ impl InvocationMeter {
             let mut entry_size = 0;
             let mut new_entry_size_for_rent = 0;
             let mut entry_live_until_ledger = None;
-            let maybe_entry = curr_storage.try_get_full_with_host(key, host, None)?;
+            let maybe_entry = curr_storage.try_get_full(key, host, None)?;
             if let Some((entry, entry_live_until)) = maybe_entry {
                 let mut buf = Vec::<u8>::new();
                 metered_write_xdr(host.budget_ref(), entry.as_ref(), &mut buf)?;