feat: add various new APIs for Lotus (#89)

The new APIs are:

 - decoding a range of an updated empty sector
 - generating TreeRLast
 - generating TreeC
 - running SDR

Author: vmx

src/seal.rs

@@ -11,7 +11,7 @@ use filecoin_hashers::Hasher;
 use filecoin_proofs_v1::constants::{
     SectorShape16KiB, SectorShape16MiB, SectorShape1GiB, SectorShape2KiB, SectorShape32GiB,
     SectorShape32KiB, SectorShape4KiB, SectorShape512MiB, SectorShape64GiB, SectorShape8MiB,
-    SECTOR_SIZE_16_KIB, SECTOR_SIZE_16_MIB, SECTOR_SIZE_1_GIB, SECTOR_SIZE_2_KIB,
+    LAYERS, SECTOR_SIZE_16_KIB, SECTOR_SIZE_16_MIB, SECTOR_SIZE_1_GIB, SECTOR_SIZE_2_KIB,
     SECTOR_SIZE_32_GIB, SECTOR_SIZE_32_KIB, SECTOR_SIZE_4_KIB, SECTOR_SIZE_512_MIB,
     SECTOR_SIZE_64_GIB, SECTOR_SIZE_8_MIB,
 };
@@ -402,6 +402,46 @@ fn seal_pre_commit_phase1_inner<Tree: 'static + MerkleTreeTrait>(
     })
 }
 
+/// Generate label layers (SDR).
+///
+/// # Arguments
+/// * `registered_proof` - Selected seal operation.
+/// * `cache_path` - Directory path to use for generation of Merkle tree on disk.
+/// * `output_dir` - Directory where the TreeRLast(s) are stored.
+pub fn sdr<R>(
+    registered_proof: RegisteredSealProof,
+    cache_path: R,
+    replica_id: <filecoin_proofs_v1::constants::DefaultTreeHasher as Hasher>::Domain,
+) -> Result<()>
+where
+    R: AsRef<Path>,
+{
+    ensure!(
+        registered_proof.major_version() == 1,
+        "unusupported version"
+    );
+
+    with_shape!(
+        u64::from(registered_proof.sector_size()),
+        sdr_inner,
+        registered_proof,
+        cache_path.as_ref(),
+        replica_id,
+    )?;
+
+    Ok(())
+}
+
+fn sdr_inner<Tree: 'static + MerkleTreeTrait>(
+    registered_proof: RegisteredSealProof,
+    cache_path: &Path,
+    replica_id: <Tree::Hasher as Hasher>::Domain,
+) -> Result<()> {
+    let config = registered_proof.as_v1_config();
+    filecoin_proofs_v1::sdr::<_, Tree>(&config, cache_path, &replica_id)?;
+    Ok(())
+}
+
 /// Second phase of seal precommit operation, must be called with output of
 /// [`seal_pre_commit_phase1`]. Generates `comm_r` replica commitment from outputs
 /// of previous step.
@@ -477,6 +517,96 @@ fn seal_pre_commit_phase2_inner<Tree: 'static + MerkleTreeTrait>(
     })
 }
 
+/// Generate Merkle tree for sector replica (TreeRLast) and return the root hash (CommRLast).
+///
+/// # Arguments
+/// * `registered_proof` - Selected seal operation.
+/// * `replica_path` - File path of replica.
+/// * `output_dir` - Directory where the TreeRLast(s) are stored.
+pub fn generate_tree_r_last<O, R>(
+    registered_proof: RegisteredSealProof,
+    replica_path: R,
+    output_dir: O,
+) -> Result<Commitment>
+where
+    O: AsRef<Path>,
+    R: AsRef<Path>,
+{
+    ensure!(
+        registered_proof.major_version() == 1,
+        "unusupported version"
+    );
+
+    let sector_size = u64::from(registered_proof.sector_size());
+    let comm_r_last = with_shape!(
+        sector_size,
+        generate_tree_r_last_inner,
+        sector_size,
+        replica_path.as_ref(),
+        output_dir.as_ref(),
+    )?;
+
+    Ok(comm_r_last.into())
+}
+
+fn generate_tree_r_last_inner<Tree: 'static + MerkleTreeTrait>(
+    sector_size: u64,
+    replica_path: &Path,
+    output_dir: &Path,
+) -> Result<<Tree::Hasher as Hasher>::Domain> {
+    filecoin_proofs_v1::generate_tree_r_last::<_, _, Tree>(sector_size, &replica_path, &output_dir)
+}
+
+/// Generate Merkle tree for the label layers (TreeC) and return the root hash (CommC).
+///
+/// # Arguments
+/// * `registered_proof` - Selected seal operation.
+/// * `input_dir` - Directory where the label layers are stored.
+/// * `output_dir` - Directory where the TreeRLast(s) are stored.
+pub fn generate_tree_c<O, R>(
+    registered_proof: RegisteredSealProof,
+    input_dir: R,
+    output_dir: O,
+) -> Result<Commitment>
+where
+    O: AsRef<Path>,
+    R: AsRef<Path>,
+{
+    ensure!(
+        registered_proof.major_version() == 1,
+        "unusupported version"
+    );
+
+    let sector_size = u64::from(registered_proof.sector_size());
+    let comm_c = with_shape!(
+        sector_size,
+        generate_tree_c_inner,
+        sector_size,
+        input_dir.as_ref(),
+        output_dir.as_ref(),
+    )?;
+
+    Ok(comm_c.into())
+}
+
+fn generate_tree_c_inner<Tree: 'static + MerkleTreeTrait>(
+    sector_size: u64,
+    input_dir: &Path,
+    output_dir: &Path,
+) -> Result<<Tree::Hasher as Hasher>::Domain> {
+    let num_layers = *LAYERS
+        .read()
+        .expect("LAYERS poisoned")
+        .get(&sector_size)
+        .expect("unknown sector size");
+    filecoin_proofs_v1::generate_tree_c::<_, _, Tree>(
+        sector_size,
+        &input_dir,
+        &output_dir,
+        num_layers,
+    )
+}
+
 /// Computes a sectors's `comm_d` data commitment given its pieces.
 ///
 /// # Arguments

src/update.rs

@@ -1,4 +1,5 @@
 //! Update data within existing sealed sectors.
+use std::io::{Read, Write};
 use std::path::Path;
 
 use anyhow::{ensure, Result};
@@ -149,6 +150,58 @@ where
     )
 }
 
+/// Reverses the encoding process for a certain range.
+///
+/// This function is similar to [`emptu_sector_update_decode_from`], the difference is that it
+/// operates directly on the given file descriptions. The current position of the file descriptors
+/// is where the decoding starts, i.e. you need to seek to the intended offset before you call this
+/// funtion.
+///
+/// # Arguments
+/// * `registered_proof` - Selected sector update proof.
+/// * `nodes_count` - Total number of nodes within the file.
+/// * `comm_d` - Data commitment from the updated replica.
+/// * `comm_r` - Replica commitment of the empty sector.
+/// * `input_data` - File descriptor to the encoded data.
+/// * `sector_key_data` - File descriptor to replica of the empty sector.
+/// * `output_data` - File descriptor where the decoded data is written to.
+/// * `nodes_offset` - Node offset relative to the beginning of the file.
+/// * `num_nodes` - Number of nodes to be decoded starting at the current position.
+pub fn empty_sector_update_decode_from_range<R, S, W>(
+    registered_proof: RegisteredUpdateProof,
+    comm_d: Commitment,
+    comm_r: Commitment,
+    input_data: R,
+    sector_key_data: S,
+    output_data: &mut W,
+    nodes_offset: usize,
+    num_nodes: usize,
+) -> Result<()>
+where
+    R: Read,
+    S: Read,
+    W: Write,
+{
+    ensure!(
+        registered_proof.major_version() == 1,
+        "unusupported version"
+    );
+
+    let config = registered_proof.as_v1_config();
+    let update_config = SectorUpdateConfig::from_porep_config(&config);
+
+    filecoin_proofs_v1::decode_from_range(
+        update_config.nodes_count,
+        comm_d,
+        comm_r,
+        input_data,
+        sector_key_data,
+        output_data,
+        nodes_offset,
+        num_nodes,
+    )
+}
+
 fn empty_sector_update_remove_encoded_data_inner<
     Tree: 'static + MerkleTreeTrait<Hasher = TreeRHasher>,
 >(