Unify OsslAsymcipher for all asymmetric cipher ops

This patch refactors the `OsslAsymcipher` structure to provide a unified and
comprehensive interface for all asymmetric cipher operations, including
**encryption**, **decryption**, **encapsulation**, and **decapsulation**.

Previously, encapsulation and decapsulation operations were handled by a
separate `OsslEncapsulation` structure. This patch moves the operations
previously performed by `OsslEncapsulation` directly into `OsslAsymcipher`.

Key changes include:
* The `EncOp` enum now includes `Encapsulate` and `Decapsulate` variants.
* The `OsslAsymcipher::new` method is enhanced to handle initialization for
  all four operation types, taking an `Option<&OsslParam>` for parameters to
  support cases where parameters might be null.
* The `OsslEncapsulation` structure and its associated `new_encapsulation`
  and `new_decapsulation` methods have been **removed**.
* The `encapsulate` and `decapsulate` methods of `OsslAsymcipher` now include
  checks to ensure the `EncOp` matches the intended operation, returning an
  `WrapperError` if there's a mismatch.

Signed-off-by: Simo Sorce <simo@redhat.com>
Co-authored-by: Gemini <gemini@google.com>

Author: simo5

ossl/src/asymcipher.rs

@@ -76,9 +76,17 @@ pub fn rsa_enc_params(
 pub enum EncOp {
     Encrypt,
     Decrypt,
+    Encapsulate,
+    Decapsulate,
 }
 
-/// Higher level wrapper for asymmetric encryption operations with OpenSSL
+/// Higher level wrapper for asymmetric cipher operations with OpenSSL
+///
+/// Supports Encryption, Decryption, Encapsulation, Decapsulation.
+/// Whether any of these operation will work also depends on the type
+/// of key provided.
+///
+/// An OsslError is returned if an operation is unsupported.
 #[derive(Debug)]
 pub struct OsslAsymcipher {
     /// The underlying OpenSSL EVP PKEY context.
@@ -93,24 +101,28 @@ impl OsslAsymcipher {
         libctx: &OsslContext,
         op: EncOp,
         key: &mut EvpPkey,
-        params: &OsslParam,
+        params: Option<&OsslParam>,
     ) -> Result<OsslAsymcipher, Error> {
         let mut ctx = OsslAsymcipher {
             pkey_ctx: key.new_ctx(libctx)?,
             op: op,
         };
+        let params_ptr = match params {
+            Some(p) => p.as_ptr(),
+            None => std::ptr::null(),
+        };
         let ret = match ctx.op {
             EncOp::Encrypt => unsafe {
-                EVP_PKEY_encrypt_init_ex(
-                    ctx.pkey_ctx.as_mut_ptr(),
-                    params.as_ptr(),
-                )
+                EVP_PKEY_encrypt_init_ex(ctx.pkey_ctx.as_mut_ptr(), params_ptr)
             },
             EncOp::Decrypt => unsafe {
-                EVP_PKEY_decrypt_init_ex(
-                    ctx.pkey_ctx.as_mut_ptr(),
-                    params.as_ptr(),
-                )
+                EVP_PKEY_decrypt_init_ex(ctx.pkey_ctx.as_mut_ptr(), params_ptr)
+            },
+            EncOp::Encapsulate => unsafe {
+                EVP_PKEY_encapsulate_init(ctx.pkey_ctx.as_mut_ptr(), params_ptr)
+            },
+            EncOp::Decapsulate => unsafe {
+                EVP_PKEY_decapsulate_init(ctx.pkey_ctx.as_mut_ptr(), params_ptr)
             },
         };
         if ret != 1 {
@@ -121,8 +133,13 @@ impl OsslAsymcipher {
                 EncOp::Decrypt => {
                     trace_ossl!("EVP_PKEY_decrypt_init()");
                 }
+                EncOp::Encapsulate => {
+                    trace_ossl!("EVP_PKEY_encapsulate_init()");
+                }
+                EncOp::Decapsulate => {
+                    trace_ossl!("EVP_PKEY_decapsulate_init()");
+                }
             }
-            trace_ossl!("EVP_PKEY_encrypt_init()");
             return Err(Error::new(ErrorKind::OsslError));
         }
         Ok(ctx)
@@ -191,36 +208,6 @@ impl OsslAsymcipher {
         }
         Ok(outlen)
     }
-}
-
-/// Higher level wrapper for asymmetric encapsulation operations with OpenSSL
-#[derive(Debug)]
-pub struct OsslEncapsulation {
-    /// The underlying OpenSSL EVP PKEY context.
-    pkey_ctx: EvpPkeyCtx,
-}
-
-impl OsslEncapsulation {
-    /// Initializes a new encapsulation operation
-    pub fn new_encapsulation(
-        libctx: &OsslContext,
-        key: &mut EvpPkey,
-    ) -> Result<OsslEncapsulation, Error> {
-        let mut ctx = OsslEncapsulation {
-            pkey_ctx: key.new_ctx(libctx)?,
-        };
-        let ret = unsafe {
-            EVP_PKEY_encapsulate_init(
-                ctx.pkey_ctx.as_mut_ptr(),
-                std::ptr::null_mut(),
-            )
-        };
-        if ret != 1 {
-            trace_ossl!("EVP_PKEY_encapsulate_init()");
-            return Err(Error::new(ErrorKind::OsslError));
-        }
-        Ok(ctx)
-    }
 
     /// Encapsulates operation, returns ciphertext in the mutable slice and
     /// returns an encapsulated key as a vector as well as the actual size
@@ -229,6 +216,9 @@ impl OsslEncapsulation {
         &mut self,
         ciphertext: &mut [u8],
     ) -> Result<(Vec<u8>, usize), Error> {
+        if self.op != EncOp::Encapsulate {
+            return Err(Error::new(ErrorKind::WrapperError));
+        }
         let mut outlen = 0;
         let mut keylen = 0;
 
@@ -269,30 +259,12 @@ impl OsslEncapsulation {
         Ok((keydata, outlen))
     }
 
-    /// Initializes a new decapsulation operation
-    pub fn new_decapsulation(
-        libctx: &OsslContext,
-        key: &mut EvpPkey,
-    ) -> Result<OsslEncapsulation, Error> {
-        let mut ctx = OsslEncapsulation {
-            pkey_ctx: key.new_ctx(libctx)?,
-        };
-        let ret = unsafe {
-            EVP_PKEY_decapsulate_init(
-                ctx.pkey_ctx.as_mut_ptr(),
-                std::ptr::null_mut(),
-            )
-        };
-        if ret != 1 {
-            trace_ossl!("EVP_PKEY_decapsulate_init()");
-            return Err(Error::new(ErrorKind::OsslError));
-        }
-        Ok(ctx)
-    }
-
     /// Decapsulate operation, takes the ciphertext generated by the peer and
     /// returns an encapsulated key as a vector
     pub fn decapsulate(&mut self, ciphertext: &[u8]) -> Result<Vec<u8>, Error> {
+        if self.op != EncOp::Decapsulate {
+            return Err(Error::new(ErrorKind::WrapperError));
+        }
         let mut keylen = 0;
 
         let ret = unsafe {

src/ossl/mlkem.rs

@@ -11,7 +11,7 @@ use crate::object::Object;
 use crate::ossl::common::{osslctx, privkey_from_object, pubkey_from_object};
 use crate::pkcs11::*;
 
-use ossl::asymcipher::OsslEncapsulation;
+use ossl::asymcipher::{EncOp, OsslAsymcipher};
 use ossl::pkey::{EvpPkey, EvpPkeyType, MlkeyData, PkeyData};
 use ossl::ErrorKind;
 
@@ -72,8 +72,6 @@ pub fn mlkem_object_to_pkey(
 /// Performs the ML-KEM key encapsulation operation using the recipient's
 /// public key.
 ///
-/// Uses the `OsslEncapsulation` API.
-///
 /// Returns a tuple containing the derived shared secret (`Vec<u8>`) and
 /// the actual length of the generated ciphertext written to the `ciphertext`
 /// buffer.
@@ -82,7 +80,8 @@ pub fn encapsulate(
     ciphertext: &mut [u8],
 ) -> Result<(Vec<u8>, usize)> {
     let mut pubkey = pubkey_from_object(key)?;
-    let mut ctx = OsslEncapsulation::new_encapsulation(osslctx(), &mut pubkey)?;
+    let mut ctx =
+        OsslAsymcipher::new(osslctx(), EncOp::Encapsulate, &mut pubkey, None)?;
     match ctx.encapsulate(ciphertext) {
         Ok(ret) => Ok(ret),
         Err(e) => match e.kind() {
@@ -100,7 +99,8 @@ pub fn encapsulate(
 /// Returns the derived shared secret (`Vec<u8>`).
 pub fn decapsulate(key: &Object, ciphertext: &[u8]) -> Result<Vec<u8>> {
     let mut prikey = privkey_from_object(key)?;
-    let mut ctx = OsslEncapsulation::new_decapsulation(osslctx(), &mut prikey)?;
+    let mut ctx =
+        OsslAsymcipher::new(osslctx(), EncOp::Decapsulate, &mut prikey, None)?;
     Ok(ctx.decapsulate(ciphertext)?)
 }
 

src/ossl/rsa.rs

@@ -290,7 +290,7 @@ impl RsaPKCSOperation {
             osslctx(),
             op,
             &mut pkey,
-            &rsa_enc_params(alg, params.as_ref())?,
+            Some(&rsa_enc_params(alg, params.as_ref())?),
         )?;
         let keysize = Self::get_key_size(key, info)?;
         let maxinput = Self::max_message_len(keysize, mech)?;