api: Add list_transports_ex() and set_transport_unpublished() functions

Closes #7980.

Unpublished transports are not advertised to contacts, and self-sent messages are not sent there, so that we don't cause extra messages to the corresponding inbox, but can still receive messages from contacts who don't know the new relay addresses yet.

- This adds `list_transports_ex()` and `set_transport_unpublished()` JsonRPC functions
- By default, transports are published, but when updating, all existing transports except for the primary one become unpublished in order not to break existing users that followed https://delta.chat/legacy-move
- It is not possible to unpublish the primary transport, and setting a transport as primary automatically sets it to published

An alternative would be to change the existing list_transports API rather than adding a new one list_transports_ex. But to be honest, I don't mind the _ex prefix that much, and I am wary about compatibility issues. But maybe it would be fine; see b08ba4b for how this would look.

Author: Hocuri

deltachat-ffi/deltachat.h

@@ -6706,8 +6706,8 @@ void dc_event_unref(dc_event_t* event);
  * UI should update the list.
  *
  * The event is emitted when the transports are modified on another device
- * using the JSON-RPC calls `add_or_update_transport`, `add_transport_from_qr`, `delete_transport`
- * or `set_config(configured_addr)`.
+ * using the JSON-RPC calls `add_or_update_transport`, `add_transport_from_qr`, `delete_transport`,
+ * `set_transport_unpublished` or `set_config(configured_addr)`.
  */
 #define DC_EVENT_TRANSPORTS_MODIFIED           2600
 

deltachat-jsonrpc/src/api.rs

@@ -68,6 +68,7 @@ use self::types::{
     },
 };
 use crate::api::types::chat_list::{get_chat_list_item_by_id, ChatListItemFetchResult};
+use crate::api::types::login_param::Transport;
 use crate::api::types::qr::{QrObject, SecurejoinSource, SecurejoinUiPath};
 
 #[derive(Debug)]
@@ -528,6 +529,7 @@ impl CommandApi {
     ///   from a server encoded in a QR code.
     /// - [Self::list_transports()] to get a list of all configured transports.
     /// - [Self::delete_transport()] to remove a transport.
+    /// - [Self::set_transport_unpublished()] to set whether contacts see this transport.
     async fn add_or_update_transport(
         &self,
         account_id: u32,
@@ -553,7 +555,23 @@ impl CommandApi {
     /// Returns the list of all email accounts that are used as a transport in the current profile.
     /// Use [Self::add_or_update_transport()] to add or change a transport
     /// and [Self::delete_transport()] to delete a transport.
+    /// Use [Self::list_transports_ex()] to additionally query
+    /// whether the transports are marked as 'unpublished'.
     async fn list_transports(&self, account_id: u32) -> Result<Vec<EnteredLoginParam>> {
+        let ctx = self.get_context(account_id).await?;
+        let res = ctx
+            .list_transports()
+            .await?
+            .into_iter()
+            .map(|t| t.param.into())
+            .collect();
+        Ok(res)
+    }
+
+    /// Returns the list of all email accounts that are used as a transport in the current profile.
+    /// Use [Self::add_or_update_transport()] to add or change a transport
+    /// and [Self::delete_transport()] to delete a transport.
+    async fn list_transports_ex(&self, account_id: u32) -> Result<Vec<Transport>> {
         let ctx = self.get_context(account_id).await?;
         let res = ctx
             .list_transports()
@@ -571,6 +589,26 @@ impl CommandApi {
         ctx.delete_transport(&addr).await
     }
 
+    /// Change whether the transport is unpublished.
+    ///
+    /// Unpublished transports are not advertised to contacts,
+    /// and self-sent messages are not sent there,
+    /// so that we don't cause extra messages to the corresponding inbox,
+    /// but can still receive messages from contacts who don't know our new transport addresses yet.
+    ///
+    /// The default is false, but when the user updates from a version that didn't have this flag,
+    /// existing secondary transports are set to unpublished,
+    /// so that an existing transport address doesn't suddenly get spammed with a lot of messages.
+    async fn set_transport_unpublished(
+        &self,
+        account_id: u32,
+        addr: String,
+        unpublished: bool,
+    ) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        ctx.set_transport_unpublished(&addr, unpublished).await
+    }
+
     /// Signal an ongoing process to stop.
     async fn stop_ongoing_process(&self, account_id: u32) -> Result<()> {
         let ctx = self.get_context(account_id).await?;

deltachat-jsonrpc/src/api/types/login_param.rs

@@ -4,6 +4,16 @@ use serde::Deserialize;
 use serde::Serialize;
 use yerpc::TypeDef;
 
+#[derive(Serialize, TypeDef, schemars::JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct Transport {
+    /// The login data entered by the user.
+    pub param: EnteredLoginParam,
+    /// Whether this transport is set to 'unpublished'.
+    /// See `set_transport_unpublished` / `setTransportUnpublished` for details.
+    pub is_unpublished: bool,
+}
+
 /// Login parameters entered by the user.
 ///
 /// Usually it will be enough to only set `addr` and `password`,
@@ -56,6 +66,15 @@ pub struct EnteredLoginParam {
     pub oauth2: Option<bool>,
 }
 
+impl From<dc::Transport> for Transport {
+    fn from(transport: dc::Transport) -> Self {
+        Transport {
+            param: transport.param.into(),
+            is_unpublished: transport.is_unpublished,
+        }
+    }
+}
+
 impl From<dc::EnteredLoginParam> for EnteredLoginParam {
     fn from(param: dc::EnteredLoginParam) -> Self {
         let imap_security: Socket = param.imap.security.into();

src/chat.rs

@@ -2844,17 +2844,12 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) -
     let lowercase_from = from.to_lowercase();
 
     recipients.retain(|x| x.to_lowercase() != lowercase_from);
-    if context.get_config_bool(Config::BccSelf).await? {
-        smtp::add_self_recipients(context, &mut recipients, needs_encryption).await?;
-    }
-
-    // Default Webxdc integrations are hidden messages and must not be sent out
-    if msg.param.get_int(Param::WebxdcIntegration).is_some() && msg.hidden {
-        recipients.clear();
-    }
 
-    if recipients.is_empty() {
-        // may happen eg. for groups with only SELF and bcc_self disabled
+    // Default Webxdc integrations are hidden messages and must not be sent out:
+    if (msg.param.get_int(Param::WebxdcIntegration).is_some() && msg.hidden)
+        // This may happen eg. for groups with only SELF and bcc_self disabled:
+        || (!context.get_config_bool(Config::BccSelf).await? && recipients.is_empty())
+    {
         info!(
             context,
             "Message {} has no recipient, skipping smtp-send.", msg.id
@@ -2893,6 +2888,10 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) -
         );
     }
 
+    if context.get_config_bool(Config::BccSelf).await? {
+        smtp::add_self_recipients(context, &mut recipients, rendered_msg.is_encrypted).await?;
+    }
+
     if needs_encryption && !rendered_msg.is_encrypted {
         /* unrecoverable */
         message::set_msg_failed(

src/config.rs

@@ -837,7 +837,7 @@ impl Context {
                             // which only fetches from the primary transport.
                             transaction
                                 .execute(
-                                    "UPDATE transports SET add_timestamp=? WHERE addr=?",
+                                    "UPDATE transports SET add_timestamp=?, is_published=1 WHERE addr=?",
                                     (time(), addr),
                                 )
                                 .context(
@@ -964,7 +964,22 @@ impl Context {
     pub(crate) async fn get_all_self_addrs(&self) -> Result<Vec<String>> {
         self.sql
             .query_map_vec(
-                "SELECT addr FROM transports ORDER BY add_timestamp DESC",
+                "SELECT addr FROM transports ORDER BY add_timestamp, id DESC",
+                (),
+                |row| {
+                    let addr: String = row.get(0)?;
+                    Ok(addr)
+                },
+            )
+            .await
+    }
+
+    /// Returns all published self addresses, newest first.
+    /// See `[Context::set_transport_unpublished]`
+    pub(crate) async fn get_published_self_addrs(&self) -> Result<Vec<String>> {
+        self.sql
+            .query_map_vec(
+                "SELECT addr FROM transports WHERE is_published=1 ORDER BY add_timestamp, id DESC",
                 (),
                 |row| {
                     let addr: String = row.get(0)?;
@@ -982,6 +997,24 @@ impl Context {
         }).await
     }
 
+    /// Returns all published secondary self addresses.
+    /// See `[Context::set_transport_unpublished]`
+    pub(crate) async fn get_published_secondary_self_addrs(&self) -> Result<Vec<String>> {
+        self.sql
+            .query_map_vec(
+                "SELECT addr FROM transports
+                WHERE is_published
+                AND addr NOT IN (SELECT value FROM config WHERE keyname='configured_addr')
+                ORDER BY add_timestamp, id DESC",
+                (),
+                |row| {
+                    let addr: String = row.get(0)?;
+                    Ok(addr)
+                },
+            )
+            .await
+    }
+
     /// Returns the primary self address.
     /// Returns an error if no self addr is configured.
     pub async fn get_primary_self_addr(&self) -> Result<String> {

src/configure.rs

@@ -28,8 +28,8 @@ use crate::constants::NON_ALPHANUMERIC_WITHOUT_DOT;
 use crate::context::Context;
 use crate::imap::Imap;
 use crate::log::warn;
-use crate::login_param::EnteredCertificateChecks;
 pub use crate::login_param::EnteredLoginParam;
+use crate::login_param::{EnteredCertificateChecks, Transport};
 use crate::message::Message;
 use crate::net::proxy::ProxyConfig;
 use crate::oauth2::get_oauth2_addr;
@@ -110,6 +110,7 @@ impl Context {
     ///   from a server encoded in a QR code.
     /// - [Self::list_transports()] to get a list of all configured transports.
     /// - [Self::delete_transport()] to remove a transport.
+    /// - [Self::set_transport_unpublished()] to set whether contacts see this transport.
     pub async fn add_or_update_transport(&self, param: &mut EnteredLoginParam) -> Result<()> {
         self.stop_io().await;
         let result = self.add_transport_inner(param).await;
@@ -188,14 +189,22 @@ impl Context {
     /// Returns the list of all email accounts that are used as a transport in the current profile.
     /// Use [Self::add_or_update_transport()] to add or change a transport
     /// and [Self::delete_transport()] to delete a transport.
-    pub async fn list_transports(&self) -> Result<Vec<EnteredLoginParam>> {
+    pub async fn list_transports(&self) -> Result<Vec<Transport>> {
         let transports = self
             .sql
-            .query_map_vec("SELECT entered_param FROM transports", (), |row| {
-                let entered_param: String = row.get(0)?;
-                let transport: EnteredLoginParam = serde_json::from_str(&entered_param)?;
-                Ok(transport)
-            })
+            .query_map_vec(
+                "SELECT entered_param, is_published FROM transports",
+                (),
+                |row| {
+                    let param: String = row.get(0)?;
+                    let param: EnteredLoginParam = serde_json::from_str(&param)?;
+                    let is_published: bool = row.get(1)?;
+                    Ok(Transport {
+                        param,
+                        is_unpublished: !is_published,
+                    })
+                },
+            )
             .await?;
 
         Ok(transports)
@@ -261,6 +270,44 @@ impl Context {
         Ok(())
     }
 
+    /// Change whether the transport is unpublished.
+    ///
+    /// Unpublished transports are not advertised to contacts,
+    /// and self-sent messages are not sent there,
+    /// so that we don't cause extra messages to the corresponding inbox,
+    /// but can still receive messages from contacts who don't know our new transport addresses yet.
+    ///
+    /// The default is false, but when the user updates from a version that didn't have this flag,
+    /// existing secondary transports are set to unpublished,
+    /// so that an existing transport address doesn't suddenly get spammed with a lot of messages.
+    pub async fn set_transport_unpublished(&self, addr: &str, unpublished: bool) -> Result<()> {
+        self.sql
+            .transaction(|trans| {
+                let primary_addr: String = trans
+                    .query_row(
+                        "SELECT value FROM config WHERE keyname='configured_addr'",
+                        (),
+                        |row| row.get(0),
+                    )
+                    .context("Select primary address")?;
+                if primary_addr == addr && unpublished {
+                    bail!("Can't set primary relay as unpublished");
+                }
+                // We need to update the timestamp so that the key's timestamp changes
+                // and is recognized as newer by our peers
+                trans
+                    .execute(
+                        "UPDATE transports SET is_published=?, add_timestamp=? WHERE addr=? AND is_published!=?1",
+                        (!unpublished, time(), addr),
+                    )
+                    .context("Update transports")?;
+                Ok(())
+            })
+            .await?;
+        send_sync_transports(self).await?;
+        Ok(())
+    }
+
     async fn inner_configure(&self, param: &EnteredLoginParam) -> Result<()> {
         info!(self, "Configure ...");
 

src/key.rs

@@ -296,7 +296,7 @@ pub(crate) async fn load_self_public_key_opt(context: &Context) -> Result<Option
         .await?
         .context("No transports configured")?;
     let addr = context.get_primary_self_addr().await?;
-    let all_addrs = context.get_all_self_addrs().await?.join(",");
+    let all_addrs = context.get_published_self_addrs().await?.join(",");
     let signed_public_key =
         secret_key_to_public_key(context, signed_secret_key, timestamp, &addr, &all_addrs)?;
     *lock = Some(signed_public_key.clone());

src/login_param.rs

@@ -79,6 +79,16 @@ pub struct EnteredServerLoginParam {
     pub password: String,
 }
 
+/// A transport, as shown in the "relays" list in the UI.
+#[derive(Debug)]
+pub struct Transport {
+    /// The login data entered by the user.
+    pub param: EnteredLoginParam,
+    /// Whether this transport is set to 'unpublished'.
+    /// See [`Context::set_transport_unpublished`] for details.
+    pub is_unpublished: bool,
+}
+
 /// Login parameters entered by the user.
 #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub struct EnteredLoginParam {

src/mimeparser/shared_secret_decryption_tests.rs

@@ -204,7 +204,7 @@ async fn test_qr_code_security() -> Result<()> {
     let charlie_addr = charlie.get_config(Config::Addr).await?.unwrap();
 
     let alice_fp = self_fingerprint(alice).await?;
-    let secret_for_encryption = dbg!(format!("securejoin/{alice_fp}/{authcode}"));
+    let secret_for_encryption = format!("securejoin/{alice_fp}/{authcode}");
     test_shared_secret_decryption_ex(
         bob,
         &charlie_addr,

src/smtp.rs

@@ -706,7 +706,7 @@ pub(crate) async fn add_self_recipients(
         // them. Normally the user should have a non-chatmail primary transport to send unencrypted
         // messages.
         if encrypted {
-            for addr in context.get_secondary_self_addrs().await? {
+            for addr in context.get_published_secondary_self_addrs().await? {
                 recipients.push(addr);
             }
         }