refactor: significantly improve availability

Author: chadsec1

.gitignore

@@ -2,5 +2,6 @@ __pycache__/
 account.coldwire
 *.swp
 *.swo
+*.swn
 TODO.md
 TODO.txt

CONTRIBUTING.md

@@ -40,7 +40,7 @@ Here are some features that we have decided against implementing after thoughtfu
 - Voice, and video calls.
 - Voice messages
 - Compression support
-- Metadata-rich features (avatars, vanity server-side usernames, bios, delievery receipts, read receipts, online status, last seen status, user-created server authentication passwords)
+- Metadata-rich features (avatars, vanity server-side usernames, about me, delievery receipts, read receipts, online status, last seen status, user-created server authentication passwords)
 - Account recovery
 - Persistent chat history
 - Any "convenience" features that could impact security and or privacy (clickable URLs, keyboard hotkeys, keyboard shortscuts beyond the basic CTRL-C CTRL-V) 

core/crypto.py

@@ -217,7 +217,7 @@ def decrypt_shared_secrets(ciphertext_blob: bytes, private_key: bytes, algorithm
         while len(shared_secrets) < size:
             ciphertext = ciphertext_blob[cursor:cursor + cipher_size]
             if len(ciphertext) != cipher_size:
-                 raise ValueError(f"Ciphertext of {algorithm} blob is malformed or incomplete ({len(ciphertext)})")
+                raise ValueError(f"Ciphertext of {algorithm} blob is malformed or incomplete ({len(ciphertext)})")
 
             shared_secret = kem.decap_secret(ciphertext)
             shared_secrets += shared_secret

core/trad_crypto.py

@@ -69,7 +69,7 @@ def derive_key_argon2id(password: bytes, salt: bytes = None, output_length: int
     ), salt
 
 
-def encrypt_xchacha20poly1305(key: bytes, plaintext: bytes, nonce: bytes = None, counter: int = None, counter_safety: int = 255, max_padding: int = XCHACHA20POLY1305_MAX_RANODM_PAD) -> tuple[bytes, bytes]:
+def encrypt_xchacha20poly1305(key: bytes, plaintext: bytes, nonce: bytes = None, max_padding: int = XCHACHA20POLY1305_MAX_RANODM_PAD) -> tuple[bytes, bytes]:
     """
     Encrypt plaintext using XChaCha20Poly1305.
 
@@ -79,8 +79,6 @@ def encrypt_xchacha20poly1305(key: bytes, plaintext: bytes, nonce: bytes = None,
         key: A 32-byte XChaCha20Poly1305 key.
         plaintext: Data to encrypt.
         nonce: An (optional) nonce to be used.
-        counter: an (optional) number to add to nonce
-        counter_safety: an (optional) max counter number, to prevent counter overflow.
         max_padding: an (optional) maximum padding limit number to message. Cannot be larger than what `XCHACHA20POLY1305_MAX_RANODM_PAD` could store. Set to 0 for no padding.
     Returns:
         A tuple (nonce, ciphertext) where:
@@ -90,12 +88,6 @@ def encrypt_xchacha20poly1305(key: bytes, plaintext: bytes, nonce: bytes = None,
     if nonce is None:
         nonce = sha3_512(secrets.token_bytes(XCHACHA20POLY1305_NONCE_LEN))[:XCHACHA20POLY1305_NONCE_LEN]
 
-    if counter is not None:
-        if counter > counter_safety:
-            raise ValueError("ChaCha counter has overflowen")
-
-        nonce = nonce[:XCHACHA20POLY1305_NONCE_LEN - 1] + counter.to_bytes(1, "big")
-
     if max_padding < 0:
         raise ValueError(f"Max_padding is less than 0! ({max_padding})")
 

logic/background_worker.py

@@ -10,9 +10,14 @@
     SMP_TYPES,
     PFS_TYPES,
     MSG_TYPES,
-    XCHACHA20POLY1305_NONCE_LEN
+    XCHACHA20POLY1305_NONCE_LEN,
+    ML_KEM_1024_NAME,
+    ML_KEM_1024_CT_LEN
+)
+from core.crypto import (
+        random_number_range,
+        decap_shared_secret
 )
-from core.crypto import random_number_range
 from core.trad_crypto import (
         decrypt_xchacha20poly1305
 )
@@ -114,48 +119,43 @@ def background_worker(user_data, user_data_lock, ui_queue, stop_flag):
                 acks["acks"].append(ack_id)
 
             with user_data_lock:
+                try:
+                    user_data["contacts"][sender]["locked"] = True
+                except Exception:
+                    pass
+
                 user_data_copied = copy.deepcopy(user_data)
 
             # Everything from here is not validated by server
 
             blob_plaintext = None
 
             if sender in user_data_copied["contacts"]: 
-                chacha_key = user_data["contacts"][sender]["lt_sign_key_smp"]["tmp_key"]
-                contact_next_strand_nonce = user_data["contacts"][sender]["contact_next_strand_nonce"]
 
-                if chacha_key is not None:
-                    chacha_key  = b64decode(user_data["contacts"][sender]["lt_sign_key_smp"]["tmp_key"])
+                contact_next_strand_key   = user_data_copied["contacts"][sender]["contact_next_strand_key"]
+                contact_next_strand_nonce = user_data_copied["contacts"][sender]["contact_next_strand_nonce"]
 
+                if contact_next_strand_key and contact_next_strand_nonce:
                     try:
-                        try:
-                            blob_plaintext = decrypt_xchacha20poly1305(chacha_key, blob[:XCHACHA20POLY1305_NONCE_LEN], blob[XCHACHA20POLY1305_NONCE_LEN:])
-                        except Exception as e:
-                            if contact_next_strand_nonce is None:
-                                raise Exception("Unable to decrypt apparent SMP request due to missing contact strand nonce.")
+                        blob_plaintext = decrypt_xchacha20poly1305(contact_next_strand_key, contact_next_strand_nonce, blob)
+
+                        contact_next_strand_key   = blob_plaintext[:32]
+                        contact_next_strand_nonce = blob_plaintext[32:32 + XCHACHA20POLY1305_NONCE_LEN]
+                        blob_plaintext = blob_plaintext[32 + XCHACHA20POLY1305_NONCE_LEN:]
 
-                            logger.debug("Failed to decrypt blob from contact (%s) probably due to invalid nonce: %s, we will try decrypting using strand nonce", sender, str(e))
-                            blob_plaintext = decrypt_xchacha20poly1305(chacha_key, contact_next_strand_nonce, blob)
+                        with user_data_lock:
+                            user_data["contacts"][sender]["contact_next_strand_key"]   = contact_next_strand_key
+                            user_data["contacts"][sender]["contact_next_strand_nonce"] = contact_next_strand_nonce
 
                     except Exception as e:
-                        logger.error("Failed to decrypt blob from contact (%s), we just going to treat blob as plaintext. Error: %s", sender, str(e))
-                        blob_plaintext = blob
+                        logger.error("Unable to decrypt incoming data from contact (%s). Error: %s", sender, str(e))
+                        
+                        continue
                 else:
-                    chacha_key = user_data["contacts"][sender]["contact_strand_key"]
-
-                    if (chacha_key is None) and (contact_next_strand_nonce is None):
-                        # just assume at this point that it's not encrypted.
-                        blob_plaintext = blob
-                    else:
-                        # Under known laws of physics, this should never fail. Unless the contact is acting funny on purpose / invalid implementation of Coldwire + strandlock protocol.
-                        try:
-                            blob_plaintext = decrypt_xchacha20poly1305(chacha_key, contact_next_strand_nonce, blob)
-                        except Exception as e:
-                            logger.error(
-                                    "Failed to decrypt blob from contact (%s)"
-                                    "We dont know what caused this except maybe a re-SMP verification. error: %s", sender, str(e)
-                                )
-                            blob_plaintext = blob
+                    # just assume at this point that it's not encrypted.
+                    logger.debug("Contact (%s) does not have a strand key and or a strand nonce! We will just gonna assume blob_plaintext = blob", sender)
+                    blob_plaintext = blob
+
             else:
                 logger.debug("Contact (%s) not saved.. we just gonna assume blob_plaintext = blob", sender)
                 blob_plaintext = blob
@@ -180,10 +180,9 @@ def background_worker(user_data, user_data_lock, ui_queue, stop_flag):
                         sender
                     )
 
-        # *Sigh* I had to put this here because if we rotate before finishing reading all of the messages
-        # we would overwrite our own key.
-        # TODO: We need to keep the last used key and use it when decapsulation with new key gives invalid output
-        # because it might actually take some time for our keys to be uploaded to server + other servers, and to the contact.
-        #
-        # update_ephemeral_keys(user_data, user_data_lock)
 
+            with user_data_lock:
+                try:
+                    user_data["contacts"][sender]["locked"] = False
+                except Exception:
+                    pass

logic/contacts.py

@@ -41,6 +41,7 @@ def save_contact(user_data: dict, user_data_lock, contact_id: str) -> None:
 
         user_data["contacts"][contact_id] = {
                 "nickname": None,
+                "locked": False,
                 "lt_sign_keys": {
                     "contact_public_key": None,
                     "our_keys": {
@@ -60,7 +61,6 @@ def save_contact(user_data: dict, user_data_lock, contact_id: str) -> None:
                     "contact_nonce": None,
                     "smp_step": None,
                     "tmp_proof": None,
-                    "tmp_key": None,
                     "contact_kem_public_key": None,
                     "our_kem_keys": {
                         "private_key": None,
@@ -97,8 +97,8 @@ def save_contact(user_data: dict, user_data_lock, contact_id: str) -> None:
 
                     }
                 },
-                "our_strand_key": None,
-                "contact_strand_key": None,
+                "our_next_strand_key": None,
+                "contact_next_strand_key": None,
                 "our_next_strand_nonce": None,
                 "contact_next_strand_nonce": None,
                 "our_pads": None,

logic/message.py

@@ -62,7 +62,7 @@ def generate_and_send_pads(user_data, user_data_lock, contact_id: str, ui_queue)
         contact_mceliece_public_key = user_data["contacts"][contact_id]["ephemeral_keys"]["contact_public_keys"][CLASSIC_MCELIECE_8_F_NAME]
         our_lt_private_key          = user_data["contacts"][contact_id]["lt_sign_keys"]["our_keys"]["private_key"]
 
-        our_strand_key = user_data["contacts"][contact_id]["our_strand_key"]
+        our_next_strand_key = user_data["contacts"][contact_id]["our_next_strand_key"]
 
         our_next_strand_nonce = user_data["contacts"][contact_id]["our_next_strand_nonce"]
 
@@ -78,10 +78,13 @@ def generate_and_send_pads(user_data, user_data_lock, contact_id: str, ui_queue)
 
     otp_batch_signature = create_signature(ML_DSA_87_NAME, kyber_ciphertext_blob + mceliece_ciphertext_blob, our_lt_private_key)
 
-    our_new_strand_nonce = sha3_512(secrets.token_bytes(XCHACHA20POLY1305_NONCE_LEN))[:XCHACHA20POLY1305_NONCE_LEN]
+    # Here, the strandkey is actually just added to make messages structure uniform and easier to process in implementations
+    # once contact receives this, he will save this new random key, then, process the batch, and save the new key derived from the batch.
+
+    new_strand_nonce = sha3_512(secrets.token_bytes(XCHACHA20POLY1305_NONCE_LEN))[:XCHACHA20POLY1305_NONCE_LEN]
     _, ciphertext_blob = encrypt_xchacha20poly1305(
-            our_strand_key, 
-            MSG_TYPES["MSG_BATCH"] + our_new_strand_nonce + otp_batch_signature + kyber_ciphertext_blob + mceliece_ciphertext_blob + xchacha_shared_secrets,
+            our_next_strand_key, 
+            sha3_512(secrets.token_bytes(32))[:32] + new_strand_nonce + MSG_TYPES["MSG_BATCH"] + otp_batch_signature + kyber_ciphertext_blob + mceliece_ciphertext_blob + xchacha_shared_secrets,
             nonce = our_next_strand_nonce
         )
 
@@ -98,17 +101,19 @@ def generate_and_send_pads(user_data, user_data_lock, contact_id: str, ui_queue)
         ui_queue.put({"type": "showerror", "title": "Error", "message": "Failed to send our one-time-pads key batch to the server"})
         return False
 
+    # XOR shared secrets together for hybrid encryption
     pads, _ = one_time_pad(kyber_shared_secrets, mceliece_shared_secrets)
     pads, _ = one_time_pad(pads, xchacha_shared_secrets)
 
-
-    our_strand_key = pads[:32]
+    # Derive key from pad + truncate it.
+    new_strand_key = pads[:32]
+    otp_pads       = pads[32:]
 
     # We update & save only at the end, so if request fails, we do not desync our state.
     with user_data_lock:
-        user_data["contacts"][contact_id]["our_next_strand_nonce"]  = our_new_strand_nonce 
-        user_data["contacts"][contact_id]["our_strand_key"]         = our_strand_key
-        user_data["contacts"][contact_id]["our_pads"]               = pads[32:]
+        user_data["contacts"][contact_id]["our_next_strand_nonce"] = new_strand_nonce 
+        user_data["contacts"][contact_id]["our_next_strand_key"]   = new_strand_key
+        user_data["contacts"][contact_id]["our_pads"]              = otp_pads
 
 
     save_account_data(user_data, user_data_lock)
@@ -139,6 +144,8 @@ def send_message_processor(user_data, user_data_lock, contact_id: str, message:
 
         our_pads = user_data["contacts"][contact_id]["our_pads"]
 
+        if user_data["contacts"][contact_id]["locked"]:
+            return
 
 
     if contact_kyber_public_key is None or contact_mceliece_public_key is None:
@@ -189,21 +196,23 @@ def send_message_processor(user_data, user_data_lock, contact_id: str, message:
     # because a malicious server could make our requests fail to force us to re-use the same pad for our next message 
     # which would break all of our security
 
-    our_new_strand_nonce = sha3_512(secrets.token_bytes(XCHACHA20POLY1305_NONCE_LEN))[:XCHACHA20POLY1305_NONCE_LEN]
+    new_strand_key   = sha3_512(secrets.token_bytes(32))[:32]
+    new_strand_nonce = sha3_512(secrets.token_bytes(XCHACHA20POLY1305_NONCE_LEN))[:XCHACHA20POLY1305_NONCE_LEN]
 
     with user_data_lock:
         user_data["contacts"][contact_id]["our_pads"] = user_data["contacts"][contact_id]["our_pads"][len(message_encrypted):]
 
-        our_strand_key = user_data["contacts"][contact_id]["our_strand_key"]
+        our_next_strand_key = user_data["contacts"][contact_id]["our_next_strand_key"]
+        user_data["contacts"][contact_id]["our_next_strand_key"] = new_strand_key
 
         our_next_strand_nonce = user_data["contacts"][contact_id]["our_next_strand_nonce"]
-        user_data["contacts"][contact_id]["our_next_strand_nonce"]  = our_new_strand_nonce 
+        user_data["contacts"][contact_id]["our_next_strand_nonce"]  = new_strand_nonce 
 
     save_account_data(user_data, user_data_lock)
 
     _, ciphertext_blob = encrypt_xchacha20poly1305(
-            our_strand_key, 
-            MSG_TYPES["MSG_NEW"] + our_new_strand_nonce + message_encrypted,
+            our_next_strand_key, 
+            new_strand_key + new_strand_nonce + MSG_TYPES["MSG_NEW"] + message_encrypted,
             nonce = our_next_strand_nonce
         )
 
@@ -251,7 +260,7 @@ def messages_data_handler(user_data: dict, user_data_lock, user_data_copied: dic
         logger.error("Contact (%s) per-contact ML-DSA-87 public key is missing! Skipping message..", contact_id)
         return
 
-    if user_data_copied["contacts"][contact_id]["contact_strand_key"] is None:
+    if user_data_copied["contacts"][contact_id]["contact_next_strand_key"] is None:
         logger.error("Contact (%s) strand key key is missing! Skipping message...", contact_id)
         return 
 
@@ -262,14 +271,14 @@ def messages_data_handler(user_data: dict, user_data_lock, user_data_copied: dic
 
         # /32 because KEM shared_secret is 32 bytes, /64 because sha3_512 output is 64 bytes
 
-        if len(msgs_plaintext) != ( (ML_KEM_1024_CT_LEN + CLASSIC_MCELIECE_8_F_CT_LEN) * (OTP_PAD_SIZE // 32)) + (64 * (OTP_PAD_SIZE // 64)) + ML_DSA_87_SIGN_LEN + XCHACHA20POLY1305_NONCE_LEN + 1:
+        if len(msgs_plaintext) != ( (ML_KEM_1024_CT_LEN + CLASSIC_MCELIECE_8_F_CT_LEN) * (OTP_PAD_SIZE // 32)) + (64 * (OTP_PAD_SIZE // 64)) + ML_DSA_87_SIGN_LEN + 1:
             logger.error("Contact (%s) gave us a otp batch message request with malformed strand plaintext length (%d)", contact_id, len(msgs_plaintext))
             return
 
-        otp_hashchain_signature  = msgs_plaintext[1 + XCHACHA20POLY1305_NONCE_LEN: ML_DSA_87_SIGN_LEN + XCHACHA20POLY1305_NONCE_LEN + 1]
-        otp_hashchain_ciphertext = msgs_plaintext[ML_DSA_87_SIGN_LEN + XCHACHA20POLY1305_NONCE_LEN + 1: ML_DSA_87_SIGN_LEN + XCHACHA20POLY1305_NONCE_LEN + 1 + ((ML_KEM_1024_CT_LEN + CLASSIC_MCELIECE_8_F_CT_LEN) * (OTP_PAD_SIZE // 32))]
+        otp_hashchain_signature  = msgs_plaintext[1: ML_DSA_87_SIGN_LEN + 1]
+        otp_hashchain_ciphertext = msgs_plaintext[ML_DSA_87_SIGN_LEN + 1: ML_DSA_87_SIGN_LEN + 1 + ((ML_KEM_1024_CT_LEN + CLASSIC_MCELIECE_8_F_CT_LEN) * (OTP_PAD_SIZE // 32))]
 
-        xchacha_pads = msgs_plaintext[ML_DSA_87_SIGN_LEN + XCHACHA20POLY1305_NONCE_LEN + 1 + ((ML_KEM_1024_CT_LEN + CLASSIC_MCELIECE_8_F_CT_LEN) * (OTP_PAD_SIZE // 32)):]
+        xchacha_pads = msgs_plaintext[ML_DSA_87_SIGN_LEN + 1 + ((ML_KEM_1024_CT_LEN + CLASSIC_MCELIECE_8_F_CT_LEN) * (OTP_PAD_SIZE // 32)):]
 
         try:
             valid_signature = verify_signature(ML_DSA_87_NAME, otp_hashchain_ciphertext, otp_hashchain_signature, contact_public_key)
@@ -298,14 +307,14 @@ def messages_data_handler(user_data: dict, user_data_lock, user_data_copied: dic
         contact_pads, _ = one_time_pad(contact_kyber_pads, contact_mceliece_pads)
         contact_pads, _ = one_time_pad(contact_pads, xchacha_pads)
 
-        contact_strand_key = contact_pads[:32]
+        contact_next_strand_key = contact_pads[:32]
         contact_pads = contact_pads[32:]
 
 
         with user_data_lock:
             user_data["contacts"][contact_id]["contact_pads"] = contact_pads
 
-            user_data["contacts"][contact_id]["contact_strand_key"] = contact_strand_key
+            user_data["contacts"][contact_id]["contact_next_strand_key"] = contact_next_strand_key
 
             user_data["contacts"][contact_id]["ephemeral_keys"]["our_keys"][CLASSIC_MCELIECE_8_F_NAME]["rotation_counter"] += 1
 
@@ -331,12 +340,12 @@ def messages_data_handler(user_data: dict, user_data_lock, user_data_copied: dic
     elif bytes([msgs_plaintext[0]]) == MSG_TYPES["MSG_NEW"]:
         logger.debug("Received a new message from contact (%s).", contact_id)
 
-        if len(msgs_plaintext) < OTP_MAX_BUCKET + XCHACHA20POLY1305_NONCE_LEN + 1:
+        if len(msgs_plaintext) < OTP_MAX_BUCKET + 1:
             logger.error("Contact (%s) gave us a message request with malformed strand plaintext length (%d)", contact_id, len(msgs_plaintext))
             return
 
 
-        message_encrypted = msgs_plaintext[XCHACHA20POLY1305_NONCE_LEN + 1:]
+        message_encrypted = msgs_plaintext[1:]
 
 
         with user_data_lock:
@@ -378,5 +387,3 @@ def messages_data_handler(user_data: dict, user_data_lock, user_data_copied: dic
         logger.error("Received unknown message type (%d)", msgs_plaintext[0])
         return
 
-    with user_data_lock:
-        user_data["contacts"][contact_id]["contact_next_strand_nonce"]  = msgs_plaintext[1: XCHACHA20POLY1305_NONCE_LEN + 1]