Generate TRCs in PEM format (#406)

Change the data format of the TRCs from DER to PEM, both in the
generated configuration and in the internal representation in the DB.
Add a data migration to convert all existing TRCs.

Previously, we've generated the TRCs in DER format as this is currently
the default output format of `scion-pki trcs combine`. The (binary) DER
data was stored base64-encoded in the text database field.

According to the Anapayans working on this, the PEM format is should
generally be preferred, and it should become the default format for TRCs
eventually.

The practical motivation for the change is this; the CS persists any
TRCs fetched "over the wire" not only in its sqlite DB, but it also
writes them back to the configuration directory, in PEM format. As our
configuration has up to now contained TRCs in DER, installing updated
configuration using `scionlab-config` after the CS wrote such a TRC file
would lead to an apparent file conflict. Using PEM in our configuration
avoids this.

Author: matzf

scionlab/config_tar.py

@@ -34,7 +34,7 @@
 # the scionlab-config.json manifest file in the configuration tar ball.
 # This version number should be incremented whenever code changes globally affect the generated
 # configuration of hosts.
-CONFIG_GEN_VERSION = 14
+CONFIG_GEN_VERSION = 15
 
 
 def generate_user_as_config_tar(user_as, archive):

scionlab/fixtures/testdata.yaml

@@ -0,0 +1,69 @@
+# Copyright 2021 ETH Zurich
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import textwrap
+
+from django.db import migrations
+
+PEM_WIDTH = 64  # see e.g. RFC 7468
+TRC_PEM_HEADER = "-----BEGIN TRC-----\n"
+TRC_PEM_FOOTER = "-----END TRC-----\n"
+
+
+def convert_trcs_to_pem(apps, schema_editor):
+    """
+    Convert TRC data from DER to PEM.
+    As the DER blob is already stored as base64-encoded text in the DB,
+    we only need to wrap lines and add the header and footer to turn it into a PEM.
+    """
+    TRC = apps.get_model('scionlab', 'TRC')
+    for trc in TRC.objects.all():
+        trc.trc = trc_base64der_to_pem(trc.trc)
+        trc.save()
+
+
+def trc_base64der_to_pem(der: str) -> str:
+    assert not der.startswith(TRC_PEM_HEADER)
+    assert not der.endswith(TRC_PEM_HEADER)
+    body = textwrap.fill(der, width=PEM_WIDTH)
+    return TRC_PEM_HEADER + body + "\n" + TRC_PEM_FOOTER
+
+
+def revert_trcs_to_der(apps, schema_editor):
+    """
+    Convert TRC from PEM back to base64-encoded DER.
+    """
+    TRC = apps.get_model('scionlab', 'TRC')
+    for trc in TRC.objects.all():
+        trc.trc = trc_pem_to_base64der(trc.trc)
+        trc.save()
+
+
+def trc_pem_to_base64der(pem: str) -> str:
+    assert pem.startswith(TRC_PEM_HEADER)
+    assert pem.endswith(TRC_PEM_FOOTER)
+    stripped = pem[len(TRC_PEM_HEADER):-len(TRC_PEM_FOOTER)]
+    return stripped.replace("\n", "")
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scionlab', '0004_remove_link_bandwidth'),
+    ]
+
+    operations = [
+        migrations.RunPython(convert_trcs_to_pem,  # forward
+                             revert_trcs_to_der),  # backward
+    ]

scionlab/migrations/0005_trc_pem.py

@@ -35,7 +35,7 @@
 from scionlab.scion import as_ids
 from scionlab.scion.certs import verify_certificate_valid, verify_cp_as_chain
 from scionlab.scion.keys import verify_key
-from scionlab.scion.trcs import decode_trc, verify_trcs
+from scionlab.scion.trcs import verify_trcs
 from scionlab.scion.pkicommand import ScionPkiError
 from scionlab.util.django import value_set
 from scionlab.defines import (
@@ -117,7 +117,7 @@ def validate_crypto(self):
         namely that the trc chain is valid.
         Returns the number of valid trcs in this ISD.
         """
-        ts = [decode_trc(t.trc) for t in self.trcs.order_by('serial_version')]
+        ts = [t.trc for t in self.trcs.order_by('serial_version')]
         verify_trcs(ts[0], *ts)
         return len(ts)
 
@@ -306,7 +306,7 @@ def validate_crypto(self):
         latests certificates and keys and checking them against the latest TRC.
         Returns the number of certificates (with keys) that passed the check.
         """
-        trc = decode_trc(self.isd.trcs.latest().trc)
+        trc = self.isd.trcs.latest().trc
         certs = self.certificates_latest()
         cert_map = {}
         for cert in certs:

scionlab/models/core.py

@@ -112,7 +112,7 @@ def create(self, isd):
         votes_idx = prev.get_certificate_indices(votes) if prev else []
 
         trc = trcs.generate_trc(
-            prev_trc=trcs.decode_trc(prev.trc) if prev else None,
+            prev_trc=prev.trc if prev else None,
             isd_id=isd.isd_id,
             base=base,
             serial=serial,
@@ -128,7 +128,7 @@ def create(self, isd):
         )
         obj = super().create(isd=isd, serial_version=serial, base_version=base,
                              not_before=not_before, not_after=not_after,
-                             quorum=quorum, trc=trcs.encode_trc(trc))
+                             quorum=quorum, trc=trc)
         obj.core_ases.set(core_ases)
         obj.certificates.add(*certificates)
         obj.votes.set(votes)
@@ -236,9 +236,6 @@ def get_certificate_indices(self, certs):
     def filename(self) -> str:
         return f'ISD{self.isd.isd_id}-B{self.base_version}-S{self.serial_version}.trc'
 
-    def format_trcfile(self) -> bytes:
-        return trcs.decode_trc(self.trc)
-
     def check_regular_update_error(self):
         """
         Check if this TRC is a proper regular update. For testing.

scionlab/models/trc.py

@@ -75,14 +75,14 @@ def verify_certificate_valid(cert: str, cert_usage: str):
         _run_scion_pki_certificate('validate', '--type', cert_usage, '--check-time', f.name)
 
 
-def verify_cp_as_chain(cert: str, trc: bytes):
+def verify_cp_as_chain(cert: str, trc: str):
     """
     Verify that the certificate is valid, using the last TRC as anchor.
     The certificate is passed as a PEM string.
     The TRC is passed as bytes, basee 64 format.
     Raises ScionPkiError if the certificate is not valid.
     """
-    with NamedTemporaryFile(mode='wb', suffix=".trc") as trc_file,\
+    with NamedTemporaryFile(mode='wt', suffix=".trc") as trc_file,\
          NamedTemporaryFile(mode='wt', suffix=".pem") as cert_file:
         files = [trc_file, cert_file]
         for f, value in zip(files, [trc, cert]):

scionlab/scion/certs.py

@@ -128,7 +128,7 @@ def _write_trcs(self, dir):
         #   As there are no big disadvantages in always including all versions, that's what we do
         #   for now.
         for trc in TRC.objects.all():
-            self.archive.write_bytes((dir, CERT_DIR, trc.filename()), trc.format_trcfile())
+            self.archive.write_text((dir, CERT_DIR, trc.filename()), trc.trc)
 
     def _write_certs(self, dir):
         for cert in self.AS.certificates_latest().all():

scionlab/scion/config.py

@@ -17,7 +17,6 @@
 ===========================================
 """
 
-import base64
 import toml
 import yaml
 import contextlib
@@ -33,30 +32,22 @@
 from scionlab.scion.pkicommand import run_scion_pki
 
 
-def encode_trc(trc: bytes) -> str:
-    return base64.b64encode(trc).decode('ascii')
-
-
-def decode_trc(trc: str) -> bytes:
-    return base64.b64decode(trc.encode('ascii'))
-
-
-def trc_to_dict(trc: bytes) -> dict:
-    with NamedTemporaryFile('wb') as f:
+def trc_to_dict(trc: str) -> dict:
+    with NamedTemporaryFile('w') as f:
         f.write(trc)
         f.flush()
         ret = _run_scion_pki_trcs('human', '--format', 'yaml', f.name, check=True)
     return yaml.safe_load(ret.stdout)
 
 
-def verify_trcs(*trcs: bytes):
+def verify_trcs(*trcs: str):
     """
     Verify that the sequence of trcs, using the first TRC as anchor.
     TRCs are passed as bytes, base 64 format.
     Raises ScionPkiError if the TRCs are not valid.
     """
     with contextlib.ExitStack() as stack:
-        files = [stack.enter_context(NamedTemporaryFile(suffix=".trc")) for _ in range(len(trcs))]
+        files = [stack.enter_context(NamedTemporaryFile(suffix=".trc", mode="w")) for _ in trcs]
         for f, trc in zip(files, trcs):
             f.write(trc)
             f.flush()
@@ -75,7 +66,7 @@ def generate_trc(prev_trc: bytes,
                  not_after: datetime,
                  certificates: List[str],
                  signers_certs: List[str],
-                 signers_keys: List[str]) -> bytes:
+                 signers_keys: List[str]) -> str:
     """
     Generate a new TRC.
     This method is the interface between the DB objects and the scion PKI ones.
@@ -157,7 +148,7 @@ def __init__(self,
 
         self._validate()
 
-    def generate(self) -> bytes:
+    def generate(self) -> str:
         with TemporaryDirectory() as temp_dir:
             self._dump_certificates_to_files(temp_dir)
             self._gen_payload(temp_dir)
@@ -172,7 +163,7 @@ def _gen_payload(self, temp_dir: str) -> None:
         args = ['payload', '-t', self.CONFIG_FILENAME, '-o', self.PAYLOAD_FILENAME]
         if self.predecessor_trc:
             pred_filename = Path(temp_dir, self.PRED_TRC_FILENAME)
-            pred_filename.write_bytes(self.predecessor_trc)
+            pred_filename.write_text(self.predecessor_trc)
             args.extend(['-p', str(pred_filename)])
         _run_scion_pki_trcs(*args, cwd=temp_dir)
 
@@ -186,17 +177,18 @@ def _sign_payload(self, temp_dir: str, cert: str, key: str) -> bytes:
                                                     pkcs7.PKCS7Options.NoCapabilities,
                                                     pkcs7.PKCS7Options.Binary])
 
-    def _combine(self, temp_dir: str, *signed) -> bytes:
-        """ returns the final TRC by combining the signed blocks and payload """
+    def _combine(self, temp_dir: str, *signed) -> str:
+        """ returns the final TRC (in PEM format), by combining the signed blocks and payload """
         for i, s in enumerate(signed):
             Path(temp_dir, f'signed-{i}.der').write_bytes(s)
         _run_scion_pki_trcs(
             'combine', '-p', self.PAYLOAD_FILENAME,
             *(f'signed-{i}.der' for i in range(len(signed))),
             '-o', Path(temp_dir, self.TRC_FILENAME),
+            '--format', 'pem',
             cwd=temp_dir
         )
-        return Path(temp_dir, self.TRC_FILENAME).read_bytes()
+        return Path(temp_dir, self.TRC_FILENAME).read_text()
 
     def is_update(self):
         return self.base_version != self.serial_version