feat(sdk): Adds OpenTDF.open method (#485)

This method allocates and returns a reader object, instead of directly returning an annotated webstream. This has several advantages, mostly for dealing with base TDFs at the moment:

- allocating is synchronous and does not result in immediate network calls, allowing these objects to be pre-allocated.
- inspection methods, most notably `attributes` and `manifest`, can be executed with loading less than the full file and without a key access server

The attributes method does not work for remote or encrypted policies, and the manifest method doesn't have an implementation for Nano at the moment. Also potentially useful features, like accessing encrypted metadata, still also require doing a full decrypt, as the KAO processing code is still inline with the decrypt method in readStream

Author: dmihalcik-virtru

.github/workflows/roundtrip/encrypt-decrypt.sh

@@ -68,3 +68,33 @@ _tdf3_test() {
 }
 
 _tdf3_test @opentdf/ctl @opentdf/ctl
+
+_tdf3_inspect_test() {
+  counter=$((counter + 1))
+  plain="./sample-${counter}.txt"
+  echo "Hello World ${counter}" >"${plain}"
+  npx "$1" --log-level DEBUG \
+    --kasEndpoint http://localhost:65432/kas \
+    --ignoreAllowList \
+    --oidcEndpoint http://localhost:65432/auth/realms/opentdf \
+    --auth testclient:secret \
+    --output sample-with-attrs.txt.tdf \
+    --attributes 'https://attr.io/attr/a/value/1,https://attr.io/attr/x/value/2' \
+    encrypt "${plain}" \
+    --containerType tdf3 
+
+  [ -f sample-with-attrs.txt.tdf ]
+
+  npx "$1" --log-level DEBUG \
+    inspect sample-with-attrs.txt.tdf > sample_inspect_out.txt
+
+  cat sample_inspect_out.txt
+
+  [ -f sample_inspect_out.txt ]
+  grep -q 'https://attr.io/attr/a/value/1' sample_inspect_out.txt
+
+  echo "Inspect tdf3 successful!"
+  rm -f "${plain}" sample-with-attrs.txt.tdf sample_inspect_out.txt
+}
+
+_tdf3_inspect_test @opentdf/ctl

cli/bin/opentdf.bats

@@ -9,7 +9,7 @@
 @test "requires optional arguments" {
   run $BATS_TEST_DIRNAME/opentdf.mjs encrypt noone
   echo "$output"
-  [[ $output == *"Missing required"* ]]
+  [[ $output == *"must be specified"* ]]
 }
 
 @test "fails with missing file arguments" {

cli/package-lock.json

@@ -30,14 +30,21 @@ type AuthToProcess = {
   clientId?: string;
   clientSecret?: string;
   concurrencyLimit?: number;
-  oidcEndpoint: string;
+  oidcEndpoint?: string;
   userId?: string;
 };
 
 type LoggedAuthProvider = AuthProvider & {
   requestLog: HttpRequest[];
 };
 
+class InvalidAuthProvider {
+  async updateClientPublicKey(): Promise<void> {}
+  withCreds(): Promise<HttpRequest> {
+    throw new Error('Method not implemented.');
+  }
+}
+
 const bindingTypes = ['ecdsa', 'gmac'];
 
 const containerTypes = ['tdf3', 'nano', 'dataset', 'ztdf'];
@@ -59,6 +66,9 @@ async function processAuth({
   userId,
 }: AuthToProcess): Promise<LoggedAuthProvider> {
   log('DEBUG', 'Processing auth params');
+  if (!oidcEndpoint) {
+    throw new CLIError('CRITICAL', 'oidcEndpoint must be specified');
+  }
   if (auth) {
     log('DEBUG', 'Processing an auth string');
     const authParts = auth.split(':');
@@ -355,7 +365,6 @@ export const handleArgs = (args: string[]) => {
         description: 'URL to non-default KAS instance (https://mykas.net)',
       })
       .option('oidcEndpoint', {
-        demandOption: true,
         group: 'Server Endpoints:',
         type: 'string',
         description: 'URL to non-default OIDC IdP (https://myidp.net)',
@@ -500,7 +509,6 @@ export const handleArgs = (args: string[]) => {
         },
       })
 
-      // COMMANDS
       .options({
         logLevel: {
           group: 'Verbosity:',
@@ -523,7 +531,7 @@ export const handleArgs = (args: string[]) => {
 
       .command(
         'attrs',
-        'Look up defintions of attributes',
+        'Look up definitions of attributes',
         (yargs) => {
           yargs.strict();
         },
@@ -556,6 +564,36 @@ export const handleArgs = (args: string[]) => {
         }
       )
 
+      .command(
+        'inspect [file]',
+        'Inspect TDF or nanoTDF and extract header information, without decrypting',
+        (yargs) => {
+          yargs.strict().positional('file', {
+            describe: 'path to encrypted file',
+            type: 'string',
+          });
+        },
+        async (argv) => {
+          log('DEBUG', 'Running inspect command');
+          const ct = new OpenTDF({
+            authProvider: new InvalidAuthProvider(),
+          });
+          try {
+            const reader = ct.open(await parseReadOptions(argv));
+            const manifest = await reader.manifest();
+            try {
+              const dataAttributes = await reader.attributes();
+              console.log(JSON.stringify({ manifest, dataAttributes }, null, 2));
+            } catch (err) {
+              console.error(err);
+              console.log(JSON.stringify({ manifest }, null, 2));
+            }
+          } finally {
+            ct.close();
+          }
+        }
+      )
+
       .command(
         'decrypt [file]',
         'Decrypt TDF to string',
@@ -573,6 +611,9 @@ export const handleArgs = (args: string[]) => {
           }
           log('DEBUG', `Allowed KASes: ${allowedKases}`);
           const ignoreAllowList = !!argv.ignoreAllowList;
+          if (!argv.oidcEndpoint) {
+            throw new CLIError('CRITICAL', 'oidcEndpoint must be specified');
+          }
           const authProvider = await processAuth(argv);
           log('DEBUG', `Initialized auth provider ${JSON.stringify(authProvider)}`);
           const client = new OpenTDF({

cli/src/cli.ts

@@ -10,7 +10,6 @@ import {
 } from './nanotdf/index.js';
 import { keyAgreement } from './nanotdf-crypto/index.js';
 import { Policy } from './tdf/Policy.js';
-import { type TypedArray } from './tdf/TypedArray.js';
 import { createAttribute } from './tdf/AttributeObject.js';
 import { fetchECKasPubKey } from './access.js';
 import { ClientConfig } from './nanotdf/Client.js';
@@ -38,7 +37,7 @@ export class NanoTDFClient extends Client {
    *
    * @param ciphertext Ciphertext to decrypt
    */
-  async decrypt(ciphertext: string | TypedArray | ArrayBuffer): Promise<ArrayBuffer> {
+  async decrypt(ciphertext: string | ArrayBufferLike): Promise<ArrayBuffer> {
     // Parse ciphertext
     const nanotdf = NanoTDF.from(ciphertext);
 
@@ -68,7 +67,7 @@ export class NanoTDFClient extends Client {
    *
    * @param ciphertext Ciphertext to decrypt
    */
-  async decryptLegacyTDF(ciphertext: string | TypedArray | ArrayBuffer): Promise<ArrayBuffer> {
+  async decryptLegacyTDF(ciphertext: string | ArrayBufferLike): Promise<ArrayBuffer> {
     // Parse ciphertext
     const nanotdf = NanoTDF.from(ciphertext, undefined, true);
 
@@ -91,15 +90,12 @@ export class NanoTDFClient extends Client {
   /**
    * Encrypts the given data using the NanoTDF encryption scheme.
    *
-   * @param {string | TypedArray | ArrayBuffer} data - The data to be encrypted.
-   * @param {EncryptOptions} [options=defaultOptions] - The encryption options (currently unused).
-   * @returns {Promise<ArrayBuffer>} A promise that resolves to the encrypted data as an ArrayBuffer.
-   * @throws {Error} If the initialization vector is not a number.
+   * @param data The data to be encrypted.
+   * @param options The encryption options (currently unused).
+   * @returns A promise that resolves to the encrypted data as an ArrayBuffer.
+   * @throws If the initialization vector is not a number.
    */
-  async encrypt(
-    data: string | TypedArray | ArrayBuffer,
-    options?: EncryptOptions
-  ): Promise<ArrayBuffer> {
+  async encrypt(data: string | ArrayBufferLike, options?: EncryptOptions): Promise<ArrayBuffer> {
     // For encrypt always generate the client ephemeralKeyPair
     const ephemeralKeyPair = await this.ephemeralKeyPair;
     const initializationVector = this.iv;
@@ -234,10 +230,7 @@ export class NanoTDFDatasetClient extends Client {
    *
    * @param data to decrypt
    */
-  async encrypt(
-    data: string | TypedArray | ArrayBuffer,
-    options?: EncryptOptions
-  ): Promise<ArrayBuffer> {
+  async encrypt(data: string | ArrayBufferLike, options?: EncryptOptions): Promise<ArrayBuffer> {
     // Intial encrypt
     if (this.keyIterationCount == 0) {
       const mergedOptions: EncryptOptions = { ...defaultOptions, ...options };
@@ -323,7 +316,7 @@ export class NanoTDFDatasetClient extends Client {
    *
    * @param ciphertext Ciphertext to decrypt
    */
-  async decrypt(ciphertext: string | TypedArray | ArrayBuffer): Promise<ArrayBuffer> {
+  async decrypt(ciphertext: string | ArrayBufferLike): Promise<ArrayBuffer> {
     // Parse ciphertext
     const nanotdf = NanoTDF.from(ciphertext);
 

lib/src/nanoclients.ts

@@ -1,8 +1,6 @@
-import { TypedArray } from '../tdf/TypedArray.js';
-
 export default function digest(
   hashType: AlgorithmIdentifier,
-  data: TypedArray | ArrayBuffer
+  data: ArrayBufferLike
 ): Promise<ArrayBuffer> {
   return crypto.subtle.digest(hashType, data);
 }

lib/src/nanotdf-crypto/digest.ts

@@ -1,4 +1,3 @@
-import { type TypedArray } from '../tdf/TypedArray.js';
 import * as base64 from '../encodings/base64.js';
 import { generateKeyPair, keyAgreement } from '../nanotdf-crypto/index.js';
 import getHkdfSalt from './helpers/getHkdfSalt.js';
@@ -210,9 +209,9 @@ export default class Client {
    * @param clientVersion version of the client, as SemVer
    */
   async rewrapKey(
-    nanoTdfHeader: TypedArray | ArrayBuffer,
+    nanoTdfHeader: ArrayBufferLike,
     kasRewrapUrl: string,
-    magicNumberVersion: TypedArray | ArrayBuffer,
+    magicNumberVersion: ArrayBufferLike,
     clientVersion: string
   ): Promise<CryptoKey> {
     if (!this.allowedKases.allows(kasRewrapUrl)) {

lib/src/nanotdf/Client.ts

@@ -1,4 +1,3 @@
-import { TypedArray } from '../tdf/TypedArray.js';
 import { base64 } from '../encodings/index.js';
 import Header from './models/Header.js';
 import Payload from './models/Payload.js';
@@ -22,7 +21,7 @@ export default class NanoTDF {
   public signature?: Signature;
 
   static from(
-    content: TypedArray | ArrayBuffer | string,
+    content: ArrayBufferLike | string,
     encoding?: EncodingEnum,
     legacyTDF = false
   ): NanoTDF {

lib/src/nanotdf/NanoTDF.ts

@@ -3,7 +3,6 @@ import Header from './models/Header.js';
 import DefaultParams from './models/DefaultParams.js';
 import Payload from './models/Payload.js';
 import { getBitLength as authTagLengthForCipher } from './models/Ciphers.js';
-import { TypedArray } from '../tdf/TypedArray.js';
 import encrypt from '../nanotdf-crypto/encrypt.js';
 
 /**
@@ -18,7 +17,7 @@ export default async function encryptDataset(
   symmetricKey: CryptoKey,
   header: Header,
   iv: Uint8Array,
-  data: string | TypedArray | ArrayBuffer
+  data: string | ArrayBufferLike
 ): Promise<ArrayBuffer> {
   // Auth tag length for policy and payload
   const authTagLengthInBytes = authTagLengthForCipher(DefaultParams.symmetricCipher) / 8;

lib/src/nanotdf/encrypt-dataset.ts

@@ -6,7 +6,6 @@ import EmbeddedPolicy from './models/Policy/EmbeddedPolicy.js';
 import Payload from './models/Payload.js';
 import getHkdfSalt from './helpers/getHkdfSalt.js';
 import { getBitLength as authTagLengthForCipher } from './models/Ciphers.js';
-import { TypedArray } from '../tdf/TypedArray.js';
 import { GMAC_BINDING_LEN } from './constants.js';
 import { AlgorithmName, KeyFormat, KeyUsageType } from './../nanotdf-crypto/enums.js';
 
@@ -35,7 +34,7 @@ export default async function encrypt(
   kasInfo: KasPublicKeyInfo,
   ephemeralKeyPair: CryptoKeyPair,
   iv: Uint8Array,
-  data: string | TypedArray | ArrayBuffer,
+  data: string | ArrayBufferLike,
   ecdsaBinding: boolean = DefaultParams.ecdsaBinding
 ): Promise<ArrayBuffer> {
   // Generate a symmetric key.