feat(lib): Load abac config from policy service (#351)

New parameters for tdf/client allow looking up attribute definitions and KAS grants to autoconfigure with just attribute URLs.

- New arguments to cli `encrypt`
   - `encrypt --autoconfigure` enables attribute lookup during encrypt and corresponding updates to the KAO
   - `--policyEndpoint` allows KAS and policy service to be hosted separately. if not set, it is inferred by removing `/kas` off of the end of the `--kasEndpoint` argument.
- New cli command, `attrs`, which prints out the JSON hydrated version of the attributes from the policy service

Author: dmihalcik-virtru

.github/workflows/roundtrip/package-lock.json

@@ -18,6 +18,7 @@ import {
 } from '@opentdf/client';
 import { CLIError, Level, log } from './logger.js';
 import { webcrypto } from 'crypto';
+import { attributeFQNsAsValues } from '@opentdf/client/nano';
 
 type AuthToProcess = {
   auth?: string;
@@ -91,6 +92,13 @@ async function processAuth({
   };
 }
 
+const rstrip = (str: string, suffix = ' '): string => {
+  while (str && suffix && str.endsWith(suffix)) {
+    str = str.slice(0, -suffix.length);
+  }
+  return str;
+};
+
 type AnyNanoClient = NanoTDFClient | NanoTDFDatasetClient;
 
 function addParams(client: AnyNanoClient, argv: Partial<mainArgs>) {
@@ -120,6 +128,9 @@ async function tdf3EncryptParamsFor(argv: Partial<mainArgs>): Promise<EncryptPar
   if (argv.mimeType?.length) {
     c.setMimeType(argv.mimeType);
   }
+  if (argv.autoconfigure) {
+    c.withAutoconfigure();
+  }
   // use offline mode, we do not have upsert for v2
   c.setOffline();
   // FIXME TODO must call file.close() after we are done
@@ -171,53 +182,66 @@ export const handleArgs = (args: string[]) => {
       // AUTH OPTIONS
       .option('kasEndpoint', {
         demandOption: true,
-        group: 'KAS Configuration',
+        group: 'Server Endpoints:',
         type: 'string',
         description: 'URL to non-default KAS instance (https://mykas.net)',
       })
       .option('oidcEndpoint', {
         demandOption: true,
-        group: 'OIDC IdP Endpoint:',
+        group: 'Server Endpoints:',
         type: 'string',
         description: 'URL to non-default OIDC IdP (https://myidp.net)',
       })
+      .option('policyEndpoint', {
+        group: 'Server Endpoints:',
+        type: 'string',
+        description: 'Attribute and key grant service endpoint',
+      })
       .option('allowList', {
-        group: 'KAS Configuration',
+        group: 'Security:',
         desc: 'allowed KAS origins, comma separated; defaults to [kasEndpoint]',
         type: 'string',
         validate: (attributes: string) => attributes.split(','),
       })
-      .boolean('ignoreAllowList')
+      .option('ignoreAllowList', {
+        group: 'Security:',
+        desc: 'disable KAS allowlist feature for decrypt',
+        type: 'boolean',
+      })
       .option('auth', {
-        group: 'Authentication:',
+        group: 'OAuth and OIDC:',
         type: 'string',
-        description: 'Authentication string (<clientId>:<clientSecret>)',
+        description: 'Combined OAuth Client Credentials (<clientId>:<clientSecret>)',
+      })
+      .option('dpop', {
+        group: 'Security:',
+        desc: 'Use DPoP for token binding',
+        type: 'boolean',
       })
-      .boolean('dpop')
       .implies('auth', '--no-clientId')
       .implies('auth', '--no-clientSecret')
 
       .option('clientId', {
-        group: 'OIDC client credentials',
+        group: 'OAuth and OIDC:',
         alias: 'cid',
         type: 'string',
-        description: 'IdP-issued Client ID',
+        description: 'OAuth Client Credentials: IdP-issued Client ID',
       })
       .implies('clientId', 'clientSecret')
 
       .option('clientSecret', {
-        group: 'OIDC client credentials',
+        group: 'OAuth and OIDC:',
         alias: 'cs',
         type: 'string',
-        description: 'IdP-issued Client Secret',
+        description: 'OAuth Client Credentials: IdP-issued Client Secret',
       })
       .implies('clientSecret', 'clientId')
 
       .option('exchangeToken', {
-        group: 'Token from trusted external IdP to exchange for Virtru auth',
+        group: 'OAuth and OIDC:',
         alias: 'et',
         type: 'string',
-        description: 'Token issued by trusted external IdP',
+        description: 'OAuth Token Exchange: Token issued by trusted external IdP',
       })
       .implies('exchangeToken', 'clientId')
 
@@ -229,39 +253,45 @@ export const handleArgs = (args: string[]) => {
       // Policy, encryption, and container options
       .options({
         attributes: {
-          group: 'Encrypt Options',
+          group: 'Encrypt Options:',
           desc: 'Data attributes for the policy',
           type: 'string',
           default: '',
           validate: (attributes: string) => attributes.split(','),
         },
+        autoconfigure: {
+          group: 'Encrypt Options:',
+          desc: 'Enable automatic configuration from attributes using policy service',
+          type: 'boolean',
+          default: false,
+        },
         containerType: {
-          group: 'Encrypt Options',
+          group: 'Encrypt Options:',
           alias: 't',
           choices: containerTypes,
           description: 'Container format',
           default: 'nano',
         },
         policyBinding: {
-          group: 'Encrypt Options',
+          group: 'Encrypt Options:',
           choices: bindingTypes,
           description: 'Policy Binding Type (nano only)',
           default: 'gmac',
         },
         mimeType: {
-          group: 'Encrypt Options',
+          group: 'Encrypt Options:',
           desc: 'Mime type for the plain text file (only supported for ztdf)',
           type: 'string',
           default: '',
         },
         userId: {
-          group: 'Encrypt Options',
+          group: 'Encrypt Options:',
           type: 'string',
           description: 'Owner email address',
         },
         usersWithAccess: {
           alias: 'users-with-access',
-          group: 'Encrypt Options',
+          group: 'Encrypt Options:',
           desc: 'Add users to the policy',
           type: 'string',
           default: '',
@@ -272,12 +302,14 @@ export const handleArgs = (args: string[]) => {
       // COMMANDS
       .options({
         logLevel: {
+          group: 'Verbosity:',
           alias: 'log-level',
           type: 'string',
           default: 'info',
           desc: 'Set logging level',
         },
         silent: {
+          group: 'Verbosity:',
           type: 'boolean',
           default: false,
           desc: 'Disable logging',
@@ -287,6 +319,39 @@ export const handleArgs = (args: string[]) => {
         type: 'string',
         description: 'output file',
       })
+
+      .command(
+        'attrs',
+        'Look up defintions of attributes',
+        (yargs) => {
+          yargs.strict();
+        },
+        async (argv) => {
+          log('DEBUG', 'attribute value lookup');
+          const authProvider = await processAuth(argv);
+          const signingKey = await crypto.subtle.generateKey(
+            {
+              name: 'RSASSA-PKCS1-v1_5',
+              hash: 'SHA-256',
+              modulusLength: 2048,
+              publicExponent: new Uint8Array([0x01, 0x00, 0x01]),
+            },
+            true,
+            ['sign', 'verify']
+          );
+          authProvider.updateClientPublicKey(signingKey);
+          log('DEBUG', `Initialized auth provider ${JSON.stringify(authProvider)}`);
+
+          const policyUrl: string = guessPolicyUrl(argv);
+          const defs = await attributeFQNsAsValues(
+            policyUrl,
+            authProvider,
+            ...(argv.attributes as string).split(',')
+          );
+          console.log(JSON.stringify(defs, null, 2));
+        }
+      )
+
       .command(
         'decrypt [file]',
         'Decrypt TDF to string',
@@ -397,11 +462,13 @@ export const handleArgs = (args: string[]) => {
 
           if ('tdf3' === argv.containerType || 'ztdf' === argv.containerType) {
             log('DEBUG', `TDF3 Client`);
+            const policyEndpoint: string = guessPolicyUrl(argv);
             const client = new TDF3Client({
               allowedKases,
               ignoreAllowList,
               authProvider,
               kasEndpoint,
+              policyEndpoint,
               dpopEnabled: argv.dpop,
             });
             log('SILLY', `Initialized client ${JSON.stringify(client)}`);
@@ -480,3 +547,20 @@ handleArgs(hideBin(process.argv))
   .catch((err) => {
     console.error(err);
   });
+
+function guessPolicyUrl({
+  kasEndpoint,
+  policyEndpoint,
+}: {
+  kasEndpoint: string;
+  policyEndpoint?: string;
+}) {
+  let policyUrl: string;
+  if (policyEndpoint) {
+    policyUrl = rstrip(policyEndpoint, '/');
+  } else {
+    const uNoSlash = rstrip(kasEndpoint, '/');
+    policyUrl = uNoSlash.endsWith('/kas') ? uNoSlash.slice(0, -4) : uNoSlash;
+  }
+  return policyUrl;
+}

cli/package-lock.json

@@ -12,6 +12,7 @@ import { keyAgreement } from './nanotdf-crypto/index.js';
 import { TypedArray, createAttribute, Policy } from './tdf/index.js';
 import { fetchECKasPubKey } from './access.js';
 import { ClientConfig } from './nanotdf/Client.js';
+export { attributeFQNsAsValues } from './policy/api.js';
 
 // Define the EncryptOptions type
 export type EncryptOptions = {

cli/src/cli.ts

@@ -0,0 +1,62 @@
+import { AuthProvider } from '../auth/auth.js';
+import { rstrip } from '../utils.js';
+import { GetAttributeValuesByFqnsResponse, Value } from './attributes.js';
+
+export async function attributeFQNsAsValues(
+  kasUrl: string,
+  authProvider: AuthProvider,
+  ...fqns: string[]
+): Promise<Value[]> {
+  const avs = new URLSearchParams();
+  for (const fqn of fqns) {
+    avs.append('fqns', fqn);
+  }
+  avs.append('withValue.withKeyAccessGrants', 'true');
+  avs.append('withValue.withAttribute.withKeyAccessGrants', 'true');
+  const uNoSlash = rstrip(kasUrl, '/');
+  const uNoKas = uNoSlash.endsWith('/kas') ? uNoSlash.slice(0, -4) : uNoSlash;
+  const url = `${uNoKas}/attributes/*/fqn?${avs}`;
+  const req = await authProvider.withCreds({
+    url,
+    headers: {},
+    method: 'GET',
+  });
+  let response: Response;
+  try {
+    response = await fetch(req.url, {
+      mode: 'cors',
+      credentials: 'same-origin',
+      headers: req.headers,
+      redirect: 'follow',
+      referrerPolicy: 'no-referrer',
+    });
+
+    if (!response.ok) {
+      throw new Error(`${req.method} ${req.url} => ${response.status} ${response.statusText}`);
+    }
+  } catch (e) {
+    console.error(`network error [${req.method} ${req.url}]`, e);
+    throw e;
+  }
+
+  let resp: GetAttributeValuesByFqnsResponse;
+  try {
+    resp = (await response.json()) as GetAttributeValuesByFqnsResponse;
+  } catch (e) {
+    console.error(`response parse error [${req.method} ${req.url}]`, e);
+    throw e;
+  }
+
+  const values: Value[] = [];
+  for (const [fqn, av] of Object.entries(resp.fqnAttributeValues)) {
+    if (!av.value) {
+      console.log(`Missing value definition for [${fqn}]; is this a valid attribute?`);
+      continue;
+    }
+    if (av.attribute && !av.value.attribute) {
+      av.value.attribute = av.attribute;
+    }
+    values.push(av.value);
+  }
+  return values;
+}

lib/src/index.ts

@@ -89,6 +89,10 @@ export type Attribute = {
   metadata?: Metadata;
 };
 
+// This is not currently needed by the client, but may be returned.
+// Setting it to unknown to allow it to be ignored for now.
+export type SubjectMapping = unknown;
+
 export type Value = {
   id?: string;
   attribute?: Attribute;
@@ -98,6 +102,16 @@ export type Value = {
   fqn: string;
   /** active by default until explicitly deactivated */
   active?: boolean;
+  subjectMappings?: SubjectMapping[];
   /** Common metadata */
   metadata?: Metadata;
 };
+
+export type AttributeAndValue = {
+  attribute: Attribute;
+  value: Value;
+};
+
+export type GetAttributeValuesByFqnsResponse = {
+  fqnAttributeValues: Record<string, AttributeAndValue>;
+};