Reject messages with nonnormalized schema URIs (#325)

* Reject messages with nonnormalized schema URIs

* URI -> URL

* Fix README

Author: Diane Huxley

README.md

@@ -3,7 +3,7 @@
 # Decentralized Web Node (DWN) SDK
 
 Code Coverage
-![Statements](https://img.shields.io/badge/statements-94.07%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-93.6%25-brightgreen.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-91.8%25-brightgreen.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-94.07%25-brightgreen.svg?style=flat)
+![Statements](https://img.shields.io/badge/statements-94.1%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-93.71%25-brightgreen.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-91.94%25-brightgreen.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-94.1%25-brightgreen.svg?style=flat)
 
 ## Introduction
 

src/core/dwn-error.ts

@@ -32,5 +32,7 @@ export enum DwnErrorCode {
   RecordsWriteValidateIntegrityEncryptionCidMismatch = 'RecordsWriteValidateIntegrityEncryptionCidMismatch',
   Secp256k1KeyNotValid = 'Secp256k1KeyNotValid',
   UrlProtocolNotNormalized = 'UrlProtocolNotNormalized',
-  UrlProtocolNotNormalizable = 'UrlProtocolNotNormalizable'
+  UrlProtocolNotNormalizable = 'UrlProtocolNotNormalizable',
+  UrlSchemaNotNormalized = 'UrlSchemaNotNormalized',
+  UrlSchemaNotNormalizable = 'UrlSchemaNotNormalizable'
 };

src/interfaces/protocols/messages/protocols-configure.ts

@@ -5,7 +5,7 @@ import { getCurrentTimeInHighPrecision } from '../../../utils/time.js';
 import { validateAuthorizationIntegrity } from '../../../core/auth.js';
 
 import { DwnInterfaceName, DwnMethodName, Message } from '../../../core/message.js';
-import { normalizeProtocolUri, validateProtocolUriNormalized } from '../../../utils/url.js';
+import { normalizeProtocolUrl, normalizeSchemaUrl, validateProtocolUrlNormalized, validateSchemaUrlNormalized } from '../../../utils/url.js';
 
 export type ProtocolsConfigureOptions = {
   dateCreated? : string;
@@ -18,7 +18,8 @@ export class ProtocolsConfigure extends Message<ProtocolsConfigureMessage> {
 
   public static async parse(message: ProtocolsConfigureMessage): Promise<ProtocolsConfigure> {
     await validateAuthorizationIntegrity(message);
-    validateProtocolUriNormalized(message.descriptor.protocol);
+    validateProtocolUrlNormalized(message.descriptor.protocol);
+    ProtocolsConfigure.validateDefinitionNormalized(message.descriptor.definition);
 
     return new ProtocolsConfigure(message);
   }
@@ -28,8 +29,9 @@ export class ProtocolsConfigure extends Message<ProtocolsConfigureMessage> {
       interface   : DwnInterfaceName.Protocols,
       method      : DwnMethodName.Configure,
       dateCreated : options.dateCreated ?? getCurrentTimeInHighPrecision(),
-      protocol    : normalizeProtocolUri(options.protocol),
-      definition  : options.definition // TODO: #139 - move definition out of the descriptor - https://github.com/TBD54566975/dwn-sdk-js/issues/139
+      protocol    : normalizeProtocolUrl(options.protocol),
+      // TODO: #139 - move definition out of the descriptor - https://github.com/TBD54566975/dwn-sdk-js/issues/139
+      definition  : ProtocolsConfigure.normalizeDefinition(options.definition)
     };
 
     Message.validateJsonSchema({ descriptor, authorization: { } });
@@ -40,4 +42,23 @@ export class ProtocolsConfigure extends Message<ProtocolsConfigureMessage> {
     const protocolsConfigure = new ProtocolsConfigure(message);
     return protocolsConfigure;
   }
+
+  private static validateDefinitionNormalized(definition: ProtocolDefinition): void {
+    // validate schema url normalized
+    for (const labelKey in definition.labels) {
+      const schema = definition.labels[labelKey].schema;
+      validateSchemaUrlNormalized(schema);
+    }
+  }
+
+  private static normalizeDefinition(definition: ProtocolDefinition): ProtocolDefinition {
+    const definitionCopy = { ...definition };
+
+    // Normalize schema url
+    for (const labelKey in definition.labels) {
+      definitionCopy.labels[labelKey].schema = normalizeSchemaUrl(definitionCopy.labels[labelKey].schema);
+    }
+
+    return definitionCopy;
+  }
 }

src/interfaces/protocols/messages/protocols-query.ts

@@ -5,7 +5,7 @@ import { getCurrentTimeInHighPrecision } from '../../../utils/time.js';
 import { removeUndefinedProperties } from '../../../utils/object.js';
 import { validateAuthorizationIntegrity } from '../../../core/auth.js';
 import { DwnInterfaceName, DwnMethodName, Message } from '../../../core/message.js';
-import { normalizeProtocolUri, validateProtocolUriNormalized } from '../../../utils/url.js';
+import { normalizeProtocolUrl, validateProtocolUrlNormalized } from '../../../utils/url.js';
 
 export type ProtocolsQueryOptions = {
   dateCreated?: string;
@@ -19,7 +19,7 @@ export class ProtocolsQuery extends Message<ProtocolsQueryMessage> {
     await validateAuthorizationIntegrity(message);
 
     if (message.descriptor.filter !== undefined) {
-      validateProtocolUriNormalized(message.descriptor.filter.protocol);
+      validateProtocolUrlNormalized(message.descriptor.filter.protocol);
     }
 
     return new ProtocolsQuery(message);
@@ -53,7 +53,7 @@ export class ProtocolsQuery extends Message<ProtocolsQueryMessage> {
 
     return {
       ...filter,
-      protocol: normalizeProtocolUri(filter.protocol),
+      protocol: normalizeProtocolUrl(filter.protocol),
     };
   }
 }

src/interfaces/records/messages/records-query.ts

@@ -7,7 +7,7 @@ import { Message } from '../../../core/message.js';
 import { removeUndefinedProperties } from '../../../utils/object.js';
 import { validateAuthorizationIntegrity } from '../../../core/auth.js';
 import { DwnInterfaceName, DwnMethodName } from '../../../core/message.js';
-import { normalizeProtocolUri, validateProtocolUriNormalized } from '../../../utils/url.js';
+import { normalizeProtocolUrl, normalizeSchemaUrl, validateProtocolUrlNormalized, validateSchemaUrlNormalized } from '../../../utils/url.js';
 
 export enum DateSort {
   CreatedAscending = 'createdAscending',
@@ -28,8 +28,11 @@ export class RecordsQuery extends Message<RecordsQueryMessage> {
   public static async parse(message: RecordsQueryMessage): Promise<RecordsQuery> {
     await validateAuthorizationIntegrity(message);
 
-    if (message.descriptor.filter?.protocol !== undefined) {
-      validateProtocolUriNormalized(message.descriptor.filter.protocol);
+    if (message.descriptor.filter.protocol !== undefined) {
+      validateProtocolUrlNormalized(message.descriptor.filter.protocol);
+    }
+    if (message.descriptor.filter.schema !== undefined) {
+      validateSchemaUrlNormalized(message.descriptor.filter.schema);
     }
 
     return new RecordsQuery(message);
@@ -40,7 +43,7 @@ export class RecordsQuery extends Message<RecordsQueryMessage> {
       interface   : DwnInterfaceName.Records,
       method      : DwnMethodName.Query,
       dateCreated : options.dateCreated ?? getCurrentTimeInHighPrecision(),
-      filter      : RecordsQuery.normalizerFilter(options.filter),
+      filter      : RecordsQuery.normalizeFilter(options.filter),
       dateSort    : options.dateSort
     };
 
@@ -101,17 +104,25 @@ export class RecordsQuery extends Message<RecordsQueryMessage> {
     return filterCopy as Filter;
   }
 
-  public static normalizerFilter(filter: RecordsQueryFilter): RecordsQueryFilter {
+  public static normalizeFilter(filter: RecordsQueryFilter): RecordsQueryFilter {
     let protocol;
     if (filter.protocol === undefined) {
       protocol = undefined;
     } else {
-      protocol = normalizeProtocolUri(filter.protocol);
+      protocol = normalizeProtocolUrl(filter.protocol);
+    }
+
+    let schema;
+    if (filter.schema === undefined) {
+      schema = undefined;
+    } else {
+      schema = normalizeSchemaUrl(filter.schema);
     }
 
     return {
       ...filter,
       protocol,
+      schema,
     };
   }
 }

src/interfaces/records/messages/records-write.ts

@@ -28,7 +28,7 @@ import { authorize, validateAuthorizationIntegrity } from '../../../core/auth.js
 import { Cid, computeCid } from '../../../utils/cid.js';
 import { DwnError, DwnErrorCode } from '../../../core/dwn-error.js';
 import { DwnInterfaceName, DwnMethodName } from '../../../core/message.js';
-import { normalizeProtocolUri, validateProtocolUriNormalized } from '../../../utils/url.js';
+import { normalizeProtocolUrl, normalizeSchemaUrl, validateProtocolUrlNormalized, validateSchemaUrlNormalized } from '../../../utils/url.js';
 
 export type RecordsWriteOptions = {
   recipient?: string;
@@ -166,10 +166,10 @@ export class RecordsWrite extends Message<RecordsWriteMessage> {
     const descriptor: RecordsWriteDescriptor = {
       interface     : DwnInterfaceName.Records,
       method        : DwnMethodName.Write,
-      protocol      : options.protocol !== undefined ? normalizeProtocolUri(options.protocol) : undefined,
+      protocol      : options.protocol !== undefined ? normalizeProtocolUrl(options.protocol) : undefined,
       protocolPath  : options.protocolPath,
       recipient     : options.recipient!,
-      schema        : options.schema,
+      schema        : options.schema !== undefined ? normalizeSchemaUrl(options.schema) : undefined,
       parentId      : options.parentId,
       dataCid,
       dataSize,
@@ -375,7 +375,10 @@ export class RecordsWrite extends Message<RecordsWriteMessage> {
     }
 
     if (this.message.descriptor.protocol !== undefined) {
-      validateProtocolUriNormalized(this.message.descriptor.protocol);
+      validateProtocolUrlNormalized(this.message.descriptor.protocol);
+    }
+    if (this.message.descriptor.schema !== undefined) {
+      validateSchemaUrlNormalized(this.message.descriptor.schema);
     }
   }
 

src/utils/url.ts

@@ -1,20 +1,44 @@
 import { DwnError, DwnErrorCode } from '../core/dwn-error.js';
 
+export function validateProtocolUrlNormalized(url: string): void {
+  let normalized: string | undefined;
+  try {
+    normalized = normalizeProtocolUrl(url);
+  } catch {
+    normalized = undefined;
+  }
+
+  if (url !== normalized) {
+    throw new DwnError(DwnErrorCode.UrlProtocolNotNormalized, `Protocol URI ${url} must be normalized.`);
+  }
+}
+
+export function normalizeProtocolUrl(url: string): string {
+  // Keeping protocol normalization as a separate function in case
+  // protocol and schema normalization diverge in the future
+  return normalizeUrl(url);
+}
 
-export function validateProtocolUriNormalized(protocol: string): void {
+export function validateSchemaUrlNormalized(url: string): void {
   let normalized: string | undefined;
   try {
-    normalized = normalizeProtocolUri(protocol);
+    normalized = normalizeSchemaUrl(url);
   } catch {
     normalized = undefined;
   }
 
-  if (protocol !== normalized) {
-    throw new DwnError(DwnErrorCode.UrlProtocolNotNormalized, 'Protocol URI must be normalized.');
+  if (url !== normalized) {
+    throw new DwnError(DwnErrorCode.UrlSchemaNotNormalized, `Schema URI ${url} must be normalized.`);
   }
 }
 
-export function normalizeProtocolUri(url: string): string {
+export function normalizeSchemaUrl(url: string): string {
+  // Keeping schema normalization as a separate function in case
+  // protocol and schema normalization diverge in the future
+  return normalizeUrl(url);
+}
+
+function normalizeUrl(url: string): string {
   let fullUrl: string;
   if (/^[^:]+:\/\/./.test(url)) {
     fullUrl = url;

tests/interfaces/protocols/handlers/protocols-configure.spec.ts

@@ -4,6 +4,8 @@ import chaiAsPromised from 'chai-as-promised';
 import sinon from 'sinon';
 import chai, { expect } from 'chai';
 
+import dexProtocolDefinition from '../../../vectors/protocol-definitions/dex.json' assert { type: 'json' };
+
 import { DataStoreLevel } from '../../../../src/store/data-store-level.js';
 import { DidKeyResolver } from '../../../../src/did/did-key-resolver.js';
 import { EventLogLevel } from '../../../../src/event-log/event-log-level.js';
@@ -164,6 +166,30 @@ describe('ProtocolsConfigureHandler.handle()', () => {
       expect(reply.status.detail).to.contain(DwnErrorCode.UrlProtocolNotNormalized);
     });
 
+    it('should return 400 if schema is not normalized', async () => {
+      const alice = await DidKeyResolver.generate();
+
+      const protocolsConfig = await TestDataGenerator.generateProtocolsConfigure({
+        requester          : alice,
+        protocol           : 'example.com/',
+        protocolDefinition : dexProtocolDefinition,
+      });
+
+      // overwrite schema because #create auto-normalizes schema
+      protocolsConfig.message.descriptor.definition.labels.ask.schema = 'ask';
+
+      // Re-create auth because we altered the descriptor after signing
+      protocolsConfig.message.authorization = await Message.signAsAuthorization(
+        protocolsConfig.message.descriptor,
+        Jws.createSignatureInput(alice)
+      );
+
+      // Send records write message
+      const reply = await dwn.processMessage(alice.did, protocolsConfig.message);
+      expect(reply.status.code).to.equal(400);
+      expect(reply.status.detail).to.contain(DwnErrorCode.UrlSchemaNotNormalized);
+    });
+
     describe('event log', () => {
       it('should add event for ProtocolsConfigure', async () => {
         const alice = await DidKeyResolver.generate();

tests/interfaces/protocols/messages/protocols-configure.spec.ts

@@ -43,6 +43,27 @@ describe('ProtocolsConfigure', () => {
 
       expect(message.descriptor.protocol).to.eq('http://example.com');
     });
+
+    it('should auto-normalize schema URIs', async () => {
+      const alice = await TestDataGenerator.generatePersona();
+
+      const nonnormalizedDexProtocol = { ...dexProtocolDefinition };
+      nonnormalizedDexProtocol.labels.ask.schema = 'ask';
+
+      const options = {
+        recipient                   : alice.did,
+        data                        : TestDataGenerator.randomBytes(10),
+        dataFormat                  : 'application/json',
+        authorizationSignatureInput : Jws.createSignatureInput(alice),
+        protocol                    : 'example.com/',
+        definition                  : nonnormalizedDexProtocol
+      };
+      const protocolsConfig = await ProtocolsConfigure.create(options);
+
+      const message = protocolsConfig.message as ProtocolsConfigureMessage;
+
+      expect(message.descriptor.definition.labels.ask.schema).to.eq('http://ask');
+    });
   });
 });
 

tests/interfaces/records/handlers/records-query.spec.ts

@@ -616,6 +616,30 @@ describe('RecordsQueryHandler.handle()', () => {
       expect(reply.status.detail).to.contain(DwnErrorCode.UrlProtocolNotNormalized);
     });
 
+    it('should return 400 if schema is not normalized', async () => {
+      const alice = await DidKeyResolver.generate();
+
+      // query for non-normalized schema
+      const recordsQuery = await TestDataGenerator.generateRecordsQuery({
+        requester : alice,
+        filter    : { schema: 'example.com/' },
+      });
+
+      // overwrite schema because #create auto-normalizes schema
+      recordsQuery.message.descriptor.filter.schema = 'example.com/';
+
+      // Re-create auth because we altered the descriptor after signing
+      recordsQuery.message.authorization = await Message.signAsAuthorization(
+        recordsQuery.message.descriptor,
+        Jws.createSignatureInput(alice)
+      );
+
+      // Send records write message
+      const reply = await dwn.processMessage(alice.did, recordsQuery.message);
+      expect(reply.status.code).to.equal(400);
+      expect(reply.status.detail).to.contain(DwnErrorCode.UrlSchemaNotNormalized);
+    });
+
     describe('encryption scenarios', () => {
       it('should only be able to decrypt record with a correct derived private key - protocols derivation scheme', async () => {
         // scenario, Bob writes into Alice's DWN an encrypted "email", alice is able to decrypt it