Skip to content

Fix pagination cursor type validation #831

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+111 −71
Open

Fix pagination cursor type validation #831

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 44 additions & 69 deletions build/compile-validators.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,78 +17,53 @@ import Ajv from 'ajv/dist/2020.js';
import mkdirp from 'mkdirp';
import standaloneCode from 'ajv/dist/standalone/index.js';

import Authorization from '../json-schemas/authorization.json' assert { type: 'json' };
import AuthorizationDelegatedGrant from '../json-schemas/authorization-delegated-grant.json' assert { type: 'json' };
import AuthorizationOwner from '../json-schemas/authorization-owner.json' assert { type: 'json' };
import Definitions from '../json-schemas/definitions.json' assert { type: 'json' };
import GeneralJwk from '../json-schemas/jwk/general-jwk.json' assert { type: 'json' };
import GeneralJws from '../json-schemas/general-jws.json' assert { type: 'json' };
import GenericSignaturePayload from '../json-schemas/signature-payloads/generic-signature-payload.json' assert { type: 'json' };
import JwkVerificationMethod from '../json-schemas/jwk-verification-method.json' assert { type: 'json' };
import MessagesFilter from '../json-schemas/interface-methods/messages-filter.json' assert { type: 'json' };
import MessagesQuery from '../json-schemas/interface-methods/messages-query.json' assert { type: 'json' };
import MessagesRead from '../json-schemas/interface-methods/messages-read.json' assert { type: 'json' };
import MessagesSubscribe from '../json-schemas/interface-methods/messages-subscribe.json' assert { type: 'json' };
import NumberRangeFilter from '../json-schemas/interface-methods/number-range-filter.json' assert { type: 'json' };
import PaginationCursor from '../json-schemas/interface-methods/pagination-cursor.json' assert { type: 'json' };
import PermissionGrantData from '../json-schemas/permissions/permission-grant-data.json' assert { type: 'json' };
import PermissionRequestData from '../json-schemas/permissions/permission-request-data.json' assert { type: 'json' };
import PermissionRevocationData from '../json-schemas/permissions/permission-revocation-data.json' assert { type: 'json' };
import PermissionsDefinitions from '../json-schemas/permissions/permissions-definitions.json' assert { type: 'json' };
import PermissionsScopes from '../json-schemas/permissions/scopes.json' assert { type: 'json' };
import ProtocolDefinition from '../json-schemas/interface-methods/protocol-definition.json' assert { type: 'json' };
import ProtocolRuleSet from '../json-schemas/interface-methods/protocol-rule-set.json' assert { type: 'json' };
import ProtocolsConfigure from '../json-schemas/interface-methods/protocols-configure.json' assert { type: 'json' };
import ProtocolsQuery from '../json-schemas/interface-methods/protocols-query.json' assert { type: 'json' };
import PublicJwk from '../json-schemas/jwk/public-jwk.json' assert { type: 'json' };
import RecordsDelete from '../json-schemas/interface-methods/records-delete.json' assert { type: 'json' };
import RecordsFilter from '../json-schemas/interface-methods/records-filter.json' assert { type: 'json' };
import RecordsQuery from '../json-schemas/interface-methods/records-query.json' assert { type: 'json' };
import RecordsRead from '../json-schemas/interface-methods/records-read.json' assert { type: 'json' };
import RecordsSubscribe from '../json-schemas/interface-methods/records-subscribe.json' assert { type: 'json' };
import RecordsWrite from '../json-schemas/interface-methods/records-write.json' assert { type: 'json' };
import RecordsWriteDataEncoded from '../json-schemas/interface-methods/records-write-data-encoded.json' assert { type: 'json' };
import RecordsWriteSignaturePayload from '../json-schemas/signature-payloads/records-write-signature-payload.json' assert { type: 'json' };
import RecordsWriteUnidentified from '../json-schemas/interface-methods/records-write-unidentified.json' assert { type: 'json' };
import StringRangeFilter from '../json-schemas/interface-methods/string-range-filter.json' assert { type: 'json' };
const loadJson = (relativePath) => {
// Node 22 replaces JSON import assertions with `with` blocks; load via fs to keep compatibility.
const fileUrl = new URL(relativePath, import.meta.url);
return JSON.parse(fs.readFileSync(fileUrl, 'utf-8'));
};

const schemas = {
Authorization,
AuthorizationDelegatedGrant,
AuthorizationOwner,
RecordsDelete,
RecordsQuery,
RecordsSubscribe,
RecordsWrite,
RecordsWriteDataEncoded,
RecordsWriteUnidentified,
Definitions,
GeneralJwk,
GeneralJws,
JwkVerificationMethod,
MessagesFilter,
MessagesQuery,
MessagesRead,
MessagesSubscribe,
NumberRangeFilter,
PaginationCursor,
PermissionGrantData,
PermissionRequestData,
PermissionRevocationData,
PermissionsDefinitions,
PermissionsScopes,
ProtocolDefinition,
ProtocolRuleSet,
ProtocolsConfigure,
ProtocolsQuery,
RecordsRead,
RecordsFilter,
PublicJwk,
GenericSignaturePayload,
RecordsWriteSignaturePayload,
StringRangeFilter
const schemaPaths = {
Authorization: '../json-schemas/authorization.json',
AuthorizationDelegatedGrant: '../json-schemas/authorization-delegated-grant.json',
AuthorizationOwner: '../json-schemas/authorization-owner.json',
RecordsDelete: '../json-schemas/interface-methods/records-delete.json',
RecordsQuery: '../json-schemas/interface-methods/records-query.json',
RecordsSubscribe: '../json-schemas/interface-methods/records-subscribe.json',
RecordsWrite: '../json-schemas/interface-methods/records-write.json',
RecordsWriteDataEncoded: '../json-schemas/interface-methods/records-write-data-encoded.json',
RecordsWriteUnidentified: '../json-schemas/interface-methods/records-write-unidentified.json',
Definitions: '../json-schemas/definitions.json',
GeneralJwk: '../json-schemas/jwk/general-jwk.json',
GeneralJws: '../json-schemas/general-jws.json',
JwkVerificationMethod: '../json-schemas/jwk-verification-method.json',
MessagesFilter: '../json-schemas/interface-methods/messages-filter.json',
MessagesQuery: '../json-schemas/interface-methods/messages-query.json',
MessagesRead: '../json-schemas/interface-methods/messages-read.json',
MessagesSubscribe: '../json-schemas/interface-methods/messages-subscribe.json',
NumberRangeFilter: '../json-schemas/interface-methods/number-range-filter.json',
PaginationCursor: '../json-schemas/interface-methods/pagination-cursor.json',
PermissionGrantData: '../json-schemas/permissions/permission-grant-data.json',
PermissionRequestData: '../json-schemas/permissions/permission-request-data.json',
PermissionRevocationData: '../json-schemas/permissions/permission-revocation-data.json',
PermissionsDefinitions: '../json-schemas/permissions/permissions-definitions.json',
PermissionsScopes: '../json-schemas/permissions/scopes.json',
ProtocolDefinition: '../json-schemas/interface-methods/protocol-definition.json',
ProtocolRuleSet: '../json-schemas/interface-methods/protocol-rule-set.json',
ProtocolsConfigure: '../json-schemas/interface-methods/protocols-configure.json',
ProtocolsQuery: '../json-schemas/interface-methods/protocols-query.json',
RecordsRead: '../json-schemas/interface-methods/records-read.json',
RecordsFilter: '../json-schemas/interface-methods/records-filter.json',
PublicJwk: '../json-schemas/jwk/public-jwk.json',
GenericSignaturePayload: '../json-schemas/signature-payloads/generic-signature-payload.json',
RecordsWriteSignaturePayload: '../json-schemas/signature-payloads/records-write-signature-payload.json',
StringRangeFilter: '../json-schemas/interface-methods/string-range-filter.json'
};

const schemas = Object.fromEntries(
Object.entries(schemaPaths).map(([schemaName, relativePath]) => [schemaName, loadJson(relativePath)])
);

const ajv = new Ajv({ code: { source: true, esm: true }, allowUnionTypes: true });

for (const schemaName in schemas) {
Expand Down
45 changes: 44 additions & 1 deletion src/store/index-level.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,11 @@ export class IndexLevel {
queryOptions: QueryOptions,
options?: IndexLevelOptions
): Promise<IndexedItem[]> {
const { cursor: queryCursor , limit } = queryOptions;
const { cursor: queryCursor , limit, sortProperty } = queryOptions;

if (queryCursor !== undefined) {
await this.assertCursorValueMatchesSortProperty(tenant, queryCursor, sortProperty);
}

// if there is a cursor we fetch the starting key given the sort property, otherwise we start from the beginning of the index.
const startKey = queryCursor ? this.createStartingKeyFromCursor(queryCursor) : '';
Expand Down Expand Up @@ -318,6 +322,41 @@ export class IndexLevel {
return IndexLevel.keySegmentJoin(IndexLevel.encodeValue(value), messageCid);
}

private async assertCursorValueMatchesSortProperty(tenant: string, cursor: PaginationCursor, sortProperty: string): Promise<void> {
const indexes = await this.getIndexes(tenant, cursor.messageCid);
if (indexes === undefined) {
// if the messageCid cannot be resolved we cannot validate the expected type,
// preserve existing behavior and allow the query to proceed.
return;
}

const sortValue = indexes[sortProperty];
if (sortValue === undefined) {
throw new DwnError(
DwnErrorCode.IndexInvalidCursorSortProperty,
`the sort property '${sortProperty}' is not defined within the given item.`
);
}

const sortValueIsArray = Array.isArray(sortValue);
const sortValueType = sortValueIsArray ? 'array' : typeof sortValue;

if (sortValueType === 'boolean' || sortValueIsArray) {
throw new DwnError(
DwnErrorCode.IndexInvalidCursorValueType,
`only string or number values are supported for cursors, a(n) ${sortValueType} was defined for sort property '${sortProperty}'.`
);
}

const cursorValueType = typeof cursor.value;
if (cursorValueType !== sortValueType) {
throw new DwnError(
DwnErrorCode.IndexInvalidCursorValueType,
`cursor value type '${cursorValueType}' does not match expected '${sortValueType}' for sort property '${sortProperty}'.`
);
}
}

/**
* Returns a PaginationCursor using the last item of a given array of IndexedItems.
* If the given array is empty, undefined is returned.
Expand Down Expand Up @@ -370,6 +409,10 @@ export class IndexLevel {
): Promise<IndexedItem[]> {
const { sortProperty, sortDirection = SortDirection.Ascending, cursor: queryCursor, limit } = queryOptions;

if (queryCursor !== undefined) {
await this.assertCursorValueMatchesSortProperty(tenant, queryCursor, sortProperty);
}

// we get the cursor start key here so that we match the failing behavior of `queryWithIteratorPaging`
const cursorStartingKey = queryCursor ? this.createStartingKeyFromCursor(queryCursor) : undefined;

Expand Down
24 changes: 23 additions & 1 deletion tests/store/index-level.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,17 @@ describe('IndexLevel', () => {
const noResults = await testIndex.queryWithIteratorPaging(tenant, filters, { sortProperty: 'val', cursor: cursorE });
expect(noResults.length).to.eql(0);
});

it('throws when cursor value type does not match sort property type', async () => {
await testIndex.put(tenant, 'alpha', { schema: 'schema', val: 1 });
await testIndex.put(tenant, 'beta', { schema: 'schema', val: 2 });

const filters = [{ schema: 'schema' }];
const mismatchedCursor = { messageCid: 'alpha', value: '1' };

const queryPromise = testIndex.queryWithIteratorPaging(tenant, filters, { sortProperty: 'val', cursor: mismatchedCursor });
await expect(queryPromise).to.eventually.be.rejectedWith(DwnErrorCode.IndexInvalidCursorValueType);
});
});

describe('array values', () => {
Expand Down Expand Up @@ -421,6 +432,17 @@ describe('IndexLevel', () => {
expect(noResults.length).to.eql(0);
});

it('throws when in-memory cursor value type does not match sort property type', async () => {
await testIndex.put(tenant, 'alpha', { schema: 'schema', val: 1 });
await testIndex.put(tenant, 'beta', { schema: 'schema', val: 2 });

const filters = [{ schema: 'schema' }];
const mismatchedCursor = { messageCid: 'alpha', value: '1' };

const queryPromise = testIndex.queryWithInMemoryPaging(tenant, filters, { sortProperty: 'val', cursor: mismatchedCursor });
await expect(queryPromise).to.eventually.be.rejectedWith(DwnErrorCode.IndexInvalidCursorValueType);
});

it('supports range queries', async () => {
const id = uuid();
const doc1 = {
Expand Down Expand Up @@ -1477,4 +1499,4 @@ describe('IndexLevel', () => {
expect(IndexLevel.isFilterConcise({ protocol: 'protocol', protocolPath: 'path/to' }, queryOptionsWithoutCursor)).to.equal(true);
});
});
});
});