diff --git a/.changeset/smart-worlds-wait.md b/.changeset/smart-worlds-wait.md new file mode 100644 index 00000000..3b46d691 --- /dev/null +++ b/.changeset/smart-worlds-wait.md @@ -0,0 +1,5 @@ +--- +"@cipherstash/protect": minor +--- + +Added support for deeply nested protect schemas to support more complex model objects. diff --git a/docs/reference/schema.md b/docs/reference/schema.md index 6a57b3ad..b828bdf4 100644 --- a/docs/reference/schema.md +++ b/docs/reference/schema.md @@ -8,6 +8,7 @@ Protect.js lets you define a schema in TypeScript with properties that map to yo - [Understanding schema files](#understanding-schema-files) - [Defining your schema](#defining-your-schema) - [Searchable encryption](#searchable-encryption) + - [Nested objects](#nested-objects) - [Available index options](#available-index-options) - [Initializing the Protect client](#initializing-the-protect-client) @@ -75,6 +76,45 @@ export const protectedUsers = csTable("users", { }); ``` +### Nested objects + +Protect.js supports nested objects in your schema, allowing you to encrypt **but not search on** nested properties. You can define nested objects up to 3 levels deep. +This is useful for data stores that have less structured data, like NoSQL databases. + +You can define nested objects by using the `csValue` function to define a value in a nested object. The value naming convention of the `csValue` function is a dot-separated string of the nested object path, e.g. `profile.name` or `profile.address.street`. + +> [!NOTE] +> Using nested objects is not recommended for SQL databases, as it will not be searchable. +> You should either use a JSON data type and encrypt the entire object, or use a separate column for each nested property. + +```ts +import { csTable, csColumn, csValue } from "@cipherstash/protect"; + +export const protectedUsers = csTable("users", { + email: csColumn("email").freeTextSearch().equality().orderAndRange(), + profile: { + name: csValue("profile.name"), + address: { + street: csValue("profile.address.street"), + location: { + coordinates: csValue("profile.address.location.coordinates"), + }, + }, + }, +}); +``` + +When working with nested objects: +- Searchable encryption is not supported on nested objects +- Each level can have its own encrypted fields +- The maximum nesting depth is 3 levels +- Null and undefined values are supported at any level +- Optional nested objects are supported + +> [!WARNING] +> TODO: The schema builder does not validate the values you supply to the `csValue` or `csColumn` functions. +> These values are meant to be unique, and and cause unexpected behavior if they are not defined correctly. + ## Available index options The following index options are available for your schema: diff --git a/packages/protect/__tests__/nested-models.test.ts b/packages/protect/__tests__/nested-models.test.ts new file mode 100644 index 00000000..e245b4a5 --- /dev/null +++ b/packages/protect/__tests__/nested-models.test.ts @@ -0,0 +1,958 @@ +import 'dotenv/config' +import { describe, expect, it, vi } from 'vitest' + +import { LockContext, protect, csTable, csColumn, csValue } from '../src' + +const users = csTable('users', { + email: csColumn('email').freeTextSearch().equality().orderAndRange(), + address: csColumn('address').freeTextSearch(), + name: csColumn('name').freeTextSearch(), + example: { + field: csValue('example.field'), + nested: { + deeper: csValue('example.nested.deeper'), + }, + }, +}) + +type User = { + id: string + email?: string | null + createdAt?: Date + updatedAt?: Date + address?: string | null + notEncrypted?: string | null + example: { + field: string | undefined | null + nested?: { + deeper: string | undefined | null + plaintext?: string | undefined | null + notInSchema?: { + deeper: string | undefined | null + } + deeperNotInSchema?: string | undefined | null + extra?: { + plaintext: string | undefined | null + } + } + plaintext?: string | undefined | null + fieldNotInSchema?: string | undefined | null + notInSchema?: { + deeper: string | undefined | null + } + } +} + +describe('encrypt models with nested fields', () => { + it('should encrypt and decrypt a single value from a nested schema', async () => { + const protectClient = await protect({ schemas: [users] }) + + const encryptResponse = await protectClient.encrypt('hello world', { + column: users.example.field, + table: users, + }) + + if (encryptResponse.failure) { + throw new Error(`[protect]: ${encryptResponse.failure.message}`) + } + + // Verify encrypted field + expect(encryptResponse.data).toHaveProperty('c') + + const decryptResponse = await protectClient.decrypt(encryptResponse.data) + + if (decryptResponse.failure) { + throw new Error(`[protect]: ${decryptResponse.failure.message}`) + } + + expect(decryptResponse).toEqual({ + data: 'hello world', + }) + }) + + it('should encrypt and decrypt a model with nested fields', async () => { + const protectClient = await protect({ schemas: [users] }) + + const decryptedModel = { + id: '1', + email: 'test@example.com', + address: '123 Main St', + notEncrypted: 'not encrypted', + createdAt: new Date('2021-01-01'), + updatedAt: new Date('2021-01-01'), + example: { + field: 'test', + }, + } + + const encryptedModel = await protectClient.encryptModel( + decryptedModel, + users, + ) + + if (encryptedModel.failure) { + throw new Error(`[protect]: ${encryptedModel.failure.message}`) + } + + // Verify encrypted fields + expect(encryptedModel.data.email).toHaveProperty('c') + expect(encryptedModel.data.address).toHaveProperty('c') + expect(encryptedModel.data.example.field).toHaveProperty('c') + + // Verify non-encrypted fields remain unchanged + expect(encryptedModel.data.id).toBe('1') + expect(encryptedModel.data.notEncrypted).toBe('not encrypted') + expect(encryptedModel.data.createdAt).toEqual(new Date('2021-01-01')) + expect(encryptedModel.data.updatedAt).toEqual(new Date('2021-01-01')) + + const decryptedResult = await protectClient.decryptModel( + encryptedModel.data, + ) + + if (decryptedResult.failure) { + throw new Error(`[protect]: ${decryptedResult.failure.message}`) + } + + expect(decryptedResult.data).toEqual(decryptedModel) + }, 30000) + + it('should handle null values in nested fields', async () => { + const protectClient = await protect({ schemas: [users] }) + + const decryptedModel = { + id: '2', + email: null, + address: null, + example: { + field: null, + nested: { + deeper: null, + }, + }, + } + + const encryptedModel = await protectClient.encryptModel( + decryptedModel, + users, + ) + + if (encryptedModel.failure) { + throw new Error(`[protect]: ${encryptedModel.failure.message}`) + } + + // Verify null fields are preserved + expect(encryptedModel.data.email).toBeNull() + expect(encryptedModel.data.address).toBeNull() + expect(encryptedModel.data.example.field).toBeNull() + expect(encryptedModel.data.example.nested?.deeper).toBeNull() + + const decryptedResult = await protectClient.decryptModel( + encryptedModel.data, + ) + + if (decryptedResult.failure) { + throw new Error(`[protect]: ${decryptedResult.failure.message}`) + } + + expect(decryptedResult.data).toEqual(decryptedModel) + }, 30000) + + it('should handle undefined values in nested fields', async () => { + const protectClient = await protect({ schemas: [users] }) + + const decryptedModel = { + id: '3', + example: { + field: undefined, + nested: { + deeper: undefined, + }, + }, + } + + const encryptedModel = await protectClient.encryptModel( + decryptedModel, + users, + ) + + if (encryptedModel.failure) { + throw new Error(`[protect]: ${encryptedModel.failure.message}`) + } + + // Verify undefined fields are preserved + expect(encryptedModel.data.email).toBeUndefined() + expect(encryptedModel.data.example.field).toBeUndefined() + expect(encryptedModel.data.example.nested?.deeper).toBeUndefined() + + const decryptedResult = await protectClient.decryptModel( + encryptedModel.data, + ) + + if (decryptedResult.failure) { + throw new Error(`[protect]: ${decryptedResult.failure.message}`) + } + + expect(decryptedResult.data).toEqual(decryptedModel) + }, 30000) + + it('should handle mixed null and undefined values in nested fields', async () => { + const protectClient = await protect({ schemas: [users] }) + + const decryptedModel = { + id: '4', + email: 'test@example.com', + address: undefined, + notEncrypted: 'not encrypted', + createdAt: new Date('2021-01-01'), + updatedAt: new Date('2021-01-01'), + example: { + field: null, + nested: { + deeper: undefined, + }, + }, + } + + const encryptedModel = await protectClient.encryptModel( + decryptedModel, + users, + ) + + if (encryptedModel.failure) { + throw new Error(`[protect]: ${encryptedModel.failure.message}`) + } + + // Verify encrypted fields + expect(encryptedModel.data.email).toHaveProperty('c') + + // Verify null/undefined fields are preserved + expect(encryptedModel.data.address).toBeUndefined() + expect(encryptedModel.data.example.field).toBeNull() + expect(encryptedModel.data.example.nested?.deeper).toBeUndefined() + + // Verify non-encrypted fields remain unchanged + expect(encryptedModel.data.id).toBe('4') + expect(encryptedModel.data.notEncrypted).toBe('not encrypted') + expect(encryptedModel.data.createdAt).toEqual(new Date('2021-01-01')) + expect(encryptedModel.data.updatedAt).toEqual(new Date('2021-01-01')) + + const decryptedResult = await protectClient.decryptModel( + encryptedModel.data, + ) + + if (decryptedResult.failure) { + throw new Error(`[protect]: ${decryptedResult.failure.message}`) + } + + expect(decryptedResult.data).toEqual(decryptedModel) + }, 30000) + + it('should handle deeply nested fields', async () => { + const protectClient = await protect({ schemas: [users] }) + + const decryptedModel = { + id: '3', + example: { + field: 'outer', + nested: { + deeper: 'inner value', + }, + }, + } + + const encryptedModel = await protectClient.encryptModel( + decryptedModel, + users, + ) + + if (encryptedModel.failure) { + throw new Error(`[protect]: ${encryptedModel.failure.message}`) + } + + // Verify encrypted fields + expect(encryptedModel.data.example.field).toHaveProperty('c') + expect(encryptedModel.data.example.nested?.deeper).toHaveProperty('c') + + // Verify non-encrypted fields remain unchanged + expect(encryptedModel.data.id).toBe('3') + + const decryptedResult = await protectClient.decryptModel( + encryptedModel.data, + ) + + if (decryptedResult.failure) { + throw new Error(`[protect]: ${decryptedResult.failure.message}`) + } + + expect(decryptedResult.data).toEqual(decryptedModel) + }, 30000) + + it('should handle missing optional nested fields', async () => { + const protectClient = await protect({ schemas: [users] }) + + const decryptedModel = { + id: '5', + example: { + field: 'present', + }, + } + + const encryptedModel = await protectClient.encryptModel( + decryptedModel, + users, + ) + + if (encryptedModel.failure) { + throw new Error(`[protect]: ${encryptedModel.failure.message}`) + } + + // Verify encrypted fields + expect(encryptedModel.data.example.field).toHaveProperty('c') + + // Verify non-encrypted fields remain unchanged + expect(encryptedModel.data.id).toBe('5') + expect(encryptedModel.data.example.nested).toBeUndefined() + + const decryptedResult = await protectClient.decryptModel( + encryptedModel.data, + ) + + if (decryptedResult.failure) { + throw new Error(`[protect]: ${decryptedResult.failure.message}`) + } + + expect(decryptedResult.data).toEqual(decryptedModel) + }, 30000) + + describe('bulk operations with nested fields', () => { + it('should handle bulk encryption and decryption of models with nested fields', async () => { + const protectClient = await protect({ schemas: [users] }) + + const decryptedModels: User[] = [ + { + id: '1', + email: 'test1@example.com', + example: { + field: 'test1', + nested: { + deeper: 'value1', + }, + }, + }, + { + id: '2', + email: 'test2@example.com', + example: { + field: 'test2', + nested: { + deeper: 'value2', + }, + }, + }, + ] + + const encryptedModels = await protectClient.bulkEncryptModels( + decryptedModels, + users, + ) + + if (encryptedModels.failure) { + throw new Error(`[protect]: ${encryptedModels.failure.message}`) + } + + // Verify encrypted fields for each model + expect(encryptedModels.data[0].email).toHaveProperty('c') + expect(encryptedModels.data[0].example.field).toHaveProperty('c') + expect(encryptedModels.data[0].example.nested?.deeper).toHaveProperty('c') + expect(encryptedModels.data[1].email).toHaveProperty('c') + expect(encryptedModels.data[1].example.field).toHaveProperty('c') + expect(encryptedModels.data[1].example.nested?.deeper).toHaveProperty('c') + + // Verify non-encrypted fields remain unchanged + expect(encryptedModels.data[0].id).toBe('1') + expect(encryptedModels.data[1].id).toBe('2') + + const decryptedResults = await protectClient.bulkDecryptModels( + encryptedModels.data, + ) + + if (decryptedResults.failure) { + throw new Error(`[protect]: ${decryptedResults.failure.message}`) + } + + expect(decryptedResults.data).toEqual(decryptedModels) + }, 30000) + + it('should handle bulk operations with null and undefined values in nested fields', async () => { + const protectClient = await protect({ schemas: [users] }) + + const decryptedModels: User[] = [ + { + id: '1', + email: null, + example: { + field: null, + nested: { + deeper: undefined, + }, + }, + }, + { + id: '2', + email: undefined, + example: { + field: undefined, + nested: { + deeper: null, + }, + }, + }, + ] + + const encryptedModels = await protectClient.bulkEncryptModels( + decryptedModels, + users, + ) + + if (encryptedModels.failure) { + throw new Error(`[protect]: ${encryptedModels.failure.message}`) + } + + // Verify null/undefined fields are preserved + expect(encryptedModels.data[0].email).toBeNull() + expect(encryptedModels.data[0].example.field).toBeNull() + expect(encryptedModels.data[0].example.nested?.deeper).toBeUndefined() + expect(encryptedModels.data[1].email).toBeUndefined() + expect(encryptedModels.data[1].example.field).toBeUndefined() + expect(encryptedModels.data[1].example.nested?.deeper).toBeNull() + + // Verify non-encrypted fields remain unchanged + expect(encryptedModels.data[0].id).toBe('1') + expect(encryptedModels.data[1].id).toBe('2') + + const decryptedResults = await protectClient.bulkDecryptModels( + encryptedModels.data, + ) + + if (decryptedResults.failure) { + throw new Error(`[protect]: ${decryptedResults.failure.message}`) + } + + expect(decryptedResults.data).toEqual(decryptedModels) + }, 30000) + + it('should handle bulk operations with missing optional nested fields', async () => { + const protectClient = await protect({ schemas: [users] }) + + const decryptedModels: User[] = [ + { + id: '1', + email: 'test1@example.com', + example: { + field: 'test1', + }, + }, + { + id: '2', + email: 'test2@example.com', + example: { + field: 'test2', + nested: { + deeper: 'value2', + }, + }, + }, + ] + + const encryptedModels = await protectClient.bulkEncryptModels( + decryptedModels, + users, + ) + + if (encryptedModels.failure) { + throw new Error(`[protect]: ${encryptedModels.failure.message}`) + } + + // Verify encrypted fields for each model + expect(encryptedModels.data[0].email).toHaveProperty('c') + expect(encryptedModels.data[0].example.field).toHaveProperty('c') + expect(encryptedModels.data[1].email).toHaveProperty('c') + expect(encryptedModels.data[1].example.field).toHaveProperty('c') + expect(encryptedModels.data[1].example.nested?.deeper).toHaveProperty('c') + + // Verify non-encrypted fields remain unchanged + expect(encryptedModels.data[0].id).toBe('1') + expect(encryptedModels.data[0].example.nested).toBeUndefined() + expect(encryptedModels.data[1].id).toBe('2') + + const decryptedResults = await protectClient.bulkDecryptModels( + encryptedModels.data, + ) + + if (decryptedResults.failure) { + throw new Error(`[protect]: ${decryptedResults.failure.message}`) + } + + expect(decryptedResults.data).toEqual(decryptedModels) + }, 30000) + + it('should handle empty array in bulk operations', async () => { + const protectClient = await protect({ schemas: [users] }) + + const decryptedModels: User[] = [] + + const encryptedModels = await protectClient.bulkEncryptModels( + decryptedModels, + users, + ) + + if (encryptedModels.failure) { + throw new Error(`[protect]: ${encryptedModels.failure.message}`) + } + + expect(encryptedModels.data).toEqual([]) + + const decryptedResults = await protectClient.bulkDecryptModels( + encryptedModels.data, + ) + + if (decryptedResults.failure) { + throw new Error(`[protect]: ${decryptedResults.failure.message}`) + } + + expect(decryptedResults.data).toEqual([]) + }, 30000) + }) +}) + +describe('nested fields with a plaintext field', () => { + it('should handle nested fields with a plaintext field', async () => { + const protectClient = await protect({ schemas: [users] }) + + const decryptedModel = { + id: '1', + email: 'test@example.com', + address: '123 Main St', + notEncrypted: 'not encrypted', + createdAt: new Date('2021-01-01'), + updatedAt: new Date('2021-01-01'), + example: { + field: 'test', + plaintext: 'plaintext', + }, + } + + const encryptedModel = await protectClient.encryptModel( + decryptedModel, + users, + ) + + if (encryptedModel.failure) { + throw new Error(`[protect]: ${encryptedModel.failure.message}`) + } + + // Verify encrypted fields + expect(encryptedModel.data.email).toHaveProperty('c') + expect(encryptedModel.data.address).toHaveProperty('c') + expect(encryptedModel.data.example.field).toHaveProperty('c') + + // Verify non-encrypted fields remain unchanged + expect(encryptedModel.data.id).toBe('1') + expect(encryptedModel.data.notEncrypted).toBe('not encrypted') + expect(encryptedModel.data.createdAt).toEqual(new Date('2021-01-01')) + expect(encryptedModel.data.updatedAt).toEqual(new Date('2021-01-01')) + expect(encryptedModel.data.example.plaintext).toBe('plaintext') + + const decryptedResult = await protectClient.decryptModel( + encryptedModel.data, + ) + + if (decryptedResult.failure) { + throw new Error(`[protect]: ${decryptedResult.failure.message}`) + } + + expect(decryptedResult.data).toEqual(decryptedModel) + }) + + it('should handle multiple plaintext fields at different nesting levels', async () => { + const protectClient = await protect({ schemas: [users] }) + + const decryptedModel = { + id: '1', + email: 'test@example.com', + address: '123 Main St', + notEncrypted: 'not encrypted', + example: { + field: 'encrypted field', + plaintext: 'top level plaintext', + nested: { + deeper: 'encrypted deeper', + plaintext: 'nested plaintext', + extra: { + plaintext: 'deeply nested plaintext', + }, + }, + }, + } + + const encryptedModel = await protectClient.encryptModel( + decryptedModel, + users, + ) + + if (encryptedModel.failure) { + throw new Error(`[protect]: ${encryptedModel.failure.message}`) + } + + // Verify encrypted fields + expect(encryptedModel.data.email).toHaveProperty('c') + expect(encryptedModel.data.address).toHaveProperty('c') + expect(encryptedModel.data.example.field).toHaveProperty('c') + expect(encryptedModel.data.example.nested?.deeper).toHaveProperty('c') + + // Verify non-encrypted fields remain unchanged + expect(encryptedModel.data.id).toBe('1') + expect(encryptedModel.data.notEncrypted).toBe('not encrypted') + expect(encryptedModel.data.example.plaintext).toBe('top level plaintext') + expect(encryptedModel.data.example.nested?.plaintext).toBe( + 'nested plaintext', + ) + expect(encryptedModel.data.example.nested?.extra?.plaintext).toBe( + 'deeply nested plaintext', + ) + + const decryptedResult = await protectClient.decryptModel( + encryptedModel.data, + ) + + if (decryptedResult.failure) { + throw new Error(`[protect]: ${decryptedResult.failure.message}`) + } + + expect(decryptedResult.data).toEqual(decryptedModel) + }) + + it('should handle partial path matches in nested objects', async () => { + const protectClient = await protect({ schemas: [users] }) + + const decryptedModel = { + id: '1', + email: 'test@example.com', + example: { + field: 'encrypted field', + nested: { + deeper: 'encrypted deeper', + // This should not be encrypted as it's not in the schema + notInSchema: { + deeper: 'not encrypted', + }, + }, + // This should not be encrypted as it's not in the schema + notInSchema: { + deeper: 'not encrypted', + }, + }, + } + + const encryptedModel = await protectClient.encryptModel( + decryptedModel, + users, + ) + + if (encryptedModel.failure) { + throw new Error(`[protect]: ${encryptedModel.failure.message}`) + } + + // Verify encrypted fields + expect(encryptedModel.data.email).toHaveProperty('c') + expect(encryptedModel.data.example.field).toHaveProperty('c') + expect(encryptedModel.data.example.nested?.deeper).toHaveProperty('c') + + // Verify non-encrypted fields remain unchanged + expect(encryptedModel.data.id).toBe('1') + expect(encryptedModel.data.example.nested?.notInSchema?.deeper).toBe( + 'not encrypted', + ) + expect(encryptedModel.data.example.notInSchema?.deeper).toBe( + 'not encrypted', + ) + + const decryptedResult = await protectClient.decryptModel( + encryptedModel.data, + ) + + if (decryptedResult.failure) { + throw new Error(`[protect]: ${decryptedResult.failure.message}`) + } + + expect(decryptedResult.data).toEqual(decryptedModel) + }) + + it('should handle mixed encrypted and plaintext fields with similar paths', async () => { + const protectClient = await protect({ schemas: [users] }) + + const decryptedModel = { + id: '1', + email: 'test@example.com', + example: { + field: 'encrypted field', + fieldNotInSchema: 'not encrypted', + nested: { + deeper: 'encrypted deeper', + deeperNotInSchema: 'not encrypted', + }, + }, + } + + const encryptedModel = await protectClient.encryptModel( + decryptedModel, + users, + ) + + if (encryptedModel.failure) { + throw new Error(`[protect]: ${encryptedModel.failure.message}`) + } + + // Verify encrypted fields + expect(encryptedModel.data.email).toHaveProperty('c') + expect(encryptedModel.data.example.field).toHaveProperty('c') + expect(encryptedModel.data.example.nested?.deeper).toHaveProperty('c') + + // Verify non-encrypted fields remain unchanged + expect(encryptedModel.data.id).toBe('1') + expect(encryptedModel.data.example.fieldNotInSchema).toBe('not encrypted') + expect(encryptedModel.data.example.nested?.deeperNotInSchema).toBe( + 'not encrypted', + ) + + const decryptedResult = await protectClient.decryptModel( + encryptedModel.data, + ) + + if (decryptedResult.failure) { + throw new Error(`[protect]: ${decryptedResult.failure.message}`) + } + + expect(decryptedResult.data).toEqual(decryptedModel) + }) + + describe('bulk operations with plaintext fields', () => { + it('should handle bulk encryption and decryption with plaintext fields', async () => { + const protectClient = await protect({ schemas: [users] }) + + const decryptedModels: User[] = [ + { + id: '1', + email: 'test1@example.com', + address: '123 Main St', + example: { + field: 'encrypted field 1', + plaintext: 'plaintext 1', + nested: { + deeper: 'encrypted deeper 1', + plaintext: 'nested plaintext 1', + }, + }, + }, + { + id: '2', + email: 'test2@example.com', + address: '456 Main St', + example: { + field: 'encrypted field 2', + plaintext: 'plaintext 2', + nested: { + deeper: 'encrypted deeper 2', + plaintext: 'nested plaintext 2', + }, + }, + }, + ] + + const encryptedModels = await protectClient.bulkEncryptModels( + decryptedModels, + users, + ) + + if (encryptedModels.failure) { + throw new Error(`[protect]: ${encryptedModels.failure.message}`) + } + + // Verify encrypted fields + expect(encryptedModels.data[0].email).toHaveProperty('c') + expect(encryptedModels.data[0].address).toHaveProperty('c') + expect(encryptedModels.data[0].example.field).toHaveProperty('c') + expect(encryptedModels.data[0].example.nested?.deeper).toHaveProperty('c') + expect(encryptedModels.data[1].email).toHaveProperty('c') + expect(encryptedModels.data[1].address).toHaveProperty('c') + expect(encryptedModels.data[1].example.field).toHaveProperty('c') + expect(encryptedModels.data[1].example.nested?.deeper).toHaveProperty('c') + + // Verify non-encrypted fields remain unchanged + expect(encryptedModels.data[0].id).toBe('1') + expect(encryptedModels.data[0].example.plaintext).toBe('plaintext 1') + expect(encryptedModels.data[0].example.nested?.plaintext).toBe( + 'nested plaintext 1', + ) + expect(encryptedModels.data[1].id).toBe('2') + expect(encryptedModels.data[1].example.plaintext).toBe('plaintext 2') + expect(encryptedModels.data[1].example.nested?.plaintext).toBe( + 'nested plaintext 2', + ) + + const decryptedResults = await protectClient.bulkDecryptModels( + encryptedModels.data, + ) + + if (decryptedResults.failure) { + throw new Error(`[protect]: ${decryptedResults.failure.message}`) + } + + expect(decryptedResults.data).toEqual(decryptedModels) + }) + + it('should handle bulk operations with mixed encrypted and non-encrypted fields', async () => { + const protectClient = await protect({ schemas: [users] }) + + const decryptedModels: User[] = [ + { + id: '1', + email: 'test1@example.com', + example: { + field: 'encrypted field 1', + fieldNotInSchema: 'not encrypted 1', + nested: { + deeper: 'encrypted deeper 1', + deeperNotInSchema: 'not encrypted deeper 1', + }, + }, + }, + { + id: '2', + email: 'test2@example.com', + example: { + field: 'encrypted field 2', + fieldNotInSchema: 'not encrypted 2', + nested: { + deeper: 'encrypted deeper 2', + deeperNotInSchema: 'not encrypted deeper 2', + }, + }, + }, + ] + + const encryptedModels = await protectClient.bulkEncryptModels( + decryptedModels, + users, + ) + + if (encryptedModels.failure) { + throw new Error(`[protect]: ${encryptedModels.failure.message}`) + } + + // Verify encrypted fields + expect(encryptedModels.data[0].email).toHaveProperty('c') + expect(encryptedModels.data[0].example.field).toHaveProperty('c') + expect(encryptedModels.data[0].example.nested?.deeper).toHaveProperty('c') + expect(encryptedModels.data[1].email).toHaveProperty('c') + expect(encryptedModels.data[1].example.field).toHaveProperty('c') + expect(encryptedModels.data[1].example.nested?.deeper).toHaveProperty('c') + + // Verify non-encrypted fields remain unchanged + expect(encryptedModels.data[0].id).toBe('1') + expect(encryptedModels.data[0].example.fieldNotInSchema).toBe( + 'not encrypted 1', + ) + expect(encryptedModels.data[0].example.nested?.deeperNotInSchema).toBe( + 'not encrypted deeper 1', + ) + expect(encryptedModels.data[1].id).toBe('2') + expect(encryptedModels.data[1].example.fieldNotInSchema).toBe( + 'not encrypted 2', + ) + expect(encryptedModels.data[1].example.nested?.deeperNotInSchema).toBe( + 'not encrypted deeper 2', + ) + + const decryptedResults = await protectClient.bulkDecryptModels( + encryptedModels.data, + ) + + if (decryptedResults.failure) { + throw new Error(`[protect]: ${decryptedResults.failure.message}`) + } + + expect(decryptedResults.data).toEqual(decryptedModels) + }) + + it('should handle bulk operations with deeply nested plaintext fields', async () => { + const protectClient = await protect({ schemas: [users] }) + + const decryptedModels: User[] = [ + { + id: '1', + email: 'test1@example.com', + example: { + field: 'encrypted field 1', + nested: { + deeper: 'encrypted deeper 1', + extra: { + plaintext: 'deeply nested plaintext 1', + }, + }, + }, + }, + { + id: '2', + email: 'test2@example.com', + example: { + field: 'encrypted field 2', + nested: { + deeper: 'encrypted deeper 2', + extra: { + plaintext: 'deeply nested plaintext 2', + }, + }, + }, + }, + ] + + const encryptedModels = await protectClient.bulkEncryptModels( + decryptedModels, + users, + ) + + if (encryptedModels.failure) { + throw new Error(`[protect]: ${encryptedModels.failure.message}`) + } + + // Verify encrypted fields + expect(encryptedModels.data[0].email).toHaveProperty('c') + expect(encryptedModels.data[0].example.field).toHaveProperty('c') + expect(encryptedModels.data[0].example.nested?.deeper).toHaveProperty('c') + expect(encryptedModels.data[1].email).toHaveProperty('c') + expect(encryptedModels.data[1].example.field).toHaveProperty('c') + expect(encryptedModels.data[1].example.nested?.deeper).toHaveProperty('c') + + // Verify non-encrypted fields remain unchanged + expect(encryptedModels.data[0].id).toBe('1') + expect(encryptedModels.data[0].example.nested?.extra?.plaintext).toBe( + 'deeply nested plaintext 1', + ) + expect(encryptedModels.data[1].id).toBe('2') + expect(encryptedModels.data[1].example.nested?.extra?.plaintext).toBe( + 'deeply nested plaintext 2', + ) + + const decryptedResults = await protectClient.bulkDecryptModels( + encryptedModels.data, + ) + + if (decryptedResults.failure) { + throw new Error(`[protect]: ${decryptedResults.failure.message}`) + } + + expect(decryptedResults.data).toEqual(decryptedModels) + }) + }) +}) diff --git a/packages/protect/__tests__/protect.test.ts b/packages/protect/__tests__/protect.test.ts index 0dc13fd0..c685803e 100644 --- a/packages/protect/__tests__/protect.test.ts +++ b/packages/protect/__tests__/protect.test.ts @@ -32,6 +32,9 @@ describe('encryption and decryption', () => { throw new Error(`[protect]: ${ciphertext.failure.message}`) } + // Verify encrypted field + expect(ciphertext.data).toHaveProperty('c') + const plaintext = await protectClient.decrypt(ciphertext.data) expect(plaintext).toEqual({ @@ -51,6 +54,9 @@ describe('encryption and decryption', () => { throw new Error(`[protect]: ${ciphertext.failure.message}`) } + // Verify null is preserved + expect(ciphertext.data).toBeNull() + const plaintext = await protectClient.decrypt(ciphertext.data) expect(plaintext).toEqual({ @@ -81,6 +87,16 @@ describe('encryption and decryption', () => { throw new Error(`[protect]: ${encryptedModel.failure.message}`) } + // Verify encrypted fields + expect(encryptedModel.data.email).toHaveProperty('c') + expect(encryptedModel.data.address).toHaveProperty('c') + + // Verify non-encrypted fields remain unchanged + expect(encryptedModel.data.id).toBe('1') + expect(encryptedModel.data.createdAt).toEqual(new Date('2021-01-01')) + expect(encryptedModel.data.updatedAt).toEqual(new Date('2021-01-01')) + expect(encryptedModel.data.number).toBe(1) + // Decrypt the model const decryptedResult = await protectClient.decryptModel( encryptedModel.data, @@ -123,6 +139,16 @@ describe('encryption and decryption', () => { throw new Error(`[protect]: ${encryptedModel.failure.message}`) } + // Verify null fields are preserved + expect(encryptedModel.data.email).toBeNull() + expect(encryptedModel.data.address).toBeNull() + + // Verify non-encrypted fields remain unchanged + expect(encryptedModel.data.id).toBe('1') + expect(encryptedModel.data.createdAt).toEqual(new Date('2021-01-01')) + expect(encryptedModel.data.updatedAt).toEqual(new Date('2021-01-01')) + expect(encryptedModel.data.number).toBe(1) + // Decrypt the model const decryptedResult = await protectClient.decryptModel( encryptedModel.data, @@ -165,6 +191,16 @@ describe('encryption and decryption', () => { throw new Error(`[protect]: ${encryptedModel.failure.message}`) } + // Verify undefined fields are preserved + expect(encryptedModel.data.email).toBeUndefined() + expect(encryptedModel.data.address).toBeNull() + + // Verify non-encrypted fields remain unchanged + expect(encryptedModel.data.id).toBe('1') + expect(encryptedModel.data.createdAt).toEqual(new Date('2021-01-01')) + expect(encryptedModel.data.updatedAt).toEqual(new Date('2021-01-01')) + expect(encryptedModel.data.number).toBe(1) + // Decrypt the model const decryptedResult = await protectClient.decryptModel( encryptedModel.data, @@ -219,6 +255,22 @@ describe('bulk encryption', () => { throw new Error(`[protect]: ${encryptedModels.failure.message}`) } + // Verify encrypted fields for each model + expect(encryptedModels.data[0].email).toHaveProperty('c') + expect(encryptedModels.data[0].address).toHaveProperty('c') + expect(encryptedModels.data[1].email).toHaveProperty('c') + expect(encryptedModels.data[1].address).toBeNull() + + // Verify non-encrypted fields remain unchanged + expect(encryptedModels.data[0].id).toBe('1') + expect(encryptedModels.data[0].createdAt).toEqual(new Date('2021-01-01')) + expect(encryptedModels.data[0].updatedAt).toEqual(new Date('2021-01-01')) + expect(encryptedModels.data[0].number).toBe(1) + expect(encryptedModels.data[1].id).toBe('2') + expect(encryptedModels.data[1].createdAt).toEqual(new Date('2021-01-01')) + expect(encryptedModels.data[1].updatedAt).toEqual(new Date('2021-01-01')) + expect(encryptedModels.data[1].number).toBe(2) + // Decrypt the models const decryptedResult = await protectClient.bulkDecryptModels( encryptedModels.data, @@ -318,6 +370,28 @@ describe('bulk encryption edge cases', () => { throw new Error(`[protect]: ${encryptedModels.failure.message}`) } + // Verify encrypted fields for each model + expect(encryptedModels.data[0].email).toHaveProperty('c') + expect(encryptedModels.data[0].address).toBeNull() + expect(encryptedModels.data[1].email).toBeNull() + expect(encryptedModels.data[1].address).toHaveProperty('c') + expect(encryptedModels.data[2].email).toHaveProperty('c') + expect(encryptedModels.data[2].address).toHaveProperty('c') + + // Verify non-encrypted fields remain unchanged + expect(encryptedModels.data[0].id).toBe('1') + expect(encryptedModels.data[0].createdAt).toEqual(new Date('2021-01-01')) + expect(encryptedModels.data[0].updatedAt).toEqual(new Date('2021-01-01')) + expect(encryptedModels.data[0].number).toBe(1) + expect(encryptedModels.data[1].id).toBe('2') + expect(encryptedModels.data[1].createdAt).toEqual(new Date('2021-01-01')) + expect(encryptedModels.data[1].updatedAt).toEqual(new Date('2021-01-01')) + expect(encryptedModels.data[1].number).toBe(2) + expect(encryptedModels.data[2].id).toBe('3') + expect(encryptedModels.data[2].createdAt).toEqual(new Date('2021-01-01')) + expect(encryptedModels.data[2].updatedAt).toEqual(new Date('2021-01-01')) + expect(encryptedModels.data[2].number).toBe(3) + // Decrypt the models const decryptedResult = await protectClient.bulkDecryptModels( encryptedModels.data, @@ -369,6 +443,28 @@ describe('bulk encryption edge cases', () => { throw new Error(`[protect]: ${encryptedModels.failure.message}`) } + // Verify encrypted fields for each model + expect(encryptedModels.data[0].email).toHaveProperty('c') + expect(encryptedModels.data[0].address).toBeUndefined() + expect(encryptedModels.data[1].email).toBeNull() + expect(encryptedModels.data[1].address).toHaveProperty('c') + expect(encryptedModels.data[2].email).toHaveProperty('c') + expect(encryptedModels.data[2].address).toHaveProperty('c') + + // Verify non-encrypted fields remain unchanged + expect(encryptedModels.data[0].id).toBe('1') + expect(encryptedModels.data[0].createdAt).toEqual(new Date('2021-01-01')) + expect(encryptedModels.data[0].updatedAt).toEqual(new Date('2021-01-01')) + expect(encryptedModels.data[0].number).toBe(1) + expect(encryptedModels.data[1].id).toBe('2') + expect(encryptedModels.data[1].createdAt).toEqual(new Date('2021-01-01')) + expect(encryptedModels.data[1].updatedAt).toEqual(new Date('2021-01-01')) + expect(encryptedModels.data[1].number).toBe(2) + expect(encryptedModels.data[2].id).toBe('3') + expect(encryptedModels.data[2].createdAt).toEqual(new Date('2021-01-01')) + expect(encryptedModels.data[2].updatedAt).toEqual(new Date('2021-01-01')) + expect(encryptedModels.data[2].number).toBe(3) + // Decrypt the models const decryptedResult = await protectClient.bulkDecryptModels( encryptedModels.data, @@ -415,6 +511,28 @@ describe('bulk encryption edge cases', () => { throw new Error(`[protect]: ${encryptedModels.failure.message}`) } + // Verify encrypted fields for each model + expect(encryptedModels.data[0].email).toBeUndefined() + expect(encryptedModels.data[0].address).toBeUndefined() + expect(encryptedModels.data[1].email).toHaveProperty('c') + expect(encryptedModels.data[1].address).toBeUndefined() + expect(encryptedModels.data[2].email).toBeUndefined() + expect(encryptedModels.data[2].address).toBeUndefined() + + // Verify non-encrypted fields remain unchanged + expect(encryptedModels.data[0].id).toBe('1') + expect(encryptedModels.data[0].createdAt).toEqual(new Date('2021-01-01')) + expect(encryptedModels.data[0].updatedAt).toEqual(new Date('2021-01-01')) + expect(encryptedModels.data[0].number).toBe(1) + expect(encryptedModels.data[1].id).toBe('2') + expect(encryptedModels.data[1].createdAt).toEqual(new Date('2021-01-01')) + expect(encryptedModels.data[1].updatedAt).toEqual(new Date('2021-01-01')) + expect(encryptedModels.data[1].number).toBe(2) + expect(encryptedModels.data[2].id).toBe('3') + expect(encryptedModels.data[2].createdAt).toEqual(new Date('2021-01-01')) + expect(encryptedModels.data[2].updatedAt).toEqual(new Date('2021-01-01')) + expect(encryptedModels.data[2].number).toBe(3) + // Decrypt the models const decryptedResult = await protectClient.bulkDecryptModels( encryptedModels.data, diff --git a/packages/protect/__tests__/schema.test.ts b/packages/protect/__tests__/schema.test.ts new file mode 100644 index 00000000..dd5a8677 --- /dev/null +++ b/packages/protect/__tests__/schema.test.ts @@ -0,0 +1,132 @@ +import { csColumn, csTable, buildEncryptConfig, csValue } from '../src/schema' +import { describe, it, expect } from 'vitest' + +describe('Schema with nested columns', () => { + it('should handle nested column structures in encrypt config', () => { + const users = csTable('users', { + email: csColumn('email').freeTextSearch().equality().orderAndRange(), + address: csColumn('address').freeTextSearch(), + example: { + field: csValue('example.field'), + nested: { + deep: csValue('example.nested.deep'), + }, + }, + } as const) + + const config = buildEncryptConfig(users) + + // Verify basic structure + expect(config).toEqual({ + v: 2, + tables: { + users: expect.any(Object), + }, + }) + + // Verify all columns are present with correct names + const columns = config.tables.users + expect(Object.keys(columns)).toEqual([ + 'email', + 'address', + 'example.field', + 'example.nested.deep', + ]) + + // Verify email column configuration + expect(columns.email).toEqual({ + cast_as: 'text', + indexes: { + match: expect.any(Object), + unique: expect.any(Object), + ore: {}, + }, + }) + + // Verify address column configuration + expect(columns.address).toEqual({ + cast_as: 'text', + indexes: { + match: expect.any(Object), + }, + }) + + // Verify nested field configuration + expect(columns['example.field']).toEqual({ + cast_as: 'text', + indexes: {}, + }) + + // Verify deeply nested field configuration + expect(columns['example.nested.deep']).toEqual({ + cast_as: 'text', + indexes: {}, + }) + }) + + it('should handle multiple tables with nested columns', () => { + const users = csTable('users', { + email: csColumn('email').equality(), + profile: { + name: csValue('profile.name'), + }, + } as const) + + const posts = csTable('posts', { + title: csColumn('title').freeTextSearch(), + metadata: { + tags: csValue('metadata.tags'), + }, + } as const) + + const config = buildEncryptConfig(users, posts) + + // Verify both tables are present + expect(Object.keys(config.tables)).toEqual(['users', 'posts']) + + // Verify users table columns + expect(Object.keys(config.tables.users)).toEqual(['email', 'profile.name']) + expect(config.tables.users.email.indexes).toHaveProperty('unique') + + // Verify posts table columns + expect(Object.keys(config.tables.posts)).toEqual(['title', 'metadata.tags']) + expect(config.tables.posts.title.indexes).toHaveProperty('match') + }) + + it('should handle complex nested structures with multiple index types', () => { + const complex = csTable('complex', { + id: csColumn('id').equality(), + content: { + text: csValue('content.text'), + metadata: { + tags: csValue('content.metadata.tags'), + stats: { + views: csValue('content.metadata.stats.views'), + }, + }, + }, + } as const) + + const config = buildEncryptConfig(complex) + + // Verify all columns are present + expect(Object.keys(config.tables.complex)).toEqual([ + 'id', + 'content.text', + 'content.metadata.tags', + 'content.metadata.stats.views', + ]) + + // Verify complex nested column with multiple indexes + expect(config.tables.complex['content.metadata.tags']).toEqual({ + cast_as: 'text', + indexes: {}, + }) + + // Verify deeply nested column with order and range + expect(config.tables.complex['content.metadata.stats.views']).toEqual({ + cast_as: 'text', + indexes: {}, + }) + }) +}) diff --git a/packages/protect/src/ffi/model-helpers.ts b/packages/protect/src/ffi/model-helpers.ts index 390b15b2..742ecc1e 100644 --- a/packages/protect/src/ffi/model-helpers.ts +++ b/packages/protect/src/ffi/model-helpers.ts @@ -119,41 +119,118 @@ async function handleMultiModelBulkOperation( } /** - * Helper function to prepare fields for encryption/decryption + * Helper function to prepare fields for decryption */ -function prepareFieldsForOperation>( +function prepareFieldsForDecryption>( model: T, - table?: ProtectTable, ): { otherFields: Record operationFields: Record keyMap: Record nullFields: Record } { - const otherFields = extractOtherFields(model) + const otherFields = { ...model } as Record const operationFields: Record = {} const nullFields: Record = {} const keyMap: Record = {} let index = 0 - const fieldsToProcess = table - ? Object.entries(model).filter(([key]) => - Object.keys(table.build().columns).includes(key), - ) - : Object.entries(extractEncryptedFields(model)) + const processNestedFields = (obj: Record, prefix = '') => { + for (const [key, value] of Object.entries(obj)) { + const fullKey = prefix ? `${prefix}.${key}` : key - for (const [key, value] of fieldsToProcess) { - if (value === null || value === undefined) { - nullFields[key] = value === undefined ? undefined : null - continue + if (value === null || value === undefined) { + nullFields[fullKey] = value + continue + } + + if (typeof value === 'object' && !isEncryptedPayload(value)) { + // Recursively process nested objects + processNestedFields(value as Record, fullKey) + } else if (isEncryptedPayload(value)) { + // This is an encrypted field + const id = index.toString() + keyMap[id] = fullKey + operationFields[fullKey] = value + index++ + + // Remove from otherFields + const parts = fullKey.split('.') + let current = otherFields + for (let i = 0; i < parts.length - 1; i++) { + current = current[parts[i]] as Record + } + delete current[parts[parts.length - 1]] + } } + } + + processNestedFields(model) + return { otherFields, operationFields, keyMap, nullFields } +} + +/** + * Helper function to prepare fields for encryption + */ +function prepareFieldsForEncryption>( + model: T, + table: ProtectTable, +): { + otherFields: Record + operationFields: Record + keyMap: Record + nullFields: Record +} { + const otherFields = { ...model } as Record + const operationFields: Record = {} + const nullFields: Record = {} + const keyMap: Record = {} + let index = 0 - const id = index.toString() - keyMap[id] = key - operationFields[key] = value - index++ + const processNestedFields = ( + obj: Record, + prefix = '', + columnPaths: string[] = [], + ) => { + for (const [key, value] of Object.entries(obj)) { + const fullKey = prefix ? `${prefix}.${key}` : key + + if (value === null || value === undefined) { + nullFields[fullKey] = value + continue + } + + if (typeof value === 'object' && !isEncryptedPayload(value)) { + // Only process nested objects if they're in the schema + if (columnPaths.some((path) => path.startsWith(fullKey))) { + processNestedFields( + value as Record, + fullKey, + columnPaths, + ) + } + } else if (columnPaths.includes(fullKey)) { + // Only process fields that are explicitly defined in the schema + const id = index.toString() + keyMap[id] = fullKey + operationFields[fullKey] = value + index++ + + // Remove from otherFields + const parts = fullKey.split('.') + let current = otherFields + for (let i = 0; i < parts.length - 1; i++) { + current = current[parts[i]] as Record + } + delete current[parts[parts.length - 1]] + } + } } + // Get all column paths from the table schema + const columnPaths = Object.keys(table.build().columns) + processNestedFields(model, '', columnPaths) + return { otherFields, operationFields, keyMap, nullFields } } @@ -169,7 +246,7 @@ export async function decryptModelFields>( } const { otherFields, operationFields, keyMap, nullFields } = - prepareFieldsForOperation(model) + prepareFieldsForDecryption(model) const bulkDecryptPayload = Object.entries(operationFields).map( ([key, value]) => ({ @@ -184,7 +261,39 @@ export async function decryptModelFields>( keyMap, ) - return { ...otherFields, ...nullFields, ...decryptedFields } as Decrypted + // Helper function to set a nested value + const setNestedValue = ( + obj: Record, + path: string[], + value: unknown, + ) => { + let current = obj + for (let i = 0; i < path.length - 1; i++) { + const part = path[i] + if (!(part in current)) { + current[part] = {} + } + current = current[part] as Record + } + current[path[path.length - 1]] = value + } + + // Reconstruct the object with proper nesting + const result: Record = { ...otherFields } + + // First, reconstruct the null/undefined fields + for (const [key, value] of Object.entries(nullFields)) { + const parts = key.split('.') + setNestedValue(result, parts, value) + } + + // Then, reconstruct the decrypted fields + for (const [key, value] of Object.entries(decryptedFields)) { + const parts = key.split('.') + setNestedValue(result, parts, value) + } + + return result as Decrypted } /** @@ -200,7 +309,7 @@ export async function encryptModelFields>( } const { otherFields, operationFields, keyMap, nullFields } = - prepareFieldsForOperation(model, table) + prepareFieldsForEncryption(model, table) const bulkEncryptPayload = Object.entries(operationFields).map( ([key, value]) => ({ @@ -217,7 +326,39 @@ export async function encryptModelFields>( keyMap, ) - return { ...otherFields, ...nullFields, ...encryptedData } as T + // Helper function to set a nested value + const setNestedValue = ( + obj: Record, + path: string[], + value: unknown, + ) => { + let current = obj + for (let i = 0; i < path.length - 1; i++) { + const part = path[i] + if (!(part in current)) { + current[part] = {} + } + current = current[part] as Record + } + current[path[path.length - 1]] = value + } + + // Reconstruct the object with proper nesting + const result: Record = { ...otherFields } + + // First, reconstruct the null/undefined fields + for (const [key, value] of Object.entries(nullFields)) { + const parts = key.split('.') + setNestedValue(result, parts, value) + } + + // Then, reconstruct the encrypted fields + for (const [key, value] of Object.entries(encryptedData)) { + const parts = key.split('.') + setNestedValue(result, parts, value) + } + + return result as T } /** @@ -239,7 +380,7 @@ export async function decryptModelFieldsWithLockContext< } const { otherFields, operationFields, keyMap, nullFields } = - prepareFieldsForOperation(model) + prepareFieldsForDecryption(model) const bulkDecryptPayload = Object.entries(operationFields).map( ([key, value]) => ({ @@ -255,7 +396,39 @@ export async function decryptModelFieldsWithLockContext< keyMap, ) - return { ...otherFields, ...nullFields, ...decryptedFields } as Decrypted + // Helper function to set a nested value + const setNestedValue = ( + obj: Record, + path: string[], + value: unknown, + ) => { + let current = obj + for (let i = 0; i < path.length - 1; i++) { + const part = path[i] + if (!(part in current)) { + current[part] = {} + } + current = current[part] as Record + } + current[path[path.length - 1]] = value + } + + // Reconstruct the object with proper nesting + const result: Record = { ...otherFields } + + // First, reconstruct the null/undefined fields + for (const [key, value] of Object.entries(nullFields)) { + const parts = key.split('.') + setNestedValue(result, parts, value) + } + + // Then, reconstruct the decrypted fields + for (const [key, value] of Object.entries(decryptedFields)) { + const parts = key.split('.') + setNestedValue(result, parts, value) + } + + return result as Decrypted } /** @@ -278,7 +451,7 @@ export async function encryptModelFieldsWithLockContext< } const { otherFields, operationFields, keyMap, nullFields } = - prepareFieldsForOperation(model, table) + prepareFieldsForEncryption(model, table) const bulkEncryptPayload = Object.entries(operationFields).map( ([key, value]) => ({ @@ -296,7 +469,39 @@ export async function encryptModelFieldsWithLockContext< keyMap, ) - return { ...otherFields, ...nullFields, ...encryptedData } as T + // Helper function to set a nested value + const setNestedValue = ( + obj: Record, + path: string[], + value: unknown, + ) => { + let current = obj + for (let i = 0; i < path.length - 1; i++) { + const part = path[i] + if (!(part in current)) { + current[part] = {} + } + current = current[part] as Record + } + current[path[path.length - 1]] = value + } + + // Reconstruct the object with proper nesting + const result: Record = { ...otherFields } + + // First, reconstruct the null/undefined fields + for (const [key, value] of Object.entries(nullFields)) { + const parts = key.split('.') + setNestedValue(result, parts, value) + } + + // Then, reconstruct the encrypted fields + for (const [key, value] of Object.entries(encryptedData)) { + const parts = key.split('.') + setNestedValue(result, parts, value) + } + + return result as T } /** @@ -319,26 +524,89 @@ function prepareBulkModelsForOperation>( for (let modelIndex = 0; modelIndex < models.length; modelIndex++) { const model = models[modelIndex] - const modelOtherFields = extractOtherFields(model) + const modelOtherFields = { ...model } as Record const modelOperationFields: Record = {} const modelNullFields: Record = {} - const fieldsToProcess = table - ? Object.entries(model).filter(([key]) => - Object.keys(table.build().columns).includes(key), - ) - : Object.entries(extractEncryptedFields(model)) - - for (const [key, value] of fieldsToProcess) { - if (value === null || value === undefined) { - modelNullFields[key] = value === undefined ? undefined : null - continue + const processNestedFields = ( + obj: Record, + prefix = '', + columnPaths: string[] = [], + ) => { + for (const [key, value] of Object.entries(obj)) { + const fullKey = prefix ? `${prefix}.${key}` : key + + if (value === null || value === undefined) { + modelNullFields[fullKey] = value + continue + } + + if (typeof value === 'object' && !isEncryptedPayload(value)) { + // Only process nested objects if they're in the schema + if (columnPaths.some((path) => path.startsWith(fullKey))) { + processNestedFields( + value as Record, + fullKey, + columnPaths, + ) + } + } else if (columnPaths.includes(fullKey)) { + // Only process fields that are explicitly defined in the schema + const id = index.toString() + keyMap[id] = { modelIndex, fieldKey: fullKey } + modelOperationFields[fullKey] = value + index++ + + // Remove from otherFields + const parts = fullKey.split('.') + let current = modelOtherFields + for (let i = 0; i < parts.length - 1; i++) { + current = current[parts[i]] as Record + } + delete current[parts[parts.length - 1]] + } } + } - const id = index.toString() - keyMap[id] = { modelIndex, fieldKey: key } - modelOperationFields[key] = value - index++ + if (table) { + // Get all column paths from the table schema + const columnPaths = Object.keys(table.build().columns) + processNestedFields(model, '', columnPaths) + } else { + // For decryption, process all encrypted fields + const processEncryptedFields = ( + obj: Record, + prefix = '', + ) => { + for (const [key, value] of Object.entries(obj)) { + const fullKey = prefix ? `${prefix}.${key}` : key + + if (value === null || value === undefined) { + modelNullFields[fullKey] = value + continue + } + + if (typeof value === 'object' && !isEncryptedPayload(value)) { + // Recursively process nested objects + processEncryptedFields(value as Record, fullKey) + } else if (isEncryptedPayload(value)) { + // This is an encrypted field + const id = index.toString() + keyMap[id] = { modelIndex, fieldKey: fullKey } + modelOperationFields[fullKey] = value + index++ + + // Remove from otherFields + const parts = fullKey.split('.') + let current = modelOtherFields + for (let i = 0; i < parts.length - 1; i++) { + current = current[parts[i]] as Record + } + delete current[parts[parts.length - 1]] + } + } + } + processEncryptedFields(model) } otherFields.push(modelOtherFields) @@ -368,7 +636,6 @@ export async function bulkEncryptModels>( const { otherFields, operationFields, keyMap, nullFields } = prepareBulkModelsForOperation(models, table) - // Collect all fields that need to be encrypted into a single array const bulkEncryptPayload = operationFields.flatMap((fields, modelIndex) => Object.entries(fields).map(([key, value]) => ({ id: `${modelIndex}-${key}`, @@ -378,18 +645,40 @@ export async function bulkEncryptModels>( })), ) - // Make a single FFI call for all fields const encryptedData = await handleMultiModelBulkOperation( bulkEncryptPayload, (items) => encryptBulk(client, items), keyMap, ) - // Reconstruct models - return models.map((_, modelIndex) => ({ - ...otherFields[modelIndex], - ...nullFields[modelIndex], - ...Object.fromEntries( + // Helper function to set a nested value + const setNestedValue = ( + obj: Record, + path: string[], + value: unknown, + ) => { + let current = obj + for (let i = 0; i < path.length - 1; i++) { + const part = path[i] + if (!(part in current)) { + current[part] = {} + } + current = current[part] as Record + } + current[path[path.length - 1]] = value + } + + return models.map((_, modelIndex) => { + const result: Record = { ...otherFields[modelIndex] } + + // First, reconstruct the null/undefined fields + for (const [key, value] of Object.entries(nullFields[modelIndex])) { + const parts = key.split('.') + setNestedValue(result, parts, value) + } + + // Then, reconstruct the encrypted fields + const modelData = Object.fromEntries( Object.entries(encryptedData) .filter(([key]) => { const [idx] = key.split('-') @@ -399,8 +688,15 @@ export async function bulkEncryptModels>( const [_, fieldKey] = key.split('-') return [fieldKey, value] }), - ), - })) as T[] + ) + + for (const [key, value] of Object.entries(modelData)) { + const parts = key.split('.') + setNestedValue(result, parts, value) + } + + return result as T + }) } /** @@ -421,7 +717,6 @@ export async function bulkDecryptModels>( const { otherFields, operationFields, keyMap, nullFields } = prepareBulkModelsForOperation(models) - // Collect all fields that need to be decrypted into a single array const bulkDecryptPayload = operationFields.flatMap((fields, modelIndex) => Object.entries(fields).map(([key, value]) => ({ id: `${modelIndex}-${key}`, @@ -429,18 +724,40 @@ export async function bulkDecryptModels>( })), ) - // Make a single FFI call for all fields const decryptedFields = await handleMultiModelBulkOperation( bulkDecryptPayload, (items) => decryptBulk(client, items), keyMap, ) - // Reconstruct models - return models.map((_, modelIndex) => ({ - ...otherFields[modelIndex], - ...nullFields[modelIndex], - ...Object.fromEntries( + // Helper function to set a nested value + const setNestedValue = ( + obj: Record, + path: string[], + value: unknown, + ) => { + let current = obj + for (let i = 0; i < path.length - 1; i++) { + const part = path[i] + if (!(part in current)) { + current[part] = {} + } + current = current[part] as Record + } + current[path[path.length - 1]] = value + } + + return models.map((_, modelIndex) => { + const result: Record = { ...otherFields[modelIndex] } + + // First, reconstruct the null/undefined fields + for (const [key, value] of Object.entries(nullFields[modelIndex])) { + const parts = key.split('.') + setNestedValue(result, parts, value) + } + + // Then, reconstruct the decrypted fields + const modelData = Object.fromEntries( Object.entries(decryptedFields) .filter(([key]) => { const [idx] = key.split('-') @@ -450,8 +767,15 @@ export async function bulkDecryptModels>( const [_, fieldKey] = key.split('-') return [fieldKey, value] }), - ), - })) as Decrypted[] + ) + + for (const [key, value] of Object.entries(modelData)) { + const parts = key.split('.') + setNestedValue(result, parts, value) + } + + return result as Decrypted + }) } /** diff --git a/packages/protect/src/ffi/operations/decrypt-model.ts b/packages/protect/src/ffi/operations/decrypt-model.ts index 1ce17d1b..57e7e056 100644 --- a/packages/protect/src/ffi/operations/decrypt-model.ts +++ b/packages/protect/src/ffi/operations/decrypt-model.ts @@ -8,12 +8,14 @@ import { decryptModelFields, decryptModelFieldsWithLockContext, } from '../model-helpers' +import type { ProtectTable, ProtectTableColumn } from '../../schema' export class DecryptModelOperation> implements PromiseLike, ProtectError>> { private client: Client private model: T + private table?: ProtectTable constructor(client: Client, model: T) { this.client = client @@ -59,6 +61,7 @@ export class DecryptModelOperation> public getOperation(): { client: Client model: T + table?: ProtectTable } { return { client: this.client, diff --git a/packages/protect/src/ffi/operations/encrypt.ts b/packages/protect/src/ffi/operations/encrypt.ts index 48f0f862..336c0294 100644 --- a/packages/protect/src/ffi/operations/encrypt.ts +++ b/packages/protect/src/ffi/operations/encrypt.ts @@ -12,6 +12,7 @@ import type { } from '../../types' import type { ProtectColumn, + ProtectValue, ProtectTable, ProtectTableColumn, } from '../../schema' @@ -21,7 +22,7 @@ export class EncryptOperation { private client: Client private plaintext: EncryptPayload - private column: ProtectColumn + private column: ProtectColumn | ProtectValue private table: ProtectTable constructor(client: Client, plaintext: EncryptPayload, opts: EncryptOptions) { @@ -86,7 +87,7 @@ export class EncryptOperation public getOperation(): { client: Client plaintext: EncryptPayload - column: ProtectColumn + column: ProtectColumn | ProtectValue table: ProtectTable } { return { diff --git a/packages/protect/src/index.ts b/packages/protect/src/index.ts index 6a7b4e1b..0deb7832 100644 --- a/packages/protect/src/index.ts +++ b/packages/protect/src/index.ts @@ -60,8 +60,13 @@ export const protect = async ( export type { Result } from '@byteslice/result' export type { ProtectClient } from './ffi' -export { csTable, csColumn } from './schema' -export type { ProtectColumn, ProtectTable, ProtectTableColumn } from './schema' +export { csTable, csColumn, csValue } from './schema' +export type { + ProtectColumn, + ProtectTable, + ProtectTableColumn, + ProtectValue, +} from './schema' export * from './helpers' export * from './identify' export * from './types' diff --git a/packages/protect/src/schema/index.ts b/packages/protect/src/schema/index.ts index 215f68b5..1431403f 100644 --- a/packages/protect/src/schema/index.ts +++ b/packages/protect/src/schema/index.ts @@ -88,12 +88,55 @@ type UniqueIndexOpts = z.infer type OreIndexOpts = z.infer type ColumnSchema = z.infer -export type ProtectTableColumn = Record +export type ProtectTableColumn = { + [key: string]: + | ProtectColumn + | { + [key: string]: + | ProtectValue + | { + [key: string]: + | ProtectValue + | { + [key: string]: ProtectValue + } + } + } +} export type EncryptConfig = z.infer // ------------------------ // Interface definitions // ------------------------ +export class ProtectValue { + private valueName: string + private castAsValue: CastAs + + constructor(valueName: string) { + this.valueName = valueName + this.castAsValue = 'text' + } + + /** + * Set or override the cast_as value. + */ + dataType(castAs: CastAs) { + this.castAsValue = castAs + return this + } + + build() { + return { + cast_as: this.castAsValue, + indexes: {}, + } + } + + getName() { + return this.valueName + } +} + export class ProtectColumn { private columnName: string private castAsValue: CastAs @@ -190,8 +233,36 @@ export class ProtectTable { */ build(): TableDefinition { const builtColumns: Record = {} + + const processColumn = ( + builder: + | ProtectColumn + | Record< + string, + | ProtectValue + | Record< + string, + | ProtectValue + | Record> + > + >, + colName: string, + ) => { + if (builder instanceof ProtectColumn) { + builtColumns[colName] = builder.build() + } else { + for (const [key, value] of Object.entries(builder)) { + if (value instanceof ProtectValue) { + builtColumns[value.getName()] = value.build() + } else { + processColumn(value, key) + } + } + } + } + for (const [colName, builder] of Object.entries(this.columnBuilders)) { - builtColumns[colName] = builder.build() + processColumn(builder, colName) } return { @@ -222,6 +293,10 @@ export function csColumn(columnName: string) { return new ProtectColumn(columnName) } +export function csValue(valueName: string) { + return new ProtectValue(valueName) +} + // ------------------------ // Internal functions // ------------------------ diff --git a/packages/protect/src/types.ts b/packages/protect/src/types.ts index 2c43c3dc..bea3f5d2 100644 --- a/packages/protect/src/types.ts +++ b/packages/protect/src/types.ts @@ -1,5 +1,5 @@ import type { newClient, Encrypted } from '@cipherstash/protect-ffi' -import type { ProtectTableColumn } from './schema' +import type { ProtectTableColumn, ProtectValue } from './schema' import type { ProtectTable } from './schema' import type { ProtectColumn } from './schema' @@ -41,7 +41,7 @@ export type EncryptPayload = string | null * Represents the options for encrypting a payload using the `encrypt` function */ export type EncryptOptions = { - column: ProtectColumn + column: ProtectColumn | ProtectValue table: ProtectTable }