diff --git a/.github/workflows/release-eql.yml b/.github/workflows/release-eql.yml index f51a2833..b707b4d8 100644 --- a/.github/workflows/release-eql.yml +++ b/.github/workflows/release-eql.yml @@ -54,3 +54,5 @@ jobs: files: | release/cipherstash-encrypt.sql release/cipherstash-encrypt-uninstall.sql + release/cipherstash-encrypt-supabase.sql + release/cipherstash-encrypt-uninstall-supabase.sql diff --git a/.github/workflows/test-eql.yml b/.github/workflows/test-eql.yml index 050e9dd9..1d34d5ac 100644 --- a/.github/workflows/test-eql.yml +++ b/.github/workflows/test-eql.yml @@ -5,16 +5,17 @@ on: - main paths: - ".github/workflows/test-eql.yml" - - "sql/*.sql" + - "src/**/*.sql" + - "sql/**/*.sql" - "tests/**/*" - "tasks/**/*" pull_request: - branches: - - main + # run on all pull requests paths: - ".github/workflows/test-eql.yml" - - "sql/*.sql" + - "src/**/*.sql" + - "sql/**/*.sql" - "tests/**/*" - "tasks/**/*" diff --git a/.gitignore b/.gitignore index dfcfe631..09f943d7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,19 @@ + + +.DS_Store +.mise.* + +deps.txt +deps-ordered.txt + +deps-supabase.txt +deps-ordered-supabase.txt + # Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore -sql/000-version.sql +src/version.sql + + # Logs @@ -186,7 +199,7 @@ cipherstash-proxy.toml # build artifacts release/ -.mise.* + # jupyter notebook .ipynb_checkpoints diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 00000000..9f788cfc --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,392 @@ +# Developing CipherStash EQL + +## Table of Contents + +- [How this project is organised](#how-this-project-is-organised) +- [Set up a local development environment](#set-up-a-local-development-environment) + - [Installing mise](#installing-mise) +- [Testing](#testing) + - [Running tests locally](#running-tests-locally) +- [Releasing](#releasing) +- [Building](#building) + - [Dependencies](#dependencies) + - [Building a release locally](#building-a-release-locally) +- [Structure](#structure) + - [Schema](#schema) + - [Types](#types) + - [Encrypted column type](#encrypted-column-type) + - [Encrypted index term types](#encrypted-index-term-types) + - [Operators](#operators) + - [Working without operators](#working-without-operators) + - [Configuration table](#configuration-table) + +### How this project is organised + +Development is managed through [mise](https://mise.jdx.dev/), both locally and [in CI](https://github.com/cipherstash/encrypt-query-language/actions). + +mise has tasks for: + +- Building EQL install and uninstall scripts (`build`) +- Starting and stopping PostgreSQL containers (`postgres:up`, `postgres:down`) +- Running unit and integration tests (`test`, `reset`) + +These are the important files in the repo: + +``` +. +├── mise.toml <-- the main config file for mise +├── tasks/ <-- mise tasks +├── src/ <-- The individual SQL components that make up EQL +│ ├── blake3/ <-- blake3 index term type +│ ├── encrypted/ <-- Encrypted column type +│ ├── operators/ <-- Operators for the encrypted column type +│ ├── match/ <-- match index term type +│ ├── unique/ <-- unique index term type +│ ├── ore/ <-- ore index term type +│ ├── ore_cllw_u64_8/ <-- ore-cllw fixed index term type +│ ├── ore_cllw_var_8/ <-- ore-cllw variable index term type +│ ├── config/ <-- Configuration management for encrypted columns +│ ├── schema.sql <-- Defines the PostgreSQL schema for namespacing EQL +│ ├── crypto.sql <-- Installs pg_crypto extension, required by ORE +│ ├── common.sql <-- Shared helper functions +│ └── version.sql <-- Defines function to query current EQL version - automatically generated on build +├── docs/ <-- Tutorial, reference, and concept documentation +├── tests/ <-- Unit and integration tests +│ ├── docker-compose.yml <-- Docker configuration for running PostgreSQL instances +│ └── *.sql <-- Helpers and test data loaded during test runs +├── release/ <-- Build artifacts produced by the `build` task +├── examples/ <-- Example uses of EQL in different languages +└── playground/ <-- Playground enviroment for experimenting with EQL and CipherStash Proxy +``` + +Tests live alongside the individual SQL files, with a filename ending with `_test.sql` + +We break SQL into small modules named after what they do. + +In general, operator functions are thin wrappers around larger functions that do the actual work. +Put the wrapper functions in `operators.sql` and the larger functions in `functions.sql`. + +Dependencies between SQL in `src/` are declared in a comment at the top of each file. +All SQL files should `REQUIRE` the source file of any other object they reference. + +All files must have at least one declaration, and the default is to reference the schema: + +``` +-- REQUIRE: src/schema.sql +``` + +## Set up a local development environment + +> [!IMPORTANT] > **Before you follow this how-to** you need to have this software installed: +> +> - [mise](https://mise.jdx.dev/) — see the [installing mise](#installing-mise) instructions +> - [Docker](https://www.docker.com/) — see Docker's [documentation for installing](https://docs.docker.com/get-started/get-docker/) + +Local development quickstart: + +```shell +# Clone the repo +git clone https://github.com/cipherstash/encrypt-query-language +cd encrypt-query-language + +# Install dependencies +mise trust --yes + +# Build EQL installer and uninstaller, outputting to release/ +mise run build + +# Start a postgres instance (defaults to PostgreSQL 17) +mise run postgres:up --extra-args "--detach --wait" + +# Run the tests (defaults to PostgreSQL 17) +mise run test + +# Stop and remove all containers and networks +mise run postgres:down +``` + +### Installing mise + +> [!IMPORTANT] +> You must complete this step to set up a local development environment. + +Local development and task running in CI is managed through [mise](https://mise.jdx.dev/). + +To install mise: + +- If you're on macOS, run `brew install mise` +- If you're on another platform, check out the mise [installation methods documentation](https://mise.jdx.dev/installing-mise.html#installation-methods) + +Then add mise to your shell: + +```shell +# If you're running Bash +echo 'eval "$(mise activate bash)"' >> ~/.bashrc + +# If you're running Zsh +echo 'eval "$(mise activate zsh)"' >> ~/.zshrc +``` + +We use [`cargo-binstall`](https://github.com/cargo-bins/cargo-binstall) for faster installation of tools installed via `mise` and Cargo. +We install `cargo-binstall` via `mise` when installing development and testing dependencies. + +> [!TIP] +> We provide abbreviations for most of the commands that follow. +> For example, `mise run postgres:setup` can be abbreviated to `mise r s`. +> Run `mise tasks --extended` to see the task shortcuts. + +## Testing + +There are tests for checking EQL against PostgreSQL versions 14–17, that verify: + +- Adding, removing, and modifying encrypted data and indexes +- Validating, applying, and removing configuration for encrypted data and encrypted indexes +- Validating schemas for EQL configuration, encrypted data, and encrypted indexes +- Using PostgreSQL operators on encrypted data and indexes (`=`, `<>`, `@>`) + +The easiest way to run the tests [is in GitHub Actions](./.github/workflows/test-eql.yml): + +- Automatically whenever there are changes in the `sql/`, `tests/`, or `tasks/` directories +- By manually running [the workflow](https://github.com/cipherstash/encrypt-query-language/actions/workflows/test-eql.yml) + +This is how the `test-eql.yml` workflow functions: + +```mermaid +--- +title: Testing EQL +--- +stateDiagram-v2 + direction LR + classDef code font-family:monospace; + + + state "🧍 Human makes changes to EQL sources" as changes + state sources_fork <> + state sources_join <> + state "src/*.sql" as source_sql + state "tasks/**/*" as source_tasks + state "tests/**/*" as source_tests + state sources_changed <> + + state "🛠️ Trigger GitHub Actions workflow test-eql.yml" as build_triggered + state "Matrix: Test EQL SQL components" as matrix + state "Test with Postgres 14" as pg14 + state "Test with Postgres 15" as pg15 + state "Test with Postgres 16" as pg16 + state "Test with Postgres 17" as pg17 + state "Check build results" as check + state if_state <> + + changes --> sources_fork + sources_fork --> source_sql:::code + sources_fork --> source_tests:::code + sources_fork --> source_tasks:::code + source_sql --> sources_join + source_tests --> sources_join + source_tasks --> sources_join + sources_join --> source_changed_check + source_changed_check --> sources_changed + sources_changed --> build_triggered : Some changes + sources_changed --> [*]: No changes + + state "Check source changes" as source_changed_check + + [*] --> changes + + build_triggered --> matrix + + state fork_state <> + matrix --> fork_state + fork_state --> pg14 + fork_state --> pg15 + fork_state --> pg16 + fork_state --> pg17 + + state join_state <> + pg14 --> join_state + pg15 --> join_state + pg16 --> join_state + pg17 --> join_state + + state "✅ Pass build" as build_pass + state "❌ Fail build" as build_fail + join_state --> check + check --> if_state + if_state --> build_pass: All success + if_state --> build_fail : Any failures + build_pass --> [*] + build_fail --> [*] +``` + +You can also [run the tests locally](#running-tests-locally) when doing local development. + +### Running tests locally + +> [!IMPORTANT] > **Before you run the tests locally** you need to [set up a local dev environment](#set-up-a-local-development-environment). + +To run tests locally with PostgreSQL 17: + +```shell +# Start a postgres instance (defaults to PostgreSQL 17) +mise run postgres:up --extra-args "--detach --wait" + +# Run the tests (defaults to PostgreSQL 17) +mise run test + +# Stop and remove all containers and networks +mise run postgres:down +``` + +You can run the same tasks for Postgres 14, 15, 16, and 17 by specifying arguments: + +```shell +# Start a postgres 14 instance +mise run postgres:up postgres-14 --extra-args "--detach --wait" + +# Run the tests against postgres 14 +mise run test --postgres 14 + +# Stop postgres and remove all containers and networks +mise run postgres:down +``` + +The configuration for the Postgres containers in `tests/docker-compose.yml`. + +Limitations: + +- **Volumes for Postgres containers are not persistent.** + If you need to look at data in the container, uncomment a volume in + `tests/docker-compose.yml` +- **You can't run multiple Postgres containers at the same time.** + All the containers bind to the same port (`7543`). If you want to run + multiple containers at the same time, you have to change the ports by + editing `tests/docker-compose.yml` + +## Releasing + +To cut a [release](https://github.com/cipherstash/encrypt-query-language/releases) of EQL: + +1. Draft a [new release](https://github.com/cipherstash/encrypt-query-language/releases/new) on GitHub. +1. Choose a tag, and create a new one with the prefix `eql-` followed by a [semver](https://semver.org/) (for example, `eql-1.2.3`). +1. Generate the release notes. +1. Optionally set the release to be the latest (you can set a release to be latest later on if you are testing out a release first). +1. Click `Publish release`. + +This will trigger the [Release EQL](https://github.com/cipherstash/encrypt-query-language/actions/workflows/release-eql.yml) workflow, which will build and attach artifacts to [the release](https://github.com/cipherstash/encrypt-query-language/releases/). + +## Building + +### Dependencies + +SQL sources are split into smaller files in `src/`. +Dependencies are resolved at build time to construct a single SQL file with the correct ordering. + +### Building a release locally + +To build a release locally, run: + +```bash +mise run build +``` + +This produces two SQL files in `releases/`: + +- An installer (`cipherstash-encrypt.sql`), and +- An uninstaller (`cipherstash-encrypt-uninstall.sql`) + +## Structure + +### Schema + +EQL is installed into the `eql_v2` PostgreSQL schema. + +### Types + +#### Encrypted column type + +`public.eql_v2_encrypted` is EQL's encrypted column type, defined as PostgreSQL composite type. + +This column type is used for storing the encrypted value and any associated indexes for searching. +The associated indexes are described in the [index term types](#index-term-types) section. + +`public.eql_v2_encrypted` is in the public schema, because once it's used by a user in one of their tables, encrypted column types cannot be dropped without dropping data. + +#### Encrypted index term types + +Each type of encrypted index (`unique`, `match`, `ore`) has an associated type, functions, and operators. + +These are transient runtime types, used internally by EQL functions and operators: + +- `eql_v2.blake3` +- `eql_v2.unique_index` +- `eql_v2.match` +- `eql_v2.ore_cllw_u64_8` +- `eql_v2.ore_cllw_var_8` +- `eql_v2.ore_64_8_v2` +- `eql_v2.ore_64_8_v2_term` + +The data in the column is converted into these types, when any operations are being performed on that encrypted data. + +### Operators + +Searchable encryption functionality is driven by operators on two types: + +- EQL's `eql_v2_encrypted` column type +- PostgreSQL's `jsonb` column type + +For convenience, operators allow comparisons between `eql_v2_encrypted` and `jsonb` column types. + +Operators allow comparisons between: + +- `eql_v2_encrypted` and `eql_v2_encrypted` +- `jsonb` and `eql_v2_encrypted` +- `eql_v2_encrypted` and `jsonb` + +Operators defined on the `eql_v2_encrypted` dispatch to the underlying index terms based on the most efficient order of operations. + +For example, it is possible to have both `unique` and `ore` indexes defined. +For equality (`=`, `<>`) operations, a `unique` index term is a text comparison and should be preferred over an `ore` index term. + +The index term types and functions are internal implementation details and should not be exposed as operators on the `eql_v2_encrypted` type. +For example, `eql_v2_encrypted` should not have an operator with the `ore_64_8_v2` type. +Users should never need to think about or interact with EQL internals. + +#### Working without operators + +There are scenarios where users are unable to install EQL operators in your database. +Users will experience this in more restrictive environments like Supabase. + +EQL can still be used, but requires the use of functions instead of operators. + +For example, to perform an equality query: + +```sql +SELECT email FROM users WHERE eql_v2.eq(email, $1); +``` + +### Configuration table + +EQL uses a table for tracking configuration state in the database, called `public.eql_v2_configuration`. + +This table should never be dropped, except by a user explicitly uninstalling EQL. + + diff --git a/README.md b/README.md index 84101c9e..caf2cd61 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,7 @@ Encrypt Query Language (EQL) is a set of abstractions for transmitting, storing, and interacting with encrypted data and indexes in PostgreSQL. -> [!TIP] -> **New to EQL?** Start with the higher level helpers for EQL in [Python](https://github.com/cipherstash/eqlpy), [Go](https://github.com/cipherstash/goeql), or [JavaScript](https://github.com/cipherstash/jseql) and [TypeScript](https://github.com/cipherstash/jseql), or the [examples](#helper-packages-and-examples) for those languages. +> [!TIP] > **New to EQL?** Start with the higher level helpers for EQL in [Python](https://github.com/cipherstash/eqlpy), [Go](https://github.com/cipherstash/goeql), or [JavaScript](https://github.com/cipherstash/jseql) and [TypeScript](https://github.com/cipherstash/jseql), or the [examples](#helper-packages-and-examples) for those languages. Store encrypted data alongside your existing data: @@ -29,16 +28,15 @@ Store encrypted data alongside your existing data: - [Inserting Data](#inserting-data) - [Reading Data](#reading-data) - [Configuring indexes for searching data](#configuring-indexes-for-searching-data) - - [Adding an index (`cs_add_index_v1`)](#adding-an-index-cs_add_index_v1) + - [Adding an index](#adding-an-index) - [Searching data with EQL](#searching-data-with-eql) - - [Equality search (`cs_unique_v1`)](#equality-search-cs_unique_v1) - - [Full-text search (`cs_match_v1`)](#full-text-search-cs_match_v1) - - [Range queries (`cs_ore_64_8_v1`)](#range-queries-cs_ore_64_8_v1) + - [Equality search](#equality-search) + - [Full-text search](#full-text-search) + - [Range queries](#range-queries) + - [Array Operations](#array-operations) + - [JSON Path Operations](#json-path-operations) - [JSON and JSONB support](#json-and-jsonb-support) - [Frequently Asked Questions](#frequently-asked-questions) - - [How do I integrate CipherStash EQL with my application?](#how-do-i-integrate-cipherstash-eql-with-my-application) - - [Can I use EQL without the CipherStash Proxy?](#can-i-use-eql-without-the-cipherstash-proxy) - - [How is data encrypted in the database?](#how-is-data-encrypted-in-the-database) - [Helper packages](#helper-packages-and-examples) - [Releasing](#releasing) - [Developing](#developing) @@ -79,34 +77,34 @@ Once the custom types and functions are installed in your PostgreSQL database, y ### Enable encrypted columns -Define encrypted columns using the `cs_encrypted_v1` domain type, which extends the `jsonb` type with additional constraints to ensure data integrity. +Define encrypted columns using the `eql_v2_encrypted` type, which extends the `jsonb` type with additional constraints to ensure data integrity. **Example:** ```sql CREATE TABLE users ( id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - encrypted_email cs_encrypted_v1 + encrypted_email eql_v2_encrypted ); ``` ### Configuring the column -Initialize the column using the `cs_add_column_v1` function to enable encryption and decryption via CipherStash Proxy. +Initialize the column using the `eql_v2.add_column` function to enable encryption and decryption via CipherStash Proxy. ```sql -SELECT cs_add_column_v1('users', 'encrypted_email'); +SELECT eql_v2.add_column('users', 'encrypted_email'); ``` -**Note:** This function allows you to encrypt and decrypt data but does not enable searchable encryption. See [Querying Data with EQL](#querying-data-with-eql) for enabling searchable encryption. +**Note:** This function allows you to encrypt and decrypt data but does not enable searchable encryption. See [Searching data with EQL](#searching-data-with-eql) for enabling searchable encryption. ### Activating configuration After modifying configurations, activate them by running: ```sql -SELECT cs_encrypt_v1(); -SELECT cs_activate_v1(); +SELECT eql_v2.encrypt(); +SELECT eql_v2.activate(); ``` **Important:** These functions must be run after any modifications to the configuration. @@ -116,7 +114,7 @@ SELECT cs_activate_v1(); CipherStash Proxy refreshes the configuration every 60 seconds. To force an immediate refresh, run: ```sql -SELECT cs_refresh_encrypt_config(); +SELECT eql_v2.reload_config(); ``` > Note: This statement must be executed when connected to CipherStash Proxy. @@ -191,13 +189,12 @@ In order to perform searchable operations on encrypted data, you must configure > **IMPORTANT:** If you have existing data that's encrypted and you add or modify an index, all the data will need to be re-encrypted. > This is due to the way CipherStash Proxy handles searchable encryption operations. -### Adding an index (`cs_add_index_v1`) +### Adding an index -Add an index to an encrypted column. -This function also behaves the same as `cs_add_column_v1` but with the additional index configuration. +Add an index to an encrypted column using the `eql_v2.add_index` function: ```sql -SELECT cs_add_index_v1( +SELECT eql_v2.add_index( 'table_name', -- Name of the table 'column_name', -- Name of the column 'index_name', -- Index kind ('unique', 'match', 'ore', 'ste_vec') @@ -211,7 +208,7 @@ You can read more about the index configuration options [here](docs/reference/IN **Example (Unique index):** ```sql -SELECT cs_add_index_v1( +SELECT eql_v2.add_index( 'users', 'encrypted_email', 'unique', @@ -219,11 +216,11 @@ SELECT cs_add_index_v1( ); ``` -After adding an index, you have to activate the configuration. +After adding an index, you have to activate the configuration: ```sql -SELECT cs_encrypt_v1(); -SELECT cs_activate_v1(); +SELECT eql_v2.encrypt(); +SELECT eql_v2.activate(); ``` ## Searching data with EQL @@ -232,14 +229,14 @@ EQL provides specialized functions to interact with encrypted data, supporting o In order to use the specialized functions, you must first configure the corresponding indexes. -### Equality search (`cs_unique_v1`) +### Equality search -Enable equality search on encrypted data. +Enable equality search on encrypted data using the `eql_v2.unique` function. **Index configuration example:** ```sql -SELECT cs_add_index_v1( +SELECT eql_v2.add_index( 'users', 'encrypted_email', 'unique', @@ -251,7 +248,7 @@ SELECT cs_add_index_v1( ```sql SELECT * FROM users -WHERE cs_unique_v1(encrypted_email) = cs_unique_v1( +WHERE eql_v2.unique(encrypted_email) = eql_v2.unique( '{"v":1,"k":"pt","p":"test@example.com","i":{"t":"users","c":"encrypted_email"},"q":"unique"}' ); ``` @@ -262,14 +259,14 @@ Equivalent plaintext query: SELECT * FROM users WHERE email = 'test@example.com'; ``` -### Full-text search (`cs_match_v1`) +### Full-text search -Enables basic full-text search on encrypted data. +Enables basic full-text search on encrypted data using the `eql_v2.match` function. **Index configuration example:** ```sql -SELECT cs_add_index_v1( +SELECT eql_v2.add_index( 'users', 'encrypted_email', 'match', @@ -282,7 +279,7 @@ SELECT cs_add_index_v1( ```sql SELECT * FROM users -WHERE cs_match_v1(encrypted_email) @> cs_match_v1( +WHERE eql_v2.match(encrypted_email) @> eql_v2.match( '{"v":1,"k":"pt","p":"test","i":{"t":"users","c":"encrypted_email"},"q":"match"}' ); ``` @@ -293,9 +290,9 @@ Equivalent plaintext query: SELECT * FROM users WHERE email LIKE '%test%'; ``` -### Range queries (`cs_ore_64_8_v1`) +### Range queries -Enable range queries on encrypted data. Supports: +Enable range queries on encrypted data using the `eql_v2.ore_64_8_v2`, `eql_v2.ore_cllw_u64_8`, or `eql_v2.ore_cllw_var_8` functions. Supports: - `ORDER BY` - `WHERE` @@ -304,7 +301,7 @@ Enable range queries on encrypted data. Supports: ```sql SELECT * FROM users -WHERE cs_ore_64_8_v1(encrypted_date) < cs_ore_64_8_v1( +WHERE eql_v2.ore_64_8_v2(encrypted_date) < eql_v2.ore_64_8_v2( '{"v":1,"k":"pt","p":"2023-10-05","i":{"t":"users","c":"encrypted_date"},"q":"ore"}' ); ``` @@ -319,7 +316,7 @@ SELECT * FROM users WHERE date < '2023-10-05'; ```sql SELECT id FROM users -ORDER BY cs_ore_64_8_v1(encrypted_field) DESC; +ORDER BY eql_v2.ore_64_8_v2(encrypted_field) DESC; ``` Equivalent plaintext query: @@ -328,18 +325,31 @@ Equivalent plaintext query: SELECT id FROM users ORDER BY field DESC; ``` -**Example (Grouping):** +### Array Operations + +EQL supports array operations on encrypted data: ```sql -SELECT cs_grouped_value_v1(encrypted_field) COUNT(*) - FROM users - GROUP BY cs_ore_64_8_v1(encrypted_field) +-- Get array length +SELECT eql_v2.jsonb_array_length(encrypted_array) FROM users; + +-- Get array elements +SELECT eql_v2.jsonb_array_elements(encrypted_array) FROM users; + +-- Get array element ciphertexts +SELECT eql_v2.jsonb_array_elements_text(encrypted_array) FROM users; ``` -Equivalent plaintext query: +### JSON Path Operations + +EQL supports JSON path operations on encrypted data using the `->` and `->>` operators: ```sql -SELECT field, COUNT(*) FROM users GROUP BY field; +-- Get encrypted value at path +SELECT encrypted_data->'$.field' FROM users; + +-- Get ciphertext at path +SELECT encrypted_data->>'$.field' FROM users; ``` ## JSON and JSONB support @@ -363,15 +373,15 @@ No, CipherStash Proxy is required to handle the encryption and decryption operat ### How is data encrypted in the database? -Data is encrypted using CipherStash's cryptographic schemes and stored in the `cs_encrypted_v1` column as a JSONB payload. +Data is encrypted using CipherStash's cryptographic schemes and stored in the `eql_v2_encrypted` column as a JSONB payload. Encryption and decryption are handled by CipherStash Proxy. ## Helper packages and examples We've created a few langauge specific packages to help you interact with the payloads: -| Language | ORM | Example | Package | -| ---------- | ----------- | ----------------------------------------------------------------- | ---------------------------------------------------------------- | +| Language | ORM | Example | Package | +| ---------- | ----------- | ---------------------------------------------------------------- | --------------------------------------------- | | Go | Xorm | [Go/Xorm examples](./examples/go/xorm/README.md) | [goeql](https://github.com/cipherstash/goeql) | | TypeScript | Drizzle | [Drizzle examples](./examples/javascript/apps/drizzle/README.md) | [jseql](https://github.com/cipherstash/jseql) | | TypeScript | Prisma | [Drizzle examples](./examples/javascript/apps/prisma/README.md) | [jseql](https://github.com/cipherstash/jseql) | @@ -383,234 +393,6 @@ We've created a few langauge specific packages to help you interact with the pay - [JavaScript/TypeScript](https://github.com/cipherstash/jseql) - [Python](https://github.com/cipherstash/eqlpy) -## Releasing - -To cut a [release](https://github.com/cipherstash/encrypt-query-language/releases) of EQL: - -1. Draft a [new release](https://github.com/cipherstash/encrypt-query-language/releases/new) on GitHub. -1. Choose a tag, and create a new one with the prefix `eql-` followed by a [semver](https://semver.org/) (for example, `eql-1.2.3`). -1. Generate the release notes. -1. Optionally set the release to be the latest (you can set a release to be latest later on if you are testing out a release first). -1. Click `Publish release`. - -This will trigger the [Release EQL](https://github.com/cipherstash/encrypt-query-language/actions/workflows/release-eql.yml) workflow, which will build and attach artifacts to [the release](https://github.com/cipherstash/encrypt-query-language/releases/). - ## Developing -> [!IMPORTANT] -> **Before you follow the quickstart** you need to have this software installed: -> - [mise](https://mise.jdx.dev/) — see the [installing mise](#installing-mise) instructions -> - [Docker](https://www.docker.com/) — see Docker's [documentation for installing](https://docs.docker.com/get-started/get-docker/) - -Local development quickstart: - -``` shell -# Clone the repo -git clone https://github.com/cipherstash/encrypt-query-language -cd encrypt-query-language - -# Install dependencies -mise trust --yes - -# Build EQL installer and uninstaller, outputting to release/ -mise run build - -# Start a postgres instance (defaults to PostgreSQL 17) -mise run postgres:up --extra-args "--detach --wait" - -# Run the tests (defaults to PostgreSQL 17) -mise run test - -# Stop and remove all containers and networks -mise run postgres:down -``` - -### Installing mise - -> [!IMPORTANT] -> You must complete this step to set up a local development environment. - -Local development and task running in CI is managed through [mise](https://mise.jdx.dev/). - -To install mise: - -- If you're on macOS, run `brew install mise` -- If you're on another platform, check out the mise [installation methods documentation](https://mise.jdx.dev/installing-mise.html#installation-methods) - -Then add mise to your shell: - -```shell -# If you're running Bash -echo 'eval "$(mise activate bash)"' >> ~/.bashrc - -# If you're running Zsh -echo 'eval "$(mise activate zsh)"' >> ~/.zshrc -``` - -We use [`cargo-binstall`](https://github.com/cargo-bins/cargo-binstall) for faster installation of tools installed via `mise` and Cargo. -We install `cargo-binstall` via `mise` when installing development and testing dependencies. - -> [!TIP] -> We provide abbreviations for most of the commands that follow. -> For example, `mise run postgres:setup` can be abbreviated to `mise r s`. -> Run `mise tasks --extended` to see the task shortcuts. - -### How this project is organised - -Development is managed through [mise](https://mise.jdx.dev/), both locally and [in CI](https://github.com/cipherstash/encrypt-query-language/actions). - -mise has tasks for: - -- Building EQL install and uninstall scripts (`build`) -- Starting and stopping PostgreSQL containers (`postgres:up`, `postgres:down`) -- Running unit and integration tests (`test`, `reset`) - -These are the important files in the repo: - -``` -. -├── mise.toml <-- the main config file for mise -├── tasks/ <-- mise tasks -├── sql/ <-- The individual SQL components that make up EQL -├── docs/ <-- Tutorial, reference, and concept documentation -├── tests/ <-- Unit and integration tests -│ ├── docker-compose.yml <-- Docker configuration for running PostgreSQL instances -│ └── *.sql <-- Individual unit and integration tests -├── release/ <-- Build artifacts produced by the `build` task -├── examples/ <-- Example uses of EQL in different languages -└── playground/ <-- Playground enviroment for experimenting with EQL and CipherStash Proxy -``` - -## Testing - -There are tests for checking EQL against PostgreSQL versions 14–17, that verify: - -- Adding, removing, and modifying encrypted data and indexes -- Validating, applying, and removing configuration for encrypted data and encrypted indexes -- Validating schemas for EQL configuration, encrypted data, and encrypted indexes -- Using PostgreSQL operators on encrypted data and indexes (`=`, `<>`, `@>`) - -The easiest way to run the tests [is in GitHub Actions](./.github/workflows/test-eql.yml): - -- Automatically whenever there are changes in the `sql/`, `tests/`, or `tasks/` directories -- By manually running [the workflow](https://github.com/cipherstash/encrypt-query-language/actions/workflows/test-eql.yml) - -This is how the `test-eql.yml` workflow functions: - -```mermaid ---- -title: Testing EQL ---- -stateDiagram-v2 - direction LR - classDef code font-family:monospace; - - - state "🧍 Human makes changes to EQL sources" as changes - state sources_fork <> - state sources_join <> - state "sql/*.sql" as source_sql - state "tasks/**/*" as source_tasks - state "tests/**/*" as source_tests - state sources_changed <> - - state "🛠️ Trigger GitHub Actions workflow test-eql.yml" as build_triggered - state "Matrix: Test EQL SQL components" as matrix - state "Test with Postgres 14" as pg14 - state "Test with Postgres 15" as pg15 - state "Test with Postgres 16" as pg16 - state "Test with Postgres 17" as pg17 - state "Check build results" as check - state if_state <> - - changes --> sources_fork - sources_fork --> source_sql:::code - sources_fork --> source_tests:::code - sources_fork --> source_tasks:::code - source_sql --> sources_join - source_tests --> sources_join - source_tasks --> sources_join - sources_join --> source_changed_check - source_changed_check --> sources_changed - sources_changed --> build_triggered : Some changes - sources_changed --> [*]: No changes - - state "Check source changes" as source_changed_check - - [*] --> changes - - build_triggered --> matrix - - state fork_state <> - matrix --> fork_state - fork_state --> pg14 - fork_state --> pg15 - fork_state --> pg16 - fork_state --> pg17 - - state join_state <> - pg14 --> join_state - pg15 --> join_state - pg16 --> join_state - pg17 --> join_state - - state "✅ Pass build" as build_pass - state "❌ Fail build" as build_fail - join_state --> check - check --> if_state - if_state --> build_pass: All success - if_state --> build_fail : Any failures - build_pass --> [*] - build_fail --> [*] -``` - -You can also [run the tests locally](#running-tests-locally) when doing local development. - -### Running tests locally - -> [!IMPORTANT] -> **Before you run the tests locally** you need to [set up a local dev environment](#developing). - -To run tests locally with PostgreSQL 17: - -``` shell -# Start a postgres instance (defaults to PostgreSQL 17) -mise run postgres:up --extra-args "--detach --wait" - -# Run the tests (defaults to PostgreSQL 17) -mise run test - -# Stop and remove all containers and networks -mise run postgres:down -``` - -You can run the same tasks for Postgres 14, 15, 16, and 17 by specifying arguments: - -```shell -# Start a postgres 14 instance -mise run postgres:up postgres-14 --extra-args "--detach --wait" - -# Run the tests against postgres 14 -mise run test --postgres 14 - -# Stop postgres and remove all containers and networks -mise run postgres:down -``` - -The configuration for the Postgres containers in `tests/docker-compose.yml`. - -Limitations: - -- **Volumes for Postgres containers are not persistent.** - If you need to look at data in the container, uncomment a volume in - `tests/docker-compose.yml` -- **You can't run multiple Postgres containers at the same time.** - All the containers bind to the same port (`7543`). If you want to run - multiple containers at the same time, you have to change the ports by - editing `tests/docker-compose.yml` - ---- - -### Didn't find what you wanted? - -[Click here to let us know what was missing from our docs.](https://github.com/cipherstash/encrypt-query-language/issues/new?template=docs-feedback.yml&title=[Docs:]%20Feedback%20on%20README.md) +See the [development guide](./DEVELOPMENT.md). diff --git a/SUPABASE.md b/SUPABASE.md new file mode 100644 index 00000000..c944cfb3 --- /dev/null +++ b/SUPABASE.md @@ -0,0 +1,146 @@ +# Supabase + +## No operators, no problems + +Supabase [does not currently support](https://github.com/supabase/supautils/issues/72) custom operators. +The EQL operator functions can be used in this situation. + +In EQL, PostgreSQL operators are an alias for a function, so the implementation and behaviour remains the same across operators and functions. + +| Operator | Function | Example | +| -------- | ------------------------------------------------------------- | ---------------------------------------------------------------------------- | +| `=` | `eql_v2.eq(eql_v2_encrypted, eql_v2_encrypted)` | `SELECT * FROM users WHERE eql_v2.eq(encrypted_email, $1)`
| +| `<>` | `eql_v2.neq(eql_v2_encrypted, eql_v2_encrypted)` | `SELECT * FROM users WHERE eql_v2.neq(encrypted_email, $1)`
| +| `<` | `eql_v2.lt(eql_v2_encrypted, eql_v2_encrypted)` | `SELECT * FROM users WHERE eql_v2.lt(encrypted_email, $1)`
| +| `<=` | `eql_v2.lte(eql_v2_encrypted, eql_v2_encrypted)` | `SELECT * FROM users WHERE eql_v2.lte(encrypted_email, $1)`
| +| `>` | `eql_v2.gt(eql_v2_encrypted, eql_v2_encrypted)` | `SELECT * FROM users WHERE eql_v2.gt(encrypted_email, $1)`
| +| `>=` | `eql_v2.gte(eql_v2_encrypted, eql_v2_encrypted)` | `SELECT * FROM users WHERE eql_v2.gte(encrypted_email, $1)`
| +| `~~` | `eql_v2.like(eql_v2_encrypted, eql_v2_encrypted)` | `SELECT * FROM users WHERE eql_v2.like(encrypted_email, $1)`
| +| `~~*` | `eql_v2.ilike(eql_v2_encrypted, eql_v2_encrypted)` | `SELECT * FROM users WHERE eql_v2.ilike(encrypted_email, $1)`
| +| `LIKE` | `eql_v2.like(eql_v2_encrypted, eql_v2_encrypted)` | `SELECT * FROM users WHERE eql_v2.like(encrypted_email, $1)`
| +| `ILIKE` | `eql_v2.ilike(eql_v2_encrypted, eql_v2_encrypted)` | `SELECT * FROM users WHERE eql_v2.ilike(encrypted_email, $1)`
| +| `@>` | `eql_v2.ste_vec_contains(eql_v2_encrypted, eql_v2_encrypted)` | `SELECT * FROM users WHERE eql_v2.ste_vec_contains(encrypted_array, $1)`
| +| `<@` | `eql_v2.ste_vec_contains(eql_v2_encrypted, eql_v2_encrypted)` | `SELECT * FROM users WHERE eql_v2.ste_vec_contains($1, encrypted_array)`
| + +### Core Functions + +| Function | Description | Example | +| ---------------------------- | ---------------------------------------------------- | ----------------------------------------------- | +| `eql_v2.ciphertext(val)` | Extract ciphertext from encrypted value | `SELECT eql_v2.ciphertext(encrypted_field)` | +| `eql_v2.blake3(val)` | Extract blake3 hash from encrypted value | `SELECT eql_v2.blake3(encrypted_field)` | +| `eql_v2.unique(val)` | Extract unique index from encrypted value | `SELECT eql_v2.unique(encrypted_field)` | +| `eql_v2.match(val)` | Extract match index from encrypted value | `SELECT eql_v2.match(encrypted_field)` | +| `eql_v2.ore_64_8_v2(val)` | Extract ORE index from encrypted value | `SELECT eql_v2.ore_64_8_v2(encrypted_field)` | +| `eql_v2.ore_cllw_u64_8(val)` | Extract CLLW ORE index from encrypted value | `SELECT eql_v2.ore_cllw_u64_8(encrypted_field)` | +| `eql_v2.ore_cllw_var_8(val)` | Extract variable CLLW ORE index from encrypted value | `SELECT eql_v2.ore_cllw_var_8(encrypted_field)` | + +### Aggregate Functions + +| Function | Description | Example | +| ----------------- | --------------------------------------- | ------------------------------------ | +| `eql_v2.min(val)` | Get minimum value from encrypted column | `SELECT eql_v2.min(encrypted_field)` | +| `eql_v2.max(val)` | Get maximum value from encrypted column | `SELECT eql_v2.max(encrypted_field)` | + +### Configuration Functions + +| Function | Description | Example | +| ---------------------------------------------------------------------------- | ------------------------------- | ------------------------------------------------------------------------- | +| `eql_v2.config_default(config)` | Get default configuration | `SELECT eql_v2.config_default(NULL)` | +| `eql_v2.config_add_table(table_name, config)` | Add table to configuration | `SELECT eql_v2.config_add_table('users', config)` | +| `eql_v2.config_add_column(table_name, column_name, config)` | Add column to configuration | `SELECT eql_v2.config_add_column('users', 'email', config)` | +| `eql_v2.config_add_cast(table_name, column_name, cast_as, config)` | Add cast configuration | `SELECT eql_v2.config_add_cast('users', 'email', 'text', config)` | +| `eql_v2.config_add_index(table_name, column_name, index_name, opts, config)` | Add index to configuration | `SELECT eql_v2.config_add_index('users', 'email', 'match', opts, config)` | +| `eql_v2.config_match_default()` | Get default match index options | `SELECT eql_v2.config_match_default()` | + +### Example SQL Statements + +#### Equality `=` + +**Operator** + +```sql +SELECT * FROM users WHERE encrypted_email = $1 +``` + +**Function** + +```sql +SELECT * FROM users WHERE eql_v2.eq(encrypted_email, $1) +``` + +#### Like & ILIKE `~~, ~~*` + +**Operator** + +```sql +SELECT * FROM users WHERE encrypted_email LIKE $1 +``` + +**Function** + +```sql +SELECT * FROM users WHERE eql_v2.like(encrypted_email, $1) +``` + +#### Case Sensitivity + +The EQL `eql_v2.like` and `eql_v2.ilike` functions are equivalent. + +The behaviour of EQL's encrypted `LIKE` operators is slightly different to the behaviour of PostgreSQL's `LIKE` operator. +In EQL, the `LIKE` operator can be used on `match` indexes. +Case sensitivity is determined by the [index term configuration](./docs/reference/INDEX.md#options-for-match-indexes-opts) of `match` indexes. +A `match` index term can be configured to enable case sensitive searches with token filters (for example, `downcase` and `upcase`). +The data is encrypted based on the index term configuration. +The `LIKE` operation is always the same, even if the data is tokenised differently. +The different operators are kept to preserve the semantics of SQL statements in client applications. + +### `ORDER BY` + +Ordering requires wrapping the ordered column in the `eql_v2.order_by` function, like this: + +```sql +SELECT * FROM users ORDER BY eql_v2.order_by(encrypted_created_at) DESC +``` + +PostgreSQL uses operators when handling `ORDER BY` operations. The `eql_v2.order_by` function behaves in the same way as the comparison operators, using the appropriate index type (ore_64_8_v2, ore_cllw_u64_8, or ore_cllw_var_8) to determine the ordering. + +### JSONB Support + +All comparison functions also support `jsonb` parameters through automatic type casting. This means you can use either `eql_v2_encrypted` or `jsonb` values in your queries: + +```sql +-- Using eql_v2_encrypted +SELECT * FROM users WHERE eql_v2.eq(encrypted_email, encrypted_value); + +-- Using jsonb +SELECT * FROM users WHERE eql_v2.eq(encrypted_email, jsonb_value); +``` + +The functions will automatically cast the `jsonb` value to `eql_v2_encrypted` before performing the comparison. + +### Array Operations + +EQL supports array operations on encrypted data: + +```sql +-- Get array length +SELECT eql_v2.jsonb_array_length(encrypted_array) FROM users; + +-- Get array elements +SELECT eql_v2.jsonb_array_elements(encrypted_array) FROM users; + +-- Get array element ciphertexts +SELECT eql_v2.jsonb_array_elements_text(encrypted_array) FROM users; +``` + +### JSON Path Operations + +EQL supports JSON path operations on encrypted data: + +```sql +-- Get encrypted value at path +SELECT encrypted_data->'$.field' FROM users; + +-- Get ciphertext at path +SELECT encrypted_data->>'$.field' FROM users; +``` diff --git a/diagrams/overview-insert.drawio.svg b/diagrams/overview-insert.drawio.svg deleted file mode 100644 index cef7acf7..00000000 --- a/diagrams/overview-insert.drawio.svg +++ /dev/null @@ -1,457 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - -
-
-
-
-
- - INSERT INTO - - - users - - - ( - - - name - - - ) - -
-
-
- - - VALUES ( - - - {"p": "plaintext"} - - - ) - - -
-
-
-
-
- - - INSERT INTO users (name)... - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
-
- Cipherstash Proxy -
-
-
- - - Cipherstash Proxy - - - - - - - -
-
-
- PostgreSQL -
-
-
- - - PostgreSQL - - - - - - - -
-
-
- Encrypt -
-
-
- - - Encrypt - - - - - - - - - - - - - - - - - - - - - users - - - - - - -
-
-
- 1 -
-
-
- - - 1 - - - - - - - -
-
-
- ******** -
-
-
- - - ******** - - - - - - - -
-
-
- 2 -
-
-
- - - 2 - - - - - - - -
-
-
- ******** -
-
-
- - - ******** - - - - - - - -
-
-
- 3 -
-
-
- - - 3 - - - - - - - -
-
-
- - ******** - -
-
-
- - - ******** - - - - - - -
-
-
-
-
-
- - - - - - - - - -
-
-
- - 1 - -
-
-
- - - 1 - - - - - - -
-
-
- - Plaintext - -
-
-
- - - Plaintext - - - - - - - -
-
-
- - 2 - -
-
-
- - - 2 - - - - - - -
-
-
- - Encryption - -
-
-
- - - Encryption - - - - - - - -
-
-
- - 3 - -
-
-
- - - 3 - - - - - - -
-
-
- - Storage - -
-
-
- - - Storage - - - - - - - -
-
-
- - 1 - -
-
-
- - - 1 - - - - - - - -
-
-
- - 2 - -
-
-
- - - 2 - - - - - - - -
-
-
- - 3 - -
-
-
- - - 3 - - - - - - - - - Text is not SVG - cannot display - - - - \ No newline at end of file diff --git a/diagrams/overview-select.drawio.svg b/diagrams/overview-select.drawio.svg deleted file mode 100644 index 1effb5a9..00000000 --- a/diagrams/overview-select.drawio.svg +++ /dev/null @@ -1,552 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
-
- Cipherstash Proxy -
-
-
- - - Cipherstash Proxy - - - - - - - -
-
-
- PostgreSQL -
-
-
- - - PostgreSQL - - - - - - - -
-
-
- Encrypt -
-
-
- - - Encrypt - - - - - - - - - - - - - - - - - - - - - users - - - - - - -
-
-
- 1 -
-
-
- - - 1 - - - - - - - -
-
-
- ******** -
-
-
- - - ******** - - - - - - - -
-
-
- 2 -
-
-
- - - 2 - - - - - - - -
-
-
- ******** -
-
-
- - - ******** - - - - - - - -
-
-
- 3 -
-
-
- - - 3 - - - - - - - -
-
-
- - ******** - -
-
-
- - - ******** - - - - - - -
-
-
-
-
-
- - - - - - - - - -
-
-
- - 1 - -
-
-
- - - 1 - - - - - - -
-
-
- - Wrap - -
-
-
- - - Wrap - - - - - - - -
-
-
- - 2 - -
-
-
- - - 2 - - - - - - -
-
-
- - Encryption - -
-
-
- - - Encryption - - - - - - - -
-
-
- - 3 - -
-
-
- - - 3 - - - - - - -
-
-
- - Execute - -
-
-
- - - Execute - - - - - - - -
-
-
- - 1 - -
-
-
- - - 1 - - - - - - - -
-
-
- - 3 - -
-
-
- - - 3 - - - - - - -
-
-
-
-
- - - SELECT - - name - - - -
-
- - - FROM - - users - - - -
-
-
- - - - WHERE - - - - cs_match_v1( - - - - name - - - - ) - - - @> cs - - - _match_v1 - - - ($1); - - -
-
-
-
-
-
- - - SELECT name... - - - - - - - -
-
-
- - 4 - -
-
-
- - - 4 - - - - - - -
-
-
- - Decrypt - -
-
-
- - - Decrypt - - - - - - - - - - - -
-
-
- - 2 - -
-
-
- - - 2 - - - - - - - -
-
-
- - 4 - -
-
-
- - - 4 - - - - - - -
-
-
- Decrypt -
-
-
- - - Decrypt - - - - - - - - - Text is not SVG - cannot display - - - - diff --git a/docs/reference/INDEX.md b/docs/reference/INDEX.md index df263821..95d7cd48 100644 --- a/docs/reference/INDEX.md +++ b/docs/reference/INDEX.md @@ -1,17 +1,17 @@ # EQL index configuration The following functions allow you to configure indexes for encrypted columns. -All these functions modify the `cs_configuration_v1` table in your database, and are added during the EQL installation. +All these functions modify the `cs_configuration_v2` table in your database, and are added during the EQL installation. > **IMPORTANT:** When you modify or add an index, you must re-encrypt data that's already been stored in the database. -The CipherStash encryption solution will encrypt the data based on the current state of the configuration. +> The CipherStash encryption solution will encrypt the data based on the current state of the configuration. ### Adding an index (`cs_add_index`) Add an index to an encrypted column. ```sql -SELECT cs_add_index_v1( +SELECT cs_add_index_v2( 'table_name', -- Name of the table 'column_name', -- Name of the column 'index_name', -- Index kind ('unique', 'match', 'ore', 'ste_vec') @@ -185,15 +185,15 @@ Query terms are processed in the same manner as the input document. A query prior to encrypting and indexing looks like a structurally similar subset of the encrypted document. For example: ```json -{ - "account": { - "email": "alice@example.com", - "roles": "admin" - } +{ + "account": { + "email": "alice@example.com", + "roles": "admin" + } } ``` -The expression `cs_ste_vec_v1(encrypted_account) @> cs_ste_vec_v1($query)` would match all records where the `encrypted_account` column contains a JSONB object with an "account" key containing an object with an "email" key where the value is the string "alice@example.com". +The expression `cs_ste_vec_v2(encrypted_account) @> cs_ste_vec_v2($query)` would match all records where the `encrypted_account` column contains a JSONB object with an "account" key containing an object with an "email" key where the value is the string "alice@example.com". When reduced to a prefix list, it would look like this: @@ -219,11 +219,11 @@ Modifies an existing index configuration. Accepts the same parameters as `cs_add_index` ```sql -SELECT cs_modify_index_v1( - table_name text, - column_name text, - index_name text, - cast_as text, +SELECT cs_modify_index_v2( + table_name text, + column_name text, + index_name text, + cast_as text, opts jsonb ); ``` @@ -233,12 +233,13 @@ SELECT cs_modify_index_v1( Removes an index configuration from the column. ```sql -SELECT cs_remove_index_v1( - table_name text, - column_name text, +SELECT cs_remove_index_v2( + table_name text, + column_name text, index_name text ); ``` + --- ### Didn't find what you wanted? diff --git a/docs/reference/JSON.md b/docs/reference/JSON.md index 3e8fb1d7..7d637788 100644 --- a/docs/reference/JSON.md +++ b/docs/reference/JSON.md @@ -8,9 +8,9 @@ EQL supports encrypting, decrypting, and searching JSON and JSONB objects. - [Inserting JSON data](#inserting-json-data) - [Reading JSON data](#reading-json-data) - [Querying JSONB data with EQL](#querying-jsonb-data-with-eql) - - [Containment queries (`cs_ste_vec_v1`)](#containment-queries-cs_ste_vec_v1) - - [Field extraction (`cs_ste_vec_value_v1`)](#field-extraction-cs_ste_vec_value_v1) - - [Field comparison (`cs_ste_vec_term_v1`)](#field-comparison-cs_ste_vec_term_v1) + - [Containment queries (`cs_ste_vec_v2`)](#containment-queries-cs_ste_vec_v2) + - [Field extraction (`cs_ste_vec_value_v2`)](#field-extraction-cs_ste_vec_value_v2) + - [Field comparison (`cs_ste_vec_term_v2`)](#field-comparison-cs_ste_vec_term_v2) - [Grouping data](#grouping-data) - [EQL functions for JSONB and `ste_vec`](#eql-functions-for-jsonb-and-ste_vec) - [EJSON paths](#ejson-paths) @@ -36,9 +36,9 @@ Similar to how you configure indexes for text data, you can configure indexes fo The only difference is that you need to specify the `cast_as` parameter as `json` or `jsonb`. ```sql -SELECT cs_add_index_v1( - 'users', - 'encrypted_json', +SELECT cs_add_index_v2( + 'users', + 'encrypted_json', 'ste_vec', 'jsonb', '{"prefix": "users/encrypted_json"}' -- The prefix is in the form of "table/column" @@ -49,7 +49,7 @@ You can read more about the index configuration options [here](https://github.co ### Inserting JSON data -When inserting JSON data, this works the same as inserting text data. +When inserting JSON data, this works the same as inserting text data. You need to wrap the JSON data in the appropriate EQL payload. CipherStash Proxy will **encrypt** the data automatically. @@ -61,7 +61,7 @@ Assuming you want to store the following JSON data: { "name": "John Doe", "metadata": { - "age": 42, + "age": 42 } } ``` @@ -84,9 +84,7 @@ Data is stored in the database as: }, "k": "sv", "v": 1, - "sv": [ - ["ciphertext"] - ] + "sv": [["ciphertext"]] } ``` @@ -119,7 +117,7 @@ Data is returned as: EQL provides specialized functions to interact with encrypted JSONB data, supporting operations like containment queries, field extraction, and comparisons. -### Containment queries (`cs_ste_vec_v1`) +### Containment queries (`cs_ste_vec_v2`) Retrieve the Structured Encryption Vector for JSONB containment queries. @@ -141,7 +139,7 @@ We can query records that contain a specific structure. ```sql SELECT * FROM examples -WHERE cs_ste_vec_v1(encrypted_json) @> cs_ste_vec_v1( +WHERE cs_ste_vec_v2(encrypted_json) @> cs_ste_vec_v2( '{ "v":1, "k":"pt", @@ -169,7 +167,7 @@ If we query for a value that does not exist in the data: ```sql SELECT * FROM examples -WHERE cs_ste_vec_v1(encrypted_json) @> cs_ste_vec_v1( +WHERE cs_ste_vec_v2(encrypted_json) @> cs_ste_vec_v2( '{ "v":1, "k":"pt", @@ -182,7 +180,7 @@ WHERE cs_ste_vec_v1(encrypted_json) @> cs_ste_vec_v1( This query would return no results, as the value `"d"` is not present in the `"nested"` array. -### Field extraction (`cs_ste_vec_value_v1`) +### Field extraction (`cs_ste_vec_value_v2`) Extract a field from an encrypted JSONB object. @@ -203,7 +201,7 @@ We can extract the value of the `"top"` key. **SQL query:** ```sql -SELECT cs_ste_vec_value_v1(encrypted_json, +SELECT cs_ste_vec_value_v2(encrypted_json, '{ "v":1, "k":"pt", @@ -230,7 +228,7 @@ FROM examples; } ``` -### Field comparison (`cs_ste_vec_term_v1`) +### Field comparison (`cs_ste_vec_term_v2`) Select rows based on a field value in an encrypted JSONB object. @@ -250,7 +248,7 @@ We can query records where the `"num"` field is greater than `2`. ```sql SELECT * FROM examples -WHERE cs_ste_vec_term_v1(encrypted_json, +WHERE cs_ste_vec_term_v2(encrypted_json, '{ "v":1, "k":"pt", @@ -258,7 +256,7 @@ WHERE cs_ste_vec_term_v1(encrypted_json, "i":{"t":"examples","c":"encrypted_json"}, "q":"ejson_path" }' -) > cs_ste_vec_term_v1( +) > cs_ste_vec_term_v2( '{ "v":1, "k":"pt", @@ -278,7 +276,7 @@ WHERE (jsonb_column->>'num')::int > 2; ### Grouping data -Use `cs_ste_vec_term_v1` along with `cs_grouped_value_v1` to group by a field in an encrypted JSONB column. +Use `cs_ste_vec_term_v2` along with `cs_grouped_value_v2` to group by a field in an encrypted JSONB column. **Example:** @@ -298,7 +296,7 @@ We can group the data by the `"color"` field and count occurrences. **SQL query:** ```sql -SELECT cs_grouped_value_v1(cs_ste_vec_value_v1(encrypted_json, +SELECT cs_grouped_value_v2(cs_ste_vec_value_v2(encrypted_json, '{ "v":1, "k":"pt", @@ -308,7 +306,7 @@ SELECT cs_grouped_value_v1(cs_ste_vec_value_v1(encrypted_json, }' )) AS color, COUNT(*) FROM examples -GROUP BY cs_ste_vec_term_v1(encrypted_json, +GROUP BY cs_ste_vec_term_v2(encrypted_json, '{ "v":1, "k":"pt", @@ -330,25 +328,25 @@ GROUP BY jsonb_column->>'color'; **Result:** | color | count | -|-------|-------| -| blue | 3 | -| green | 2 | -| red | 1 | +| ----- | ----- | +| blue | 3 | +| green | 2 | +| red | 1 | ## EQL Functions for JSONB and `ste_vec` - **Index management** - - `cs_add_index_v1(table_name text, column_name text, 'ste_vec', 'jsonb', opts jsonb)`: Adds an `ste_vec` index configuration. + - `cs_add_index_v2(table_name text, column_name text, 'ste_vec', 'jsonb', opts jsonb)`: Adds an `ste_vec` index configuration. - `opts` must include the `"context"` key. - + - **Query functions** - - `cs_ste_vec_v1(val jsonb)`: Retrieves the STE vector for JSONB containment queries. - - `cs_ste_vec_term_v1(val jsonb, epath jsonb)`: Retrieves the encrypted term associated with an encrypted JSON path. - - `cs_ste_vec_value_v1(val jsonb, epath jsonb)`: Retrieves the decrypted value associated with an encrypted JSON path. - - `cs_ste_vec_terms_v1(val jsonb, epath jsonb)`: Retrieves an array of encrypted terms for elements in an array at the given JSON path (used for comparisons). - - `cs_grouped_value_v1(val jsonb)`: Used with `ste_vec` indexes for grouping. + - `cs_ste_vec_v2(val jsonb)`: Retrieves the STE vector for JSONB containment queries. + - `cs_ste_vec_term_v2(val jsonb, epath jsonb)`: Retrieves the encrypted term associated with an encrypted JSON path. + - `cs_ste_vec_value_v2(val jsonb, epath jsonb)`: Retrieves the decrypted value associated with an encrypted JSON path. + - `cs_ste_vec_terms_v2(val jsonb, epath jsonb)`: Retrieves an array of encrypted terms for elements in an array at the given JSON path (used for comparisons). + - `cs_grouped_value_v2(val jsonb)`: Used with `ste_vec` indexes for grouping. ## EJSON paths @@ -390,10 +388,10 @@ EQL JSONB functions accept an [eJSONPath](#ejson-paths) as an argument (instead #### Decryption example -`cs_ste_vec_value_v1` returns the plaintext EQL payload to the client. +`cs_ste_vec_value_v2` returns the plaintext EQL payload to the client. ```sql -SELECT cs_ste_vec_value_v1(encrypted_json, $1) FROM examples; +SELECT cs_ste_vec_value_v2(encrypted_json, $1) FROM examples; ``` ```javascript @@ -417,11 +415,11 @@ SELECT cs_ste_vec_value_v1(encrypted_json, $1) FROM examples; #### Comparison example -`cs_ste_vec_term_v1` returns an ORE term for comparison. +`cs_ste_vec_term_v2` returns an ORE term for comparison. ```sql SELECT * FROM examples -WHERE cs_ste_vec_term_v1(examples.encrypted_json, $1) > cs_ste_vec_term_v1($2) +WHERE cs_ste_vec_term_v2(examples.encrypted_json, $1) > cs_ste_vec_term_v2($2) ``` ```javascript @@ -473,15 +471,15 @@ EQL JSONB functions accept an [eJSONPath](#ejson-paths) as an argument (instead #### Decryption example -EQL currently doesn't support returning a specific array element for decryption, but `cs_ste_vec_value_v1` can be used to return an array to the client to process. +EQL currently doesn't support returning a specific array element for decryption, but `cs_ste_vec_value_v2` can be used to return an array to the client to process. The query: ```sql -SELECT cs_ste_vec_value_v1(encrypted_json, $1) AS val FROM examples; +SELECT cs_ste_vec_value_v2(encrypted_json, $1) AS val FROM examples; ``` -With the params: +With the params: ```javascript // Assume that examples.encrypted_json has JSON objects with the shape: @@ -520,19 +518,19 @@ Would return the EQL plaintext payload with an array (`[1, 2, 3]` for example): #### Comparison example -`cs_ste_vec_terms_v1` can be used with the native PostgreSQL array access operator to get a term for comparison by array index. +`cs_ste_vec_terms_v2` can be used with the native PostgreSQL array access operator to get a term for comparison by array index. -The eJSONPath used with `cs_ste_vec_terms_v1` needs to end with `[*]` (`$.some_array_field[*]` for example). +The eJSONPath used with `cs_ste_vec_terms_v2` needs to end with `[*]` (`$.some_array_field[*]` for example). > [!IMPORTANT] -> Array access with `cs_ste_vec_terms_v1` only works when the given eJSONPath only matches a single array. -> Accessing array elements from `cs_ste_vec_terms_v1` when the eJSONPath matches multiple arrays (for example, when there are nested arrays or multiple arrays at the same depth) can return unexpected results. +> Array access with `cs_ste_vec_terms_v2` only works when the given eJSONPath only matches a single array. +> Accessing array elements from `cs_ste_vec_terms_v2` when the eJSONPath matches multiple arrays (for example, when there are nested arrays or multiple arrays at the same depth) can return unexpected results. The following query compares the first item in the array at the eJSONPath in `$1` to the value in `$2`. ```sql SELECT * FROM examples -WHERE (cs_ste_vec_terms_v1(examples.encrypted_json, $1))[1] > cs_ste_vec_term_v1($2) +WHERE (cs_ste_vec_terms_v2(examples.encrypted_json, $1))[1] > cs_ste_vec_term_v2($2) ``` ```javascript @@ -587,10 +585,10 @@ The difference in these examples is that the path does a lookup multiple levels #### Decryption example -`cs_ste_vec_value_v1` returns the plaintext EQL payload to the client. +`cs_ste_vec_value_v2` returns the plaintext EQL payload to the client. ```sql -SELECT cs_ste_vec_value_v1(encrypted_json, $1) FROM examples; +SELECT cs_ste_vec_value_v2(encrypted_json, $1) FROM examples; ``` ```javascript @@ -616,11 +614,11 @@ SELECT cs_ste_vec_value_v1(encrypted_json, $1) FROM examples; #### Comparison example -`cs_ste_vec_term_v1` returns an ORE term for comparison. +`cs_ste_vec_term_v2` returns an ORE term for comparison. ```sql SELECT * FROM examples -WHERE cs_ste_vec_term_v1(examples.encrypted_json, $1) > cs_ste_vec_term_v1($2) +WHERE cs_ste_vec_term_v2(examples.encrypted_json, $1) > cs_ste_vec_term_v2($2) ``` ```javascript @@ -670,18 +668,18 @@ SELECT '{"b":2}'::jsonb <@ '{"a":1, "b":2}'::jsonb; **EQL example** -EQL uses the same operators for containment (`@>` and `<@`) queries, but the args need to be wrapped in `cs_ste_vec_v1`. +EQL uses the same operators for containment (`@>` and `<@`) queries, but the args need to be wrapped in `cs_ste_vec_v2`. Example query: ```sql -- Checks if the left arg (the `examples.encrypted_json` column) contains the right arg ($1). -- Would return `true` for the example data and param below. -SELECT * WHERE cs_ste_vec_v1(encrypted_json) @> cs_ste_vec_v1($1) FROM examples; +SELECT * WHERE cs_ste_vec_v2(encrypted_json) @> cs_ste_vec_v2($1) FROM examples; -- Checks if the the right arg ($1) contains left arg (the `examples.encrypted_json` column). -- Would return `false` for the example data and param below. -SELECT * WHERE cs_ste_vec_v1(encrypted_json) <@ cs_ste_vec_v1($1) FROM examples; +SELECT * WHERE cs_ste_vec_v2(encrypted_json) <@ cs_ste_vec_v2($1) FROM examples; ``` Example params: @@ -733,12 +731,12 @@ SELECT * from jsonb_array_elements_text('["a", "b"]'); #### Decryption example -EQL currently doesn't support returning a `SETOF` values for decryption (for returning a row per item in an array), but `cs_ste_vec_value_v1` can be used to return an array to the client to process. +EQL currently doesn't support returning a `SETOF` values for decryption (for returning a row per item in an array), but `cs_ste_vec_value_v2` can be used to return an array to the client to process. The query: ```sql -SELECT cs_ste_vec_value_v1(encrypted_json, $1) AS val FROM examples; +SELECT cs_ste_vec_value_v2(encrypted_json, $1) AS val FROM examples; ``` With the params: @@ -780,10 +778,10 @@ Would return the EQL plaintext payload with an array (`[1, 2, 3]` for example): #### Comparison example -`cs_ste_vec_terms_v1` (note that terms is plural) can be used to return an array of ORE terms for comparison. +`cs_ste_vec_terms_v2` (note that terms is plural) can be used to return an array of ORE terms for comparison. The array can be `unnest`ed to work with a `SETOF` ORE terms for comparison. -The eJSONPath used with `cs_ste_vec_terms_v1` needs to end with `[*]` (`$.some_array_field[*]` for example). +The eJSONPath used with `cs_ste_vec_terms_v2` needs to end with `[*]` (`$.some_array_field[*]` for example). Example query: @@ -791,8 +789,8 @@ Example query: SELECT id FROM examples e WHERE EXISTS ( SELECT 1 - FROM unnest(cs_ste_vec_terms_v1(e.encrypted_json, $1)) AS term - WHERE term > cs_ste_vec_term_v1($2) + FROM unnest(cs_ste_vec_terms_v2(e.encrypted_json, $1)) AS term + WHERE term > cs_ste_vec_term_v2($2) ); ``` @@ -840,23 +838,23 @@ SELECT jsonb_array_length('[1, 2, 3]'); **EQL example** -The PostgreSQL `array_length` function can be used with `cs_ste_vec_terms_v1` to find the length of an array. +The PostgreSQL `array_length` function can be used with `cs_ste_vec_terms_v2` to find the length of an array. -The eJSONPath used with `cs_ste_vec_terms_v1` needs to end with `[*]` (`$.some_array_field[*]` for example). +The eJSONPath used with `cs_ste_vec_terms_v2` needs to end with `[*]` (`$.some_array_field[*]` for example). > [!IMPORTANT] -> Determining array length with `cs_ste_vec_terms_v1` only works when the given eJSONPath only matches a single array. -> Attempting to determine array length using `cs_ste_vec_terms_v1` when the eJSONPath matches multiple arrays (for example, when there are nested arrays or multiple arrays at the same depth) can return unexpected results. +> Determining array length with `cs_ste_vec_terms_v2` only works when the given eJSONPath only matches a single array. +> Attempting to determine array length using `cs_ste_vec_terms_v2` when the eJSONPath matches multiple arrays (for example, when there are nested arrays or multiple arrays at the same depth) can return unexpected results. Example query: ```sql -SELECT COALESCE( -- We `COALESCE` because cs_ste_vec_terms_v1 will return `NULL` for empty arrays. - array_length( -- `cs_ste_vec_terms_v1` returns an array type (not JSON(B)), so we use `array_length`. - cs_ste_vec_terms_v1(encrypted_json, $1), -- Pluck out the array of terms at the path in $1. +SELECT COALESCE( -- We `COALESCE` because cs_ste_vec_terms_v2 will return `NULL` for empty arrays. + array_length( -- `cs_ste_vec_terms_v2` returns an array type (not JSON(B)), so we use `array_length`. + cs_ste_vec_terms_v2(encrypted_json, $1), -- Pluck out the array of terms at the path in $1. 1 -- The array dimension to find the length of (term array are flat, so this should always be 1). ), - 0 -- Assume a length of `0` when `cs_ste_vec_terms_v1` returns `NULL`. + 0 -- Assume a length of `0` when `cs_ste_vec_terms_v2` returns `NULL`. ) AS len FROM examples; ``` @@ -880,6 +878,7 @@ Example data and params: "q": "ejson_path" } ``` + --- ### Didn't find what you wanted? diff --git a/docs/reference/PAYLOAD.md b/docs/reference/PAYLOAD.md index 2bf011a6..e3914c50 100644 --- a/docs/reference/PAYLOAD.md +++ b/docs/reference/PAYLOAD.md @@ -37,7 +37,7 @@ CipherStash Proxy will handle the plaintext payload and create the encrypted pay ## Data format -The format is defined as a [JSON Schema](../../sql/schemas/cs_encrypted_v1.schema.json). +The format is defined as a [JSON Schema](../../sql/schemas/cs_encrypted_v2.schema.json). It should never be necessary to directly interact with the stored `jsonb`. CipherStash Proxy handles the encoding, and EQL provides the functions. diff --git a/docs/tutorials/GETTINGSTARTED.md b/docs/tutorials/GETTINGSTARTED.md index d0a7c44f..185e7fec 100644 --- a/docs/tutorials/GETTINGSTARTED.md +++ b/docs/tutorials/GETTINGSTARTED.md @@ -132,19 +132,19 @@ In the previous step we: In this part we will add a new column to store our encrypted email data. -When we add the column we use a `Type` of `cs_encrypted_v1`. +When we add the column we use a `Type` of `cs_encrypted_v2`. This type will enforce constraints on the field to ensure that: - the payload is in the format EQL and CipherStash Proxy expects. - the payload has been encrypted before inserting. -If there are issues with the payload being inserted into a field with a type of `cs_encrypted_v1`, an error will be returned describing what the issue with the payload is. +If there are issues with the payload being inserted into a field with a type of `cs_encrypted_v2`, an error will be returned describing what the issue with the payload is. -To add a new column called `email_encrypted` with a type of `cs_encrypted_v1`: +To add a new column called `email_encrypted` with a type of `cs_encrypted_v2`: ```sql -ALTER TABLE users ADD email_encrypted cs_encrypted_v1; +ALTER TABLE users ADD email_encrypted cs_encrypted_v2; ``` Our `users` schema now looks like this: @@ -152,7 +152,7 @@ Our `users` schema now looks like this: | Column | Type | Nullable | | ----------------- | ------------------------ | -------- | | `email` | `character varying(100)` | | -| `email_encrypted` | `cs_encrypted_v1` | | +| `email_encrypted` | `cs_encrypted_v2` | | ### Adding indexes @@ -160,7 +160,7 @@ We now have our database schema setup to store encrypted data. In this part we will learn about why we need to add indexes and how to add them. -When you install EQL, a table called `cs_configuration_v1` is created in your database. +When you install EQL, a table called `cs_configuration_v2` is created in your database. Adding indexes updates this table with the details and configuration needed for CipherStash Proxy to know how to encrypt your data, and what types of queries are able to be performed @@ -178,25 +178,25 @@ This means that we need to add the below indexes for our new `email_encrypted` f For free text queries (e.g `LIKE`, `ILIKE`) we add a `match` index and a GIN index: ```sql -SELECT cs_add_index_v1('users', 'email_encrypted', 'match', 'text'); -CREATE INDEX ON users USING GIN (cs_match_v1(email_encrypted)); +SELECT cs_add_index_v2('users', 'email_encrypted', 'match', 'text'); +CREATE INDEX ON users USING GIN (cs_match_v2(email_encrypted)); ``` For equality queries we add a `unique` index: ```sql -SELECT cs_add_index_v1('users', 'email_encrypted', 'unique', 'text', '{"token_filters": [{"kind": "downcase"}]}'); -CREATE UNIQUE INDEX ON users(cs_unique_v1(email_encrypted)); +SELECT cs_add_index_v2('users', 'email_encrypted', 'unique', 'text', '{"token_filters": [{"kind": "downcase"}]}'); +CREATE UNIQUE INDEX ON users(cs_unique_v2(email_encrypted)); ``` For ordering or comparison queries we add an `ore` index: ```sql -SELECT cs_add_index_v1('users', 'email_encrypted', 'ore', 'text'); -CREATE INDEX ON users (cs_ore_64_8_v1(email_encrypted)); +SELECT cs_add_index_v2('users', 'email_encrypted', 'ore', 'text'); +CREATE INDEX ON users (cs_ore_64_8_v2(email_encrypted)); ``` -After adding these indexes, our `cs_configuration_v1` table will look like this: +After adding these indexes, our `cs_configuration_v2` table will look like this: ```bash id | 1 @@ -209,11 +209,11 @@ The initial `state` will be set as pending. To activate this configuration run: ```sql -SELECT cs_encrypt_v1(); -SELECT cs_activate_v1(); +SELECT cs_encrypt_v2(); +SELECT cs_activate_v2(); ``` -The `cs_configured_v1` table will now have a state of `active`. +The `cs_configured_v2` table will now have a state of `active`. ```bash id | 1 @@ -228,7 +228,7 @@ Prerequisites: - [Database is setup](#setup-your-database) - [Indexes added](#adding-indexes) -Ensure CipherStash Proxy has the most up to date configuration from the `cs_configuration_v1` table. +Ensure CipherStash Proxy has the most up to date configuration from the `cs_configuration_v2` table. CipherStash Proxy pings the database every 60 seconds to refresh the configuration but we can force the refresh by running: @@ -423,7 +423,7 @@ Prerequsites: - A [match index](#adding-indexes) is needed on the encrypted column to support this operation. - Connected to the database via the Proxy. -EQL function to use: `cs_match_v1(val JSONB)` +EQL function to use: `cs_match_v2(val JSONB)` EQL query payload for a match query: @@ -449,7 +449,7 @@ SELECT * FROM users WHERE email LIKE '%grace%'; The EQL equivalent of this query is: ```sql -SELECT * FROM users WHERE cs_match_v1(email_encrypted) @> cs_match_v1( +SELECT * FROM users WHERE cs_match_v2(email_encrypted) @> cs_match_v2( '{"v":1,"k":"pt","p":"grace","i":{"t":"users","c":"email_encrypted"},"q":"match"}' ); ``` @@ -466,7 +466,7 @@ Prerequsites: - A [unique index](#adding-indexes) is needed on the encrypted column to support this operation. -EQL function to use: `cs_unique_v1(val JSONB)` +EQL function to use: `cs_unique_v2(val JSONB)` EQL query payload for a match query: @@ -492,7 +492,7 @@ SELECT * FROM users WHERE email = 'adalovelace@example.com'; The EQL equivalent of this query is: ```sql -SELECT * FROM users WHERE cs_unique_v1(email_encrypted) = cs_unique_v1( +SELECT * FROM users WHERE cs_unique_v2(email_encrypted) = cs_unique_v2( '{"v":1,"k":"pt","p":"adalovelace@example.com","i":{"t":"users","c":"email_encrypted"},"q":"unique"}' ); ``` @@ -509,7 +509,7 @@ Prerequsites: - An [ore index](#adding-indexes) is needed on the encrypted column to support this operation. -EQL function to use: `cs_ore_64_8_v1(val JSONB)`. +EQL function to use: `cs_ore_64_8_v2(val JSONB)`. A plaintext query order by email looks like this: @@ -520,7 +520,7 @@ SELECT * FROM users ORDER BY email ASC; The EQL equivalent of this query is: ```sql -SELECT * FROM users ORDER BY cs_ore_64_8_v1(email_encrypted) ASC; +SELECT * FROM users ORDER BY cs_ore_64_8_v2(email_encrypted) ASC; ``` This query returns: @@ -538,7 +538,7 @@ Prerequsites: - A [unique index](#adding-indexes) is needed on the encrypted column to support this operation. -EQL function to use: `cs_ore_64_8_v1(val JSONB)`. +EQL function to use: `cs_ore_64_8_v2(val JSONB)`. EQL query payload for a comparison query: @@ -564,7 +564,7 @@ SELECT * FROM users WHERE email > 'gracehopper@test.com'; The EQL equivalent of this query is: ```sql -SELECT * FROM users WHERE cs_ore_64_8_v1(email_encrypted) > cs_ore_64_8_v1( +SELECT * FROM users WHERE cs_ore_64_8_v2(email_encrypted) > cs_ore_64_8_v2( '{"v":1,"k":"pt","p":"gracehopper@test.com","i":{"t":"users","c":"email_encrypted"},"q":"ore"}' ); ``` diff --git a/examples/go/xorm/README.md b/examples/go/xorm/README.md index 8f11aef2..295c2694 100644 --- a/examples/go/xorm/README.md +++ b/examples/go/xorm/README.md @@ -12,13 +12,13 @@ 1. Set up the [playground environment](../../playground/README.md). 2. Run the setup script: - ```shell - ./run.sh setup - ``` + ```shell + ./run.sh setup + ``` 3. Run tests: ```shell ./run.sh tests - ``` + ``` ## Integrating EQL into a Xorm app @@ -116,10 +116,10 @@ Example: ```sql ALTER TABLE goexamples ADD CONSTRAINT encrypted_text_field_encrypted_check - CHECK ( cs_check_encrypted_v1(encrypted_text_field) ); + CHECK ( cs_check_encrypted_v2(encrypted_text_field) ); ALTER TABLE goexamples ADD CONSTRAINT encrypted_jsonb_encrypted_check - CHECK ( cs_check_encrypted_v1(encrypted_jsonb_field) ); + CHECK ( cs_check_encrypted_v2(encrypted_jsonb_field) ); ``` 5. [Add indexes](../../../README.md#managing-indexes-with-eql): @@ -127,22 +127,22 @@ Example: Example: ```sql - SELECT cs_add_index_v1('goexamples', 'encrypted_text_field', 'unique', 'text', '{"token_filters": [{"kind": "downcase"}]}'); - SELECT cs_add_index_v1('goexamples', 'encrypted_text_field', 'match', 'text'); - SELECT cs_add_index_v1('goexamples', 'encrypted_text_field', 'ore', 'text'); - SELECT cs_add_index_v1('goexamples', 'encrypted_jsonb_field', 'ste_vec', 'jsonb', '{"prefix": "goexamples/encrypted_jsonb_field"}'); + SELECT cs_add_index_v2('goexamples', 'encrypted_text_field', 'unique', 'text', '{"token_filters": [{"kind": "downcase"}]}'); + SELECT cs_add_index_v2('goexamples', 'encrypted_text_field', 'match', 'text'); + SELECT cs_add_index_v2('goexamples', 'encrypted_text_field', 'ore', 'text'); + SELECT cs_add_index_v2('goexamples', 'encrypted_jsonb_field', 'ste_vec', 'jsonb', '{"prefix": "goexamples/encrypted_jsonb_field"}'); -- The below indexes will also need to be added to enable full search functionality on the encrypted columns - CREATE UNIQUE INDEX ON goexamples(cs_unique_v1(encrypted_text_field)); - CREATE INDEX ON goexamples USING GIN (cs_match_v1(encrypted_text_field)); - CREATE INDEX ON goexamples (cs_ore_64_8_v1(encrypted_text_field)); - CREATE INDEX ON goexamples USING GIN (cs_ste_vec_v1(encrypted_jsonb_field)); + CREATE UNIQUE INDEX ON goexamples(cs_unique_v2(encrypted_text_field)); + CREATE INDEX ON goexamples USING GIN (cs_match_v2(encrypted_text_field)); + CREATE INDEX ON goexamples (cs_ore_64_8_v2(encrypted_text_field)); + CREATE INDEX ON goexamples USING GIN (cs_ste_vec_v2(encrypted_jsonb_field)); -- Run these functions to activate - SELECT cs_encrypt_v1(); - SELECT cs_activate_v1(); + SELECT cs_encrypt_v2(); + SELECT cs_activate_v2(); ``` ## Inserting @@ -179,7 +179,7 @@ Below is an example of running a match query on a text field. log.Fatalf("Error marshaling encrypted_text_field: %v", errTwo) } - has, errThree := engine.Where("cs_match_v1(encrypted_text_field) @> cs_match_v1(?)", query).Get(&ExampleTwo) + has, errThree := engine.Where("cs_match_v2(encrypted_text_field) @> cs_match_v2(?)", query).Get(&ExampleTwo) if errThree != nil { log.Fatalf("Could not retrieve exampleTwo: %v", errThree) } diff --git a/examples/go/xorm/custom-types.sql b/examples/go/xorm/custom-types.sql index babfb82e..0721b5f2 100644 --- a/examples/go/xorm/custom-types.sql +++ b/examples/go/xorm/custom-types.sql @@ -1,14 +1,14 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; - CREATE TYPE ore_64_8_v1_term AS ( + CREATE TYPE ore_64_8_v2_term AS ( bytes bytea ); - CREATE TYPE ore_64_8_v1 AS ( - terms ore_64_8_v1_term[] + CREATE TYPE ore_64_8_v2 AS ( + terms ore_64_8_v2_term[] ); - CREATE OR REPLACE FUNCTION compare_ore_64_8_v1_term(a ore_64_8_v1_term, b ore_64_8_v1_term) returns integer AS $$ + CREATE OR REPLACE FUNCTION compare_ore_64_8_v2_term(a ore_64_8_v2_term, b ore_64_8_v2_term) returns integer AS $$ DECLARE eq boolean := true; unequal_block smallint := 0; @@ -85,34 +85,34 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; $$ LANGUAGE plpgsql; - CREATE OR REPLACE FUNCTION ore_64_8_v1_term_eq(a ore_64_8_v1_term, b ore_64_8_v1_term) RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1_term(a, b) = 0 + CREATE OR REPLACE FUNCTION ore_64_8_v2_term_eq(a ore_64_8_v2_term, b ore_64_8_v2_term) RETURNS boolean AS $$ + SELECT compare_ore_64_8_v2_term(a, b) = 0 $$ LANGUAGE SQL; - CREATE OR REPLACE FUNCTION ore_64_8_v1_term_neq(a ore_64_8_v1_term, b ore_64_8_v1_term) RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1_term(a, b) <> 0 + CREATE OR REPLACE FUNCTION ore_64_8_v2_term_neq(a ore_64_8_v2_term, b ore_64_8_v2_term) RETURNS boolean AS $$ + SELECT compare_ore_64_8_v2_term(a, b) <> 0 $$ LANGUAGE SQL; - CREATE OR REPLACE FUNCTION ore_64_8_v1_term_lt(a ore_64_8_v1_term, b ore_64_8_v1_term) RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1_term(a, b) = -1 + CREATE OR REPLACE FUNCTION ore_64_8_v2_term_lt(a ore_64_8_v2_term, b ore_64_8_v2_term) RETURNS boolean AS $$ + SELECT compare_ore_64_8_v2_term(a, b) = -1 $$ LANGUAGE SQL; - CREATE OR REPLACE FUNCTION ore_64_8_v1_term_lte(a ore_64_8_v1_term, b ore_64_8_v1_term) RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1_term(a, b) != 1 + CREATE OR REPLACE FUNCTION ore_64_8_v2_term_lte(a ore_64_8_v2_term, b ore_64_8_v2_term) RETURNS boolean AS $$ + SELECT compare_ore_64_8_v2_term(a, b) != 1 $$ LANGUAGE SQL; - CREATE OR REPLACE FUNCTION ore_64_8_v1_term_gt(a ore_64_8_v1_term, b ore_64_8_v1_term) RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1_term(a, b) = 1 + CREATE OR REPLACE FUNCTION ore_64_8_v2_term_gt(a ore_64_8_v2_term, b ore_64_8_v2_term) RETURNS boolean AS $$ + SELECT compare_ore_64_8_v2_term(a, b) = 1 $$ LANGUAGE SQL; - CREATE OR REPLACE FUNCTION ore_64_8_v1_term_gte(a ore_64_8_v1_term, b ore_64_8_v1_term) RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1_term(a, b) != -1 + CREATE OR REPLACE FUNCTION ore_64_8_v2_term_gte(a ore_64_8_v2_term, b ore_64_8_v2_term) RETURNS boolean AS $$ + SELECT compare_ore_64_8_v2_term(a, b) != -1 $$ LANGUAGE SQL; CREATE OPERATOR = ( - PROCEDURE="ore_64_8_v1_term_eq", - LEFTARG=ore_64_8_v1_term, - RIGHTARG=ore_64_8_v1_term, + PROCEDURE="ore_64_8_v2_term_eq", + LEFTARG=ore_64_8_v2_term, + RIGHTARG=ore_64_8_v2_term, NEGATOR = <>, RESTRICT = eqsel, JOIN = eqjoinsel, @@ -121,9 +121,9 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; ); CREATE OPERATOR <> ( - PROCEDURE="ore_64_8_v1_term_neq", - LEFTARG=ore_64_8_v1_term, - RIGHTARG=ore_64_8_v1_term, + PROCEDURE="ore_64_8_v2_term_neq", + LEFTARG=ore_64_8_v2_term, + RIGHTARG=ore_64_8_v2_term, NEGATOR = =, RESTRICT = eqsel, JOIN = eqjoinsel, @@ -132,9 +132,9 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; ); CREATE OPERATOR > ( - PROCEDURE="ore_64_8_v1_term_gt", - LEFTARG=ore_64_8_v1_term, - RIGHTARG=ore_64_8_v1_term, + PROCEDURE="ore_64_8_v2_term_gt", + LEFTARG=ore_64_8_v2_term, + RIGHTARG=ore_64_8_v2_term, COMMUTATOR = <, NEGATOR = <=, RESTRICT = scalargtsel, @@ -142,9 +142,9 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; ); CREATE OPERATOR < ( - PROCEDURE="ore_64_8_v1_term_lt", - LEFTARG=ore_64_8_v1_term, - RIGHTARG=ore_64_8_v1_term, + PROCEDURE="ore_64_8_v2_term_lt", + LEFTARG=ore_64_8_v2_term, + RIGHTARG=ore_64_8_v2_term, COMMUTATOR = >, NEGATOR = >=, RESTRICT = scalarltsel, @@ -152,9 +152,9 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; ); CREATE OPERATOR <= ( - PROCEDURE="ore_64_8_v1_term_lte", - LEFTARG=ore_64_8_v1_term, - RIGHTARG=ore_64_8_v1_term, + PROCEDURE="ore_64_8_v2_term_lte", + LEFTARG=ore_64_8_v2_term, + RIGHTARG=ore_64_8_v2_term, COMMUTATOR = >=, NEGATOR = >, RESTRICT = scalarlesel, @@ -162,23 +162,23 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; ); CREATE OPERATOR >= ( - PROCEDURE="ore_64_8_v1_term_gte", - LEFTARG=ore_64_8_v1_term, - RIGHTARG=ore_64_8_v1_term, + PROCEDURE="ore_64_8_v2_term_gte", + LEFTARG=ore_64_8_v2_term, + RIGHTARG=ore_64_8_v2_term, COMMUTATOR = <=, NEGATOR = <, RESTRICT = scalarlesel, JOIN = scalarlejoinsel ); - CREATE OPERATOR FAMILY ore_64_8_v1_term_btree_ops USING btree; - CREATE OPERATOR CLASS ore_64_8_v1_term_btree_ops DEFAULT FOR TYPE ore_64_8_v1_term USING btree FAMILY ore_64_8_v1_term_btree_ops AS + CREATE OPERATOR FAMILY ore_64_8_v2_term_btree_ops USING btree; + CREATE OPERATOR CLASS ore_64_8_v2_term_btree_ops DEFAULT FOR TYPE ore_64_8_v2_term USING btree FAMILY ore_64_8_v2_term_btree_ops AS OPERATOR 1 <, OPERATOR 2 <=, OPERATOR 3 =, OPERATOR 4 >=, OPERATOR 5 >, - FUNCTION 1 compare_ore_64_8_v1_term(a ore_64_8_v1_term, b ore_64_8_v1_term); + FUNCTION 1 compare_ore_64_8_v2_term(a ore_64_8_v2_term, b ore_64_8_v2_term); -- Compare the "head" of each array and recurse if necessary -- This function assumes an empty string is "less than" everything else @@ -187,7 +187,7 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; -- doesn't always make sense but it's here for completeness. -- If both are non-empty, we compare the first element. If they are equal -- we need to consider the next block so we recurse, otherwise we return the comparison result. - CREATE OR REPLACE FUNCTION compare_ore_array(a ore_64_8_v1_term[], b ore_64_8_v1_term[]) returns integer AS $$ + CREATE OR REPLACE FUNCTION compare_ore_array(a ore_64_8_v2_term[], b ore_64_8_v2_term[]) returns integer AS $$ DECLARE cmp_result integer; BEGIN @@ -201,7 +201,7 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; RETURN 1; END IF; - cmp_result := compare_ore_64_8_v1_term(a[1], b[1]); + cmp_result := compare_ore_64_8_v2_term(a[1], b[1]); IF cmp_result = 0 THEN -- Removes the first element in the array, and calls this fn again to compare the next element/s in the array. RETURN compare_ore_array(a[2:array_length(a,1)], b[2:array_length(b,1)]); @@ -212,7 +212,7 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; $$ LANGUAGE plpgsql; -- This function uses lexicographic comparison - CREATE OR REPLACE FUNCTION compare_ore_64_8_v1(a ore_64_8_v1, b ore_64_8_v1) returns integer AS $$ + CREATE OR REPLACE FUNCTION compare_ore_64_8_v2(a ore_64_8_v2, b ore_64_8_v2) returns integer AS $$ DECLARE cmp_result integer; BEGIN @@ -221,34 +221,34 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; END $$ LANGUAGE plpgsql; - CREATE OR REPLACE FUNCTION ore_64_8_v1_eq(a ore_64_8_v1, b ore_64_8_v1) RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1(a, b) = 0 + CREATE OR REPLACE FUNCTION ore_64_8_v2_eq(a ore_64_8_v2, b ore_64_8_v2) RETURNS boolean AS $$ + SELECT compare_ore_64_8_v2(a, b) = 0 $$ LANGUAGE SQL; - CREATE OR REPLACE FUNCTION ore_64_8_v1_neq(a ore_64_8_v1, b ore_64_8_v1) RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1(a, b) <> 0 + CREATE OR REPLACE FUNCTION ore_64_8_v2_neq(a ore_64_8_v2, b ore_64_8_v2) RETURNS boolean AS $$ + SELECT compare_ore_64_8_v2(a, b) <> 0 $$ LANGUAGE SQL; - CREATE OR REPLACE FUNCTION ore_64_8_v1_lt(a ore_64_8_v1, b ore_64_8_v1) RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1(a, b) = -1 + CREATE OR REPLACE FUNCTION ore_64_8_v2_lt(a ore_64_8_v2, b ore_64_8_v2) RETURNS boolean AS $$ + SELECT compare_ore_64_8_v2(a, b) = -1 $$ LANGUAGE SQL; - CREATE OR REPLACE FUNCTION ore_64_8_v1_lte(a ore_64_8_v1, b ore_64_8_v1) RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1(a, b) != 1 + CREATE OR REPLACE FUNCTION ore_64_8_v2_lte(a ore_64_8_v2, b ore_64_8_v2) RETURNS boolean AS $$ + SELECT compare_ore_64_8_v2(a, b) != 1 $$ LANGUAGE SQL; - CREATE OR REPLACE FUNCTION ore_64_8_v1_gt(a ore_64_8_v1, b ore_64_8_v1) RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1(a, b) = 1 + CREATE OR REPLACE FUNCTION ore_64_8_v2_gt(a ore_64_8_v2, b ore_64_8_v2) RETURNS boolean AS $$ + SELECT compare_ore_64_8_v2(a, b) = 1 $$ LANGUAGE SQL; - CREATE OR REPLACE FUNCTION ore_64_8_v1_gte(a ore_64_8_v1, b ore_64_8_v1) RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1(a, b) != -1 + CREATE OR REPLACE FUNCTION ore_64_8_v2_gte(a ore_64_8_v2, b ore_64_8_v2) RETURNS boolean AS $$ + SELECT compare_ore_64_8_v2(a, b) != -1 $$ LANGUAGE SQL; CREATE OPERATOR = ( - PROCEDURE="ore_64_8_v1_eq", - LEFTARG=ore_64_8_v1, - RIGHTARG=ore_64_8_v1, + PROCEDURE="ore_64_8_v2_eq", + LEFTARG=ore_64_8_v2, + RIGHTARG=ore_64_8_v2, NEGATOR = <>, RESTRICT = eqsel, JOIN = eqjoinsel, @@ -257,9 +257,9 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; ); CREATE OPERATOR <> ( - PROCEDURE="ore_64_8_v1_neq", - LEFTARG=ore_64_8_v1, - RIGHTARG=ore_64_8_v1, + PROCEDURE="ore_64_8_v2_neq", + LEFTARG=ore_64_8_v2, + RIGHTARG=ore_64_8_v2, NEGATOR = =, RESTRICT = eqsel, JOIN = eqjoinsel, @@ -268,9 +268,9 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; ); CREATE OPERATOR > ( - PROCEDURE="ore_64_8_v1_gt", - LEFTARG=ore_64_8_v1, - RIGHTARG=ore_64_8_v1, + PROCEDURE="ore_64_8_v2_gt", + LEFTARG=ore_64_8_v2, + RIGHTARG=ore_64_8_v2, COMMUTATOR = <, NEGATOR = <=, RESTRICT = scalargtsel, @@ -278,9 +278,9 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; ); CREATE OPERATOR < ( - PROCEDURE="ore_64_8_v1_lt", - LEFTARG=ore_64_8_v1, - RIGHTARG=ore_64_8_v1, + PROCEDURE="ore_64_8_v2_lt", + LEFTARG=ore_64_8_v2, + RIGHTARG=ore_64_8_v2, COMMUTATOR = >, NEGATOR = >=, RESTRICT = scalarltsel, @@ -288,9 +288,9 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; ); CREATE OPERATOR <= ( - PROCEDURE="ore_64_8_v1_lte", - LEFTARG=ore_64_8_v1, - RIGHTARG=ore_64_8_v1, + PROCEDURE="ore_64_8_v2_lte", + LEFTARG=ore_64_8_v2, + RIGHTARG=ore_64_8_v2, COMMUTATOR = >=, NEGATOR = >, RESTRICT = scalarlesel, @@ -298,20 +298,20 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; ); CREATE OPERATOR >= ( - PROCEDURE="ore_64_8_v1_gte", - LEFTARG=ore_64_8_v1, - RIGHTARG=ore_64_8_v1, + PROCEDURE="ore_64_8_v2_gte", + LEFTARG=ore_64_8_v2, + RIGHTARG=ore_64_8_v2, COMMUTATOR = <=, NEGATOR = <, RESTRICT = scalarlesel, JOIN = scalarlejoinsel ); - CREATE OPERATOR FAMILY ore_64_8_v1_btree_ops USING btree; - CREATE OPERATOR CLASS ore_64_8_v1_btree_ops DEFAULT FOR TYPE ore_64_8_v1 USING btree FAMILY ore_64_8_v1_btree_ops AS + CREATE OPERATOR FAMILY ore_64_8_v2_btree_ops USING btree; + CREATE OPERATOR CLASS ore_64_8_v2_btree_ops DEFAULT FOR TYPE ore_64_8_v2 USING btree FAMILY ore_64_8_v2_btree_ops AS OPERATOR 1 <, OPERATOR 2 <=, OPERATOR 3 =, OPERATOR 4 >=, OPERATOR 5 >, - FUNCTION 1 compare_ore_64_8_v1(a ore_64_8_v1, b ore_64_8_v1); + FUNCTION 1 compare_ore_64_8_v2(a ore_64_8_v2, b ore_64_8_v2); diff --git a/examples/javascript/apps/drizzle/migrations/0000_broken_young_avengers.sql b/examples/javascript/apps/drizzle/migrations/0000_broken_young_avengers.sql index ec1ba7a9..f91da90f 100644 --- a/examples/javascript/apps/drizzle/migrations/0000_broken_young_avengers.sql +++ b/examples/javascript/apps/drizzle/migrations/0000_broken_young_avengers.sql @@ -1,6 +1,6 @@ CREATE TABLE IF NOT EXISTS "users" ( "id" serial PRIMARY KEY NOT NULL, "email" varchar, - "email_encrypted" "cs_encrypted_v1", + "email_encrypted" "cs_encrypted_v2", CONSTRAINT "users_email_unique" UNIQUE("email") ); diff --git a/examples/javascript/apps/drizzle/migrations/meta/0000_snapshot.json b/examples/javascript/apps/drizzle/migrations/meta/0000_snapshot.json index bc12ed7f..f083825c 100644 --- a/examples/javascript/apps/drizzle/migrations/meta/0000_snapshot.json +++ b/examples/javascript/apps/drizzle/migrations/meta/0000_snapshot.json @@ -22,7 +22,7 @@ }, "email_encrypted": { "name": "email_encrypted", - "type": "cs_encrypted_v1", + "type": "cs_encrypted_v2", "primaryKey": false, "notNull": false } diff --git a/examples/javascript/apps/drizzle/src/schema.ts b/examples/javascript/apps/drizzle/src/schema.ts index 452d8e49..1fe4e695 100644 --- a/examples/javascript/apps/drizzle/src/schema.ts +++ b/examples/javascript/apps/drizzle/src/schema.ts @@ -1,10 +1,10 @@ import type { CsPlaintextV1Schema } from '@cipherstash/jseql' import { customType, pgTable, serial, varchar } from 'drizzle-orm/pg-core' -const cs_encrypted_v1 = (name: string) => +const cs_encrypted_v2 = (name: string) => customType<{ data: TData; driverData: string }>({ dataType() { - return 'cs_encrypted_v1' + return 'cs_encrypted_v2' }, toDriver(value: TData): string { return JSON.stringify(value) @@ -14,5 +14,5 @@ const cs_encrypted_v1 = (name: string) => export const users = pgTable('users', { id: serial('id').primaryKey(), email: varchar('email').unique(), - email_encrypted: cs_encrypted_v1('email_encrypted'), + email_encrypted: cs_encrypted_v2('email_encrypted'), }) diff --git a/examples/javascript/apps/drizzle/src/select.ts b/examples/javascript/apps/drizzle/src/select.ts index a212f791..eea97668 100644 --- a/examples/javascript/apps/drizzle/src/select.ts +++ b/examples/javascript/apps/drizzle/src/select.ts @@ -1,5 +1,5 @@ import { getEmailArg } from '@cipherstash-jseql-examples/utils' -import { cs_match_v1 } from '@cipherstash/jseql/drizzle' +import { cs_match_v2 } from '@cipherstash/jseql/drizzle' import { getPlaintext } from '@cipherstash/jseql' import { db } from './db' import { users } from './schema' @@ -15,7 +15,7 @@ const sql = db .from(users) if (email) { - sql.where(cs_match_v1(users, users.email_encrypted, email)) + sql.where(cs_match_v2(users, users.email_encrypted, email)) } const sqlResult = sql.toSQL() diff --git a/examples/javascript/apps/prisma/src/db.ts b/examples/javascript/apps/prisma/src/db.ts index 371275e9..69f6e54e 100644 --- a/examples/javascript/apps/prisma/src/db.ts +++ b/examples/javascript/apps/prisma/src/db.ts @@ -29,7 +29,7 @@ export const prisma = new PrismaClient().$extends({ // TODO: Fix Prisma.raw to prevent SQL injection return prisma.$queryRaw< T[] - >`SELECT * FROM "${Prisma.raw(schema)}"."${Prisma.raw(tableName)}" WHERE cs_match_v1(${Prisma.raw(column)}) @> cs_match_v1('${Prisma.raw(payload)}')` + >`SELECT * FROM "${Prisma.raw(schema)}"."${Prisma.raw(tableName)}" WHERE cs_match_v2(${Prisma.raw(column)}) @> cs_match_v2('${Prisma.raw(payload)}')` }, }, }, diff --git a/examples/python/jupyter_notebook/CipherStash-Getting-Started.ipynb b/examples/python/jupyter_notebook/CipherStash-Getting-Started.ipynb index d11fa556..3b1285b4 100644 --- a/examples/python/jupyter_notebook/CipherStash-Getting-Started.ipynb +++ b/examples/python/jupyter_notebook/CipherStash-Getting-Started.ipynb @@ -1,1044 +1,1021 @@ { - "cells": [ - { - "cell_type": "markdown", - "id": "6f628465-5dfb-4f76-9a65-4a761cf2b940", - "metadata": {}, - "source": [ - "# Getting Started with CipherStash and Jupyter Notebook\n", - "\n", - "This notebook describes how to get started with CipherStash using Python3, Jupyter Notebook, psycopg2 and SQLAlchemy.and\n", - "\n", - "## Prerequisites\n", - "\n", - "You must have:\n", - "* [PostgreSQL **client**](https://www.postgresql.org/) (The server will be run using Docker in this notebook)\n", - "* [Python 3](https://www.python.org/)\n", - "* [Jupyter Notebook](https://jupyter.org/install)\n", - "* [Docker](https://docs.docker.com/get-started/get-docker/)\n", - "* [Docker compose](https://docs.docker.com/compose/install/)\n", - "* [curl](https://curl.se)\n", - "* [CipherStash account](https://cipherstash.com/signup)\n", - "* [CipherStash CLI](https://github.com/cipherstash/cli-releases/releases/latest)\n", - "\n", - "> Please note that on MS Windows' PowerShell, there is a built-in alias `curl` which is different from [curl listed above](https://curl.se).\n", - "A simple way around this is to start Jupyter Notebook from cmd.exe instead of PowerShell." - ] - }, - { - "cell_type": "markdown", - "id": "77e69c79-c3f6-4c5c-a746-449034b71ac3", - "metadata": {}, - "source": [ - "## Start CipherStash Proxy and PostgreSQL\n", - "\n", - "In order to run the example, you will need to start CipherStash Proxy and PostgreSQL.\n", - "\n", - "Please set up the [playground environment](../../playground/README.md) to run the the following Python examples." - ] - }, - { - "cell_type": "markdown", - "id": "d9df0769-503c-4921-bb83-f4ff2a5f8a6d", - "metadata": {}, - "source": [ - "## Installing required components and table creation\n", - "\n", - "Once the containers are up, there are a few things to be installed.\n", - "A table must also be created to store encrypted data.\n", - "Do the following steps to install them and create a table." - ] - }, - { - "cell_type": "markdown", - "id": "9b4dbe41-2fd5-4335-be55-19058190cd30", - "metadata": {}, - "source": [ - "### Install application specific database types" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "505766c0-8c17-432f-943f-875615c88e09", - "metadata": {}, - "outputs": [], - "source": [ - "%env PGPASSWORD=postgres \n", - "! psql -h localhost -p 5432 -U postgres postgres < application_types.sql" - ] - }, - { - "cell_type": "markdown", - "id": "8d8648b8-8b88-4794-9528-e781639a2f0c", - "metadata": {}, - "source": [ - "### Create a table and indexes for testing encryption" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "56e04d28-4ebc-463a-b7d0-1ea3cf843f1b", - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "%env PGPASSWORD=postgres\n", - "! psql -h localhost -p 5432 -U postgres postgres < create_examples_table.sql" - ] - }, - { - "cell_type": "markdown", - "id": "0400e206-a337-404c-acc9-9706e626e94e", - "metadata": {}, - "source": [ - "## Run Python code\n", - "\n", - "With the services running (this can be checked with `docker compose ps` from the shell), it's time to run some Python code.\n", - "\n", - "Before actual code examples, below is a short introduction of what needs to happen between the native Python data types and encrypted database types." - ] - }, - { - "cell_type": "markdown", - "id": "ece47f71-c115-4f4e-ac8e-f88440fba2dc", - "metadata": {}, - "source": [ - "### Classes that convert between the database format and Python format\n", - "\n", - "There are classes prefixed with `Eql` defined in `eqlpy` which handles conversion between the format CypherStash Proxy requires and the format for Python.\n", - "\n", - "In order to encrypt and store plaintext values, CipherStash Proxy requires encrypted columns in its specific format.\n", - "In Python, this conversion is done by creating an object of `EqlText` as:\n", - "```\n", - "txt = EqlText(\"hello, world\", \"pyexamples\", \"encrypted_utf8_str\")\n", - "txt.to_db_format()\n", - "```\n", - "\n", - "The constructor for `EqlText` takes the string value, the table name (`\"pyexamples\"`) and the column name (`\"encrypted_utf8_str\"`)." - ] - }, - { - "cell_type": "markdown", - "id": "637d0665-e53b-40e5-b685-aa1aa4354e46", - "metadata": {}, - "source": [ - "### Install psycopg2 and sqlalchemy\n", - "\n", - "Install `psycopg2` and `sqlalchemy` if you have not done so yet:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bb1cc638-bd6b-4776-98a7-0e25a778be96", - "metadata": {}, - "outputs": [], - "source": [ - "%pip install psycopg2 sqlalchemy eqlpy" - ] - }, - { - "cell_type": "markdown", - "id": "1bbca0fa-4942-4e40-bffa-a3eb2027c671", - "metadata": {}, - "source": [ - "### Import class definitions\n", - "\n", - "There are some classes defined for encrytped types in this project directory.\n", - "They are in `eql_types.py` in the `eqlpy` package if you are interested in implementation details." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ebf51d89-41a7-4988-ab8b-b3cee1c35a31", - "metadata": {}, - "outputs": [], - "source": [ - "from psycopg2.extras import RealDictCursor\n", - "import psycopg2\n", - "from eqlpy.eql_types import *\n", - "\n", - "print(\"Importing done.\")" - ] - }, - { - "cell_type": "markdown", - "id": "777d2c94-4002-4e9e-bbc3-04b598f8aee4", - "metadata": {}, - "source": [ - "## Define column-function mapping\n", - "\n", - "In order to build plain-text `EqlRow`s from encrypted records, we have to provide `EqlRow` with information on which functions should be used to convert them." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f9dcfc0a-bb52-448a-961d-ee8451a23186", - "metadata": {}, - "outputs": [], - "source": [ - "mapping = {\n", - " 'encrypted_int': EqlInt.from_parsed_json,\n", - " 'encrypted_boolean': EqlBool.from_parsed_json,\n", - " 'encrypted_date': EqlDate.from_parsed_json,\n", - " 'encrypted_float': EqlFloat.from_parsed_json,\n", - " 'encrypted_utf8_str': EqlText.from_parsed_json,\n", - " 'encrypted_jsonb': EqlJsonb.from_parsed_json\n", - "}\n", - "\n", - "print(\"column-function mapping defined\")" - ] - }, - { - "cell_type": "markdown", - "id": "6ac472ff-81d1-418a-998b-975b2b3f3a05", - "metadata": {}, - "source": [ - "## Insert test record\n", - "\n", - "With the database extensions, EQL, and application specific data types installed together with the type definitions for Python, your setup is now ready to encrypt and decrypt data.\n", - "\n", - "Run the following to create a record in the `pyexamples` table:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "698c6970-62f0-4cfb-b779-7a5dbbe11d3a", - "metadata": {}, - "outputs": [], - "source": [ - "from pprint import pprint\n", - "from datetime import datetime\n", - "\n", - "conn = psycopg2.connect(\"host=localhost dbname=postgres user=postgres password=postgres port=6432\")\n", - "\n", - "cur = conn.cursor(cursor_factory=RealDictCursor)\n", - "\n", - "cur.execute(\"delete from pyexamples\") # Clear the table in case there are records from previous runs\n", - "cur.execute(\"select cs_refresh_encrypt_config()\")\n", - "\n", - "cur.execute(\"INSERT INTO pyexamples (encrypted_int, encrypted_boolean, encrypted_date, encrypted_float, encrypted_utf8_str) VALUES (%s, %s, %s, %s, %s)\",\n", - " (\n", - " EqlInt(-51, \"pyexamples\", \"encrypted_int\").to_db_format(),\n", - " EqlBool(False, \"pyexamples\", \"encrypted_boolean\").to_db_format(),\n", - " EqlDate(datetime.now().date(), \"pyexamples\", \"encrypted_date\").to_db_format(),\n", - " EqlFloat(-0.5, \"pyexamples\", \"encrypted_float\").to_db_format(),\n", - " EqlText(\"hello, world\", \"pyexamples\", \"encrypted_utf8_str\").to_db_format()\n", - " )\n", - ")\n", - "\n", - "conn.commit()\n", - "\n", - "print(\"example row created in pyexamples table\")" - ] - }, - { - "cell_type": "markdown", - "id": "2f24b9c8-8537-4f44-93f5-925a37bf65b1", - "metadata": {}, - "source": [ - "This should insert a single row in the encrypted `pyexamples` table as:\n", - "\n", - "|encrypted_int|encrypted_boolean|encrypted_date|encrypted_float|encrypted_utf8_str|\n", - "|---|-----|--------------|----|------------|\n", - "|-51|false|2024-11-01 |-0.5|hello, world|\n", - "\n", - "You can check what it looks like from both regular PostgreSQL running on port 5432 and CipherStash Proxy running on port 6432.\n", - "To look at the data through CipherStash Proxy, run the following:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9ab7973d-1d63-43cd-81c2-dd69f544deda", - "metadata": {}, - "outputs": [], - "source": [ - "# From CipherStash Proxy; you should see plaintext JSONB\n", - "%env PGPASSWORD=postgres\n", - "! psql -h localhost -p 6432 -U postgres -x -c \"select * from pyexamples limit 1;\" postgres" - ] - }, - { - "cell_type": "markdown", - "id": "802cbfb3-9869-43ce-adae-1367b23ea52e", - "metadata": {}, - "source": [ - "To look at the data directly on the PostgreSQL server, run the following:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "85597b86-c53f-4df9-87a6-d3eec1730b77", - "metadata": {}, - "outputs": [], - "source": [ - "# From PostgreSQL; you should see JSONB with encrypted values\n", - "%env PGPASSWORD=postgres\n", - "! psql -h localhost -p 5432 -U postgres -x -c \"select * from pyexamples limit 1;\" postgres" - ] - }, - { - "cell_type": "markdown", - "id": "564d2bc7-0c45-4e5f-ba0a-5919961ee9ce", - "metadata": {}, - "source": [ - "In the above example, not all fields are populated, but the populated fields contain JSONB values including the encrypted values." - ] - }, - { - "cell_type": "markdown", - "id": "e824d962-97f2-42b1-a9a2-b283576690d6", - "metadata": {}, - "source": [ - "### Converting to Python types\n", - "\n", - "By querying the proxy, you will see the JSONB values as seen above (decrypted version in the Proxy example, not the PostgreSQL example).\n", - "\n", - "The values should then be converted to types that can be used in Python using class methods for each type:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b7acd767-74ae-4717-ab2b-b71156eb9a9f", - "metadata": {}, - "outputs": [], - "source": [ - "from IPython.display import display, Markdown\n", - "\n", - "cur.execute(\"select * from pyexamples\")\n", - "\n", - "records = cur.fetchall()\n", - "\n", - "record0 = records[0]\n", - "\n", - "# `from_parsed_json` methods convert the values into the corresponding Python types\n", - "content = f\"\"\"\n", - "### Values in the record\n", - "| | |\n", - "|--|--|\n", - "|int | {EqlInt.from_parsed_json(record0['encrypted_int'])}|\n", - "|boolean | {EqlBool.from_parsed_json(record0['encrypted_boolean'])}|\n", - "|datetime | {EqlDate.from_parsed_json(record0['encrypted_date'])}|\n", - "|float | {EqlFloat.from_parsed_json(record0['encrypted_float'])}|\n", - "|text | {EqlText.from_parsed_json(record0['encrypted_utf8_str'])}|\n", - "\"\"\"\n", - "\n", - "display(Markdown(content))\n" - ] - }, - { - "cell_type": "markdown", - "id": "94e823d5-7ee0-4e32-b820-b8fd9e5adfea", - "metadata": {}, - "source": [ - "### Querying with the encrypted fields\n", - "\n", - "You can also use the encrypted fields for queries.\n", - "\n", - "First, add some values so there are more than 1 text values and float values stored:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "36586ea6-09aa-40b0-be4a-17b3c18cfd4e", - "metadata": {}, - "outputs": [], - "source": [ - "# data for MATCH\n", - "cur.execute(\"INSERT INTO pyexamples (encrypted_utf8_str) VALUES (%s) ON CONFLICT DO NOTHING\",\n", - " (\n", - " EqlText(\"hello, python\", \"pyexamples\", \"encrypted_utf8_str\").to_db_format(),\n", - " )\n", - ")\n", - "\n", - "cur.execute(\"INSERT INTO pyexamples (encrypted_utf8_str) VALUES (%s) ON CONFLICT DO NOTHING\",\n", - " (\n", - " EqlText(\"hello, jupyter\", \"pyexamples\", \"encrypted_utf8_str\").to_db_format(),\n", - " )\n", - ")\n", - "\n", - "# data for ORE\n", - "cur.execute(\"INSERT INTO pyexamples (encrypted_float) VALUES (%s)\",\n", - " (\n", - " EqlFloat(100.1, \"pyexamples\", \"encrypted_float\").to_db_format(),\n", - " )\n", - ")\n", - "\n", - "cur.execute(\"INSERT INTO pyexamples (encrypted_float) VALUES (%s)\",\n", - " (\n", - " EqlFloat(100.2, \"pyexamples\", \"encrypted_float\").to_db_format(),\n", - " )\n", - ")\n", - "\n", - "conn.commit()\n", - "\n", - "print(\"created data for MATCH and ORE queries\")" - ] - }, - { - "cell_type": "markdown", - "id": "4e67be50-948d-4832-80e9-6411538cf974", - "metadata": {}, - "source": [ - "The example code above should insert rows like these in the pyexamples table:\n", - "\n", - "| | encrypted_utf_data | encrypted_float||\n", - "|--|---|---|---|\n", - "| |hello, python| | |\n", - "| |hello, jupyter| | |\n", - "| | | 100.1 | |\n", - "| | | 100.2 | |\n" - ] - }, - { - "cell_type": "markdown", - "id": "c2a6b812-980c-4958-a55e-45551cb474c2", - "metadata": {}, - "source": [ - "### Partial matching\n", - "\n", - "Now, a query can be run to look for a record in the `pyexamples` table where `encrypted_utf_8_str` field contains text `\"pyth\"`, which should match `\"hello, python\"`:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f9455deb-ea1f-419c-ac72-90f2eb7e76c0", - "metadata": {}, - "outputs": [], - "source": [ - "# MATCH query for \"pyth\"\n", - "cur.execute(\"SELECT * FROM pyexamples WHERE cs_match_v1(encrypted_utf8_str) @> cs_match_v1(%s)\", (EqlText(\"pyth\", \"pyexamples\", \"encrypted_utf8_str\").to_db_format(),))\n", - "\n", - "found = cur.fetchall()[0]\n", - "print(f\"Record Found with MATCH query: {EqlRow(mapping, found).row}\\n\")\n", - "print(f\"Text inside the found record: {EqlText.from_parsed_json(found['encrypted_utf8_str'])}\")" - ] - }, - { - "cell_type": "markdown", - "id": "410be894-23ff-4aed-b8c4-bc283b53a36d", - "metadata": {}, - "source": [ - "### Exact matching\n", - "\n", - "Similarly, a query for the exact text of `\"hello, jupyter\"` in the `encrypted_utf_8_str` field:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "834ce829-9d12-4d14-bdac-20abb2e76148", - "metadata": {}, - "outputs": [], - "source": [ - "# UNIQUE\n", - "cur.execute(\"SELECT * FROM pyexamples WHERE cs_unique_v1(encrypted_utf8_str) = cs_unique_v1(%s)\", (EqlText(\"hello, jupyter\", \"pyexamples\", \"encrypted_utf8_str\").to_db_format(),))\n", - "found = cur.fetchall()[0]\n", - "print(f\"Record Found with UNIQUE query: {EqlRow(mapping, found).row}\\n\")\n", - "print(f\"Text inside the found record: {EqlText.from_parsed_json(found['encrypted_utf8_str'])}\")" - ] - }, - { - "cell_type": "markdown", - "id": "40037eb7-49f9-4a55-8676-4a155056a650", - "metadata": {}, - "source": [ - "### ORE queries\n", - "\n", - "With ORE, you can run a query for a record with `encrypted_float` that is larger than `100.15` which should match `100.2`:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "235a8f52-e816-4bbb-8db9-be9330b4b238", - "metadata": {}, - "outputs": [], - "source": [ - "# ORE\n", - "cur.execute(\"SELECT * FROM pyexamples WHERE cs_ore_64_8_v1(encrypted_float) > cs_ore_64_8_v1(%s)\", (EqlFloat(100.15, \"pyexamples\", \"encrypted_float\").to_db_format(),))\n", - "found = cur.fetchall()[0]\n", - "print(f\"Record Found with ORE query: {EqlRow(mapping, found).row}\\n\")\n", - "print(f\"Float inside the found record: {EqlFloat.from_parsed_json(found['encrypted_float'])}\")\n" - ] - }, - { - "cell_type": "markdown", - "id": "e9b9ee3e-6919-4a44-b98c-6964b06c3c87", - "metadata": {}, - "source": [ - "### Updating records\n", - "\n", - "Encrypted fields can be updated too.\n", - "The interface is similar to creating and querying:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9daad342-3bd7-47a0-9744-1aba1c955506", - "metadata": {}, - "outputs": [], - "source": [ - "cur.execute(\"SELECT * FROM pyexamples WHERE cs_unique_v1(encrypted_utf8_str) = cs_unique_v1(%s)\", (EqlText(\"hello, jupyter\", \"pyexamples\", \"encrypted_utf8_str\").to_db_format(),))\n", - "found = cur.fetchall()[0]\n", - "record_id = found['id']\n", - "\n", - "cur.execute(\n", - " \"UPDATE pyexamples SET encrypted_utf8_str = %s WHERE id = %s\",\n", - " (EqlText(\"UPDATED TEXT\", \"pyexamples\", \"encrypted_utf8_str\").to_db_format(), record_id) # Replace 'column_name' and 'new_value' with actual column and value\n", - ")\n", - "cur.execute(\"SELECT * FROM pyexamples WHERE id = %s\", (record_id,))\n", - "found = cur.fetchall()[0]\n", - "print(f\"Updated row: {EqlRow(mapping, found).row}\")" - ] - }, - { - "cell_type": "markdown", - "id": "c8c82996-b91d-4536-a141-1316d293c044", - "metadata": {}, - "source": [ - "### JSONB queries and operations\n", - "\n", - "There are multiple types of JSONB queries and operations supported.\n", - "Here, we introduce:\n", - "\n", - "* Containment query\n", - "* Field extraction\n", - "* WHERE with field exctraction\n", - "* ORDER BY with field extraction\n", - "* GROUP BY with field extraction\n", - "\n", - "First, create the data for JSONB queries. The following queries will create records with JSONB values as:\n", - "\n", - "| encrypted_jsonb |\n", - "|---------|\n", - "| {\"num\": 1, \"category\": \"a\", \"top\": {\"nested\": [\"a\", \"b\", \"c\"]} |\n", - "| {\"num\": 2, \"category\": \"b\", \"top\": {\"nested\": [\"a\"]}} |\n", - "| {\"num\": 3, \"category\": \"b\", \"top\": {\"nested\": [\"z\"]}} |" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ebdf4fe5-1265-484e-8549-cbfdeab5296f", - "metadata": {}, - "outputs": [], - "source": [ - "cur.execute(\"DELETE FROM pyexamples;\")\n", - "\n", - "# data for JSONB queries\n", - "cur.execute(\"INSERT INTO pyexamples (encrypted_jsonb) VALUES (%s)\",\n", - " (\n", - " EqlJsonb({\"num\": 1, \"category\": \"a\", \"top\": {\"nested\": [\"a\", \"b\", \"c\"]}}, \"pyexamples\", \"encrypted_jsonb\").to_db_format(),\n", - " )\n", - ")\n", - "\n", - "cur.execute(\"INSERT INTO pyexamples (encrypted_jsonb) VALUES (%s)\",\n", - " (\n", - " EqlJsonb({\"num\": 2, \"category\": \"b\", \"top\": {\"nested\": [\"a\"]}}, \"pyexamples\", \"encrypted_jsonb\").to_db_format(),\n", - " )\n", - ")\n", - "\n", - "cur.execute(\"INSERT INTO pyexamples (encrypted_jsonb) VALUES (%s)\",\n", - " (\n", - " EqlJsonb({\"num\": 3, \"category\": \"b\", \"top\": {\"nested\": [\"z\"]}}, \"pyexamples\", \"encrypted_jsonb\").to_db_format(),\n", - " )\n", - ")# JSONB containment 1\n", - "\n", - "\n", - "conn.commit()\n", - "\n", - "print(\"Data for JSONB queries created\")" - ] - }, - { - "cell_type": "markdown", - "id": "c1f857e8-8fcb-46f0-b178-e17b580e7538", - "metadata": {}, - "source": [ - "### JSONB containment queries\n", - "\n", - "A record can be found using the JSONB containment.\n", - "The following matches the JSONB field containing keys `top` and `nested` with the `@>` operator:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b05bd5af-4fd7-4d74-8b41-a1c88078cbd2", - "metadata": {}, - "outputs": [], - "source": [ - "# JSONB containment 1\n", - "cur.execute(\"SELECT * from pyexamples WHERE cs_ste_vec_v1(encrypted_jsonb) @> cs_ste_vec_v1(%s)\", (EqlJsonb({'top': { 'nested': ['a', 'b'] } }, \"pyexamples\", \"encrypted_jsonb\").to_db_format(\"ste_vec\"),))\n", - "found = cur.fetchall()\n", - "for f in found:\n", - " print(f\"Record Found with JSONB query: {EqlRow(mapping, f).row}\\n\")\n", - " print(f\"JSONB inside the found record: {EqlJsonb.from_parsed_json(f['encrypted_jsonb'])}\\n\")" - ] - }, - { - "cell_type": "markdown", - "id": "97b4897b-713a-473b-9d97-25f80f263a76", - "metadata": {}, - "source": [ - "Also, the reverse operator (`<@`) is available to the above.\n", - "This query matches the JSONB field that's contained in the query:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8e68a18f-f482-473b-b05a-dd0f4291e031", - "metadata": {}, - "outputs": [], - "source": [ - "# JSONB containment 2\n", - "cur.execute(\"SELECT * from pyexamples WHERE cs_ste_vec_v1(encrypted_jsonb) <@ cs_ste_vec_v1(%s)\", (EqlJsonb({\"num\": 2, \"category\": \"b\", 'top': {'nested': ['a'] } }, \"pyexamples\", \"encrypted_jsonb\").to_db_format(\"ste_vec\"),))\n", - "found = cur.fetchall()\n", - "print(\"Record Found with JSONB query:\")\n", - "for f in found:\n", - " print(f\" {EqlRow(mapping, f).row}\")" - ] - }, - { - "cell_type": "markdown", - "id": "f8ff4434-ddad-40a1-9a28-b5be04b2d4b3", - "metadata": {}, - "source": [ - "### JSONB field extraction\n", - "\n", - "Specific JSONB fields can be accessed in queries.\n", - "\n", - "The following finds all values in `$.top.nested`:\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "909f4645-8c6a-42d2-92ea-f24181232155", - "metadata": {}, - "outputs": [], - "source": [ - "query = \"SELECT cs_ste_vec_value_v1(encrypted_jsonb, %s) AS val FROM pyexamples\"\n", - "\n", - "results = cur.execute(query, (EqlJsonb(\"$.top.nested\", \"pyexamples\", \"encrypted_jsonb\").to_db_format(\"ejson_path\"),))\n", - "\n", - "found = cur.fetchall()\n", - "print(\"values from '$.top.nested':\")\n", - "for f in found:\n", - " print(f\" {EqlRow(mapping, f).row.get('val', {}).get('p')}\")" - ] - }, - { - "cell_type": "markdown", - "id": "ff4a571a-c00e-4e95-9554-c82a5daf9352", - "metadata": {}, - "source": [ - "### JSONB field in WHERE query" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2fae30cc-c3a4-4c03-8779-4efdc0c4483c", - "metadata": {}, - "outputs": [], - "source": [ - "query = \"SELECT * FROM pyexamples WHERE cs_ste_vec_term_v1(encrypted_jsonb, %s) > cs_ste_vec_term_v1(%s)\"\n", - "\n", - "cur.execute(\n", - " query,\n", - " (EqlJsonb(\"$.num\", \"pyexamples\", \"encrypted_jsonb\").to_db_format(\"ejson_path\"),\n", - " EqlJsonb(1, \"pyexamples\", \"encrypted_jsonb\").to_db_format(\"ste_vec\")\n", - " )\n", - ")\n", - "\n", - "found = cur.fetchall()\n", - "\n", - "for f in found:\n", - " print(f\"Record Found with JSONB query: {EqlRow(mapping, f).row}\\n\")" - ] - }, - { - "cell_type": "markdown", - "id": "4690460d-bbe8-4aaa-ae27-d013cb73b690", - "metadata": {}, - "source": [ - "### JSONB field in ORDER BY\n", - "\n", - "Field extraction can also be used to order the results.\n", - "The following will demonstrate ordering by the 'num' field in descending order:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "04afc9d6-4ff3-484d-a500-205236a2c06b", - "metadata": {}, - "outputs": [], - "source": [ - "query = \"SELECT * FROM pyexamples ORDER BY cs_ste_vec_term_v1(encrypted_jsonb, %s) DESC\"\n", - "\n", - "cur.execute(\n", - " query,\n", - " (EqlJsonb(\"$.num\", \"pyexamples\", \"encrypted_jsonb\").to_db_format(\"ejson_path\"),)\n", - ")\n", - "found = cur.fetchall();\n", - "print(\"JSONB contents, ordered by 'num' desc:\")\n", - "for f in found:\n", - " print(f\" {EqlRow(mapping, f).row.get('encrypted_jsonb')}\")" - ] - }, - { - "cell_type": "markdown", - "id": "6c18cefe-f8a1-49cc-b10c-0a455e4908b2", - "metadata": {}, - "source": [ - "### JSONB field in GROUP BY\n", - "\n", - "Field extraction can also be used in GROUP BY.\n", - "This example demonstrates how to group the count results by the 'category' column:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6f3b1cd0-a82b-487b-bf32-6eeaf11c6cb3", - "metadata": {}, - "outputs": [], - "source": [ - "query = \"SELECT cs_grouped_value_v1(cs_ste_vec_value_v1(encrypted_jsonb, %s)) AS category, COUNT(*) FROM pyexamples GROUP BY cs_ste_vec_term_v1(encrypted_jsonb, %s)\"\n", - "\n", - "cur.execute(\n", - " query,\n", - " (EqlJsonb(\"$.category\", \"pyexamples\", \"encrypted_jsonb\").to_db_format(\"ejson_path\"),\n", - " EqlJsonb(\"$.category\", \"pyexamples\", \"encrypted_jsonb\").to_db_format(\"ejson_path\"))\n", - ")\n", - "\n", - "found = cur.fetchall()\n", - "\n", - "print(\"count, grouped by category:\")\n", - "for f in found:\n", - " row = EqlRow(mapping, f).row\n", - " print(f\" category: {row.get('category', {}).get('p')}, count: {row.get('count')}\")\n" - ] - }, - { - "cell_type": "markdown", - "id": "f2279c68-06ab-47cb-bd86-dfaf63a7ebb4", - "metadata": {}, - "source": [ - "### Clean up\n", - "\n", - "Clean up the data before going to the next section:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f9b74fa1-430e-49d2-868c-5acff124ceb4", - "metadata": {}, - "outputs": [], - "source": [ - "cur.execute(\"DELETE FROM pyexamples\")\n", - "conn.commit()" - ] - }, - { - "cell_type": "markdown", - "id": "bf498c2e-00fa-4746-85b3-c4008b55003d", - "metadata": {}, - "source": [ - "## Using SQLAlchemy\n", - "\n", - "### Model definitions and example data\n", - "\n", - "To use SQLAlchemy with CipherStash Encrypt, it is necessary to have model classes that can handle the format conversion.\n", - "\n", - "Import the model definition and create some records as below:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "24fda18e-f382-428a-9b49-d859eca04ace", - "metadata": {}, - "outputs": [], - "source": [ - "from sqlalchemy import create_engine, select, text\n", - "from sqlalchemy.orm import sessionmaker\n", - "from eqlpy.eqlalchemy import *\n", - "from datetime import date\n", - "from example_model import Example\n", - "\n", - "# Creating engine. Optionally add echo=True to see the SQL statetments dumped to stdout\n", - "engine = create_engine('postgresql://postgres:postgres@localhost:6432/postgres')\n", - "Session = sessionmaker(bind=engine)\n", - "session = Session()\n", - "\n", - "BaseModel.metadata.create_all(engine) # Create table for models if it's not created yetbelow and\n", - "\n", - "# Clear data if there is any from previous runs\n", - "session.query(Example).delete()\n", - "\n", - "ex = Example(e_utf8_str = \"example record 1\", e_jsonb = {'a': {'b': 1}}, e_int = 42, e_float = 3.14, e_date = date(2024, 10, 25), e_bool=False)\n", - "session.add(ex)\n", - "session.commit()\n", - "\n", - "ex = Example(e_utf8_str = \"example record 2\", e_jsonb = {'a': {'c': 2}}, e_int = 43, e_float = 1.41, e_date = date(2024, 10, 26), e_bool=True)\n", - "session.add(ex)\n", - "session.commit()\n", - "\n", - "ex = Example(e_utf8_str = \"example record 3\", e_jsonb = {'a': {'b': 1}}, e_int = 44, e_float = 2.718, e_date = date(2024, 10, 27), e_bool=True)\n", - "session.add(ex)\n", - "session.commit()\n", - "\n", - "print(\"Example data creation done\")" - ] - }, - { - "cell_type": "markdown", - "id": "97e4fea0-74e5-4f16-9040-1eecbefe5c14", - "metadata": {}, - "source": [ - "### Querying by exact match\n", - "\n", - "With the example data, you can query the data much like the psycopg2 examples above:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6182c491-cfbb-4348-b245-39511f55d1eb", - "metadata": {}, - "outputs": [], - "source": [ - "# UNIQUE\n", - "query_text = text('cs_unique_v1(encrypted_utf8_str) == cs_unique_v1(:term)')\n", - "query = select(Example).where(query_text).params(term=EqlText(\"example record 1\", \"pyexamples\", \"encrypted_utf8_str\").to_db_format())\n", - "results = session.execute(query).scalars().all()\n", - "\n", - "for e in results:\n", - " print(f\"UNIQUE query results: {e}\")\n" - ] - }, - { - "cell_type": "markdown", - "id": "3ca80e10-d31c-48fe-8908-0aa98affbe3b", - "metadata": {}, - "source": [ - "Model attributes are available as plain text: " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "23096c06-3136-4c1c-9c47-a6399dc754c8", - "metadata": {}, - "outputs": [], - "source": [ - "results[0].encrypted_utf8_str" - ] - }, - { - "cell_type": "markdown", - "id": "1c493bdf-25fa-49ab-a5ab-87e0fbf595e5", - "metadata": {}, - "source": [ - "### Querying by partial match\n", - "\n", - "Partial matching can also performed with SQLAlchemy:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2cfd86e9-70d2-420b-b56f-11744d5ae4bf", - "metadata": {}, - "outputs": [], - "source": [ - "# MATCH\n", - "query_text = text('cs_match_v1(encrypted_utf8_str) @> cs_match_v1(:term)')\n", - "query = select(Example).where(query_text).params(term=EqlText(\"example record\", \"pyexamples\", \"encrypted_utf8_str\").to_db_format())\n", - "results = session.execute(query).scalars().all()\n", - "\n", - "for e in results:\n", - " print(f\"MATCH query results: {e}\")" - ] - }, - { - "cell_type": "markdown", - "id": "c35d9fd3-bfbd-4084-a00d-fee22f7e30d0", - "metadata": {}, - "source": [ - "### Query by ORE\n", - "\n", - "ORE queries can be peformed too:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5985efb1-1944-41d6-a912-d991095bd7d8", - "metadata": {}, - "outputs": [], - "source": [ - "# ORE\n", - "cur.execute(\"SELECT * FROM pyexamples WHERE cs_ore_64_8_v1(encrypted_float) > cs_ore_64_8_v1(%s)\", (EqlFloat(100.15, \"pyexamples\", \"encrypted_float\").to_db_format(),))\n", - "\n", - "query_text = text('cs_ore_64_8_v1(encrypted_float) > cs_ore_64_8_v1(:term)')\n", - "query = select(Example).where(query_text).params(term=EqlFloat(2.0, \"pyexamples\", \"encrypted_float\").to_db_format())\n", - "results = session.execute(query).scalars().all()\n", - "\n", - "for e in results:\n", - " print(f\"ORE query results: {e}\")" - ] - }, - { - "cell_type": "markdown", - "id": "e93cf011-66c7-4d93-a3c8-cb6ebe4b34ff", - "metadata": {}, - "source": [ - "### Querying by JSONB containment\n", - "\n", - "These records can be queried by JSONB containment too:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5f442da8-55f4-4a62-ab57-26b844b97ea7", - "metadata": {}, - "outputs": [], - "source": [ - "# JSONB\n", - "\n", - "query_text = text(\n", - " \"cs_ste_vec_v1(encrypted_jsonb) @> cs_ste_vec_v1(:term)\"\n", - ")\n", - "\n", - "query = select(Example).where(query_text).params(\n", - " term=EqlJsonb({'a': { 'b': 1 } }, \"pyexamples\", \"encrypted_jsonb\").to_db_format(\"ste_vec\")\n", - ")\n", - "result = session.execute(query).scalars().all()\n", - "\n", - "for e in result:\n", - " print(f\"JSONB results: {e}\")\n", - " print(f\"JSONB field: {e.encrypted_jsonb}\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "62bb9642-fe48-4129-a2d3-7a527833952a", - "metadata": {}, - "outputs": [], - "source": [ - "# JSONB containment 1\n", - "cur.execute(\"SELECT * from pyexamples WHERE cs_ste_vec_v1(encrypted_jsonb) @> cs_ste_vec_v1(%s)\", (EqlJsonb({'a': { 'b': 1 } }, \"pyexamples\", \"encrypted_jsonb\").to_db_format(\"ste_vec\"),))\n", - "found = cur.fetchall()\n", - "for f in found:\n", - " print(f\"Record Found with JSONB query: {EqlRow(mapping, f).row}\\n\")\n", - " print(f\"JSONB inside the found record: {EqlJsonb.from_parsed_json(f['encrypted_jsonb'])}\\n\")" - ] - }, - { - "cell_type": "markdown", - "id": "022d30f5-07a7-42e0-92a1-68f85a691a5d", - "metadata": {}, - "source": [ - "### Updating the records" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "50193ce9-e717-46a0-9135-40d13b704a7b", - "metadata": {}, - "outputs": [], - "source": [ - "if len(results) > 0:\n", - " record_id = results[0].id\n", - "\n", - " results[0].encrypted_utf8_str = 'example record 1 UPDATED'\n", - " results[0].encrypted_jsonb = json.dumps({'z': {'y': 0}})\n", - " session.commit()\n", - "\n", - " updated = session.query(Example).where(Example.id == record_id).first()\n", - "\n", - " print(f\"Updated record: {updated}\")\n", - "else:\n", - " print(\"Unexpected: results are empty\")" - ] - }, - { - "cell_type": "markdown", - "id": "9a70970a-c20f-46b9-81a6-ee0b754a0210", - "metadata": {}, - "source": [ - "This updated record is still encrypted in the database:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "37e59466-3959-4c2e-ba95-219949ea9078", - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "%env PGPASSWORD=postgres\n", - "! psql -h localhost -p 5432 -U postgres -x -c \"select * from pyexamples where id = {record_id};\" postgres" - ] - }, - { - "cell_type": "markdown", - "id": "77de1404-7d72-4a5d-86a0-32ca377b6d45", - "metadata": {}, - "source": [ - "## Wrapping up\n", - "\n", - "That's all for this notebook.\n", - "\n", - "There are many more features not covered in this notebook.\n", - "Refer to [EQL repository](https://github.com/cipherstash/encrypt-query-language/) and [CipherStash documentation](https://cipherstash.com/docs) for more information." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - } - }, - "nbformat": 4, - "nbformat_minor": 5 + "cells": [ + { + "cell_type": "markdown", + "id": "6f628465-5dfb-4f76-9a65-4a761cf2b940", + "metadata": {}, + "source": [ + "# Getting Started with CipherStash and Jupyter Notebook\n", + "\n", + "This notebook describes how to get started with CipherStash using Python3, Jupyter Notebook, psycopg2 and SQLAlchemy.and\n", + "\n", + "## Prerequisites\n", + "\n", + "You must have:\n", + "* [PostgreSQL **client**](https://www.postgresql.org/) (The server will be run using Docker in this notebook)\n", + "* [Python 3](https://www.python.org/)\n", + "* [Jupyter Notebook](https://jupyter.org/install)\n", + "* [Docker](https://docs.docker.com/get-started/get-docker/)\n", + "* [Docker compose](https://docs.docker.com/compose/install/)\n", + "* [curl](https://curl.se)\n", + "* [CipherStash account](https://cipherstash.com/signup)\n", + "* [CipherStash CLI](https://github.com/cipherstash/cli-releases/releases/latest)\n", + "\n", + "> Please note that on MS Windows' PowerShell, there is a built-in alias `curl` which is different from [curl listed above](https://curl.se).\n", + "A simple way around this is to start Jupyter Notebook from cmd.exe instead of PowerShell." + ] + }, + { + "cell_type": "markdown", + "id": "77e69c79-c3f6-4c5c-a746-449034b71ac3", + "metadata": {}, + "source": [ + "## Start CipherStash Proxy and PostgreSQL\n", + "\n", + "In order to run the example, you will need to start CipherStash Proxy and PostgreSQL.\n", + "\n", + "Please set up the [playground environment](../../playground/README.md) to run the the following Python examples." + ] + }, + { + "cell_type": "markdown", + "id": "d9df0769-503c-4921-bb83-f4ff2a5f8a6d", + "metadata": {}, + "source": [ + "## Installing required components and table creation\n", + "\n", + "Once the containers are up, there are a few things to be installed.\n", + "A table must also be created to store encrypted data.\n", + "Do the following steps to install them and create a table." + ] + }, + { + "cell_type": "markdown", + "id": "9b4dbe41-2fd5-4335-be55-19058190cd30", + "metadata": {}, + "source": ["### Install application specific database types"] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "505766c0-8c17-432f-943f-875615c88e09", + "metadata": {}, + "outputs": [], + "source": [ + "%env PGPASSWORD=postgres \n", + "! psql -h localhost -p 5432 -U postgres postgres < application_types.sql" + ] + }, + { + "cell_type": "markdown", + "id": "8d8648b8-8b88-4794-9528-e781639a2f0c", + "metadata": {}, + "source": ["### Create a table and indexes for testing encryption"] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "56e04d28-4ebc-463a-b7d0-1ea3cf843f1b", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "%env PGPASSWORD=postgres\n", + "! psql -h localhost -p 5432 -U postgres postgres < create_examples_table.sql" + ] + }, + { + "cell_type": "markdown", + "id": "0400e206-a337-404c-acc9-9706e626e94e", + "metadata": {}, + "source": [ + "## Run Python code\n", + "\n", + "With the services running (this can be checked with `docker compose ps` from the shell), it's time to run some Python code.\n", + "\n", + "Before actual code examples, below is a short introduction of what needs to happen between the native Python data types and encrypted database types." + ] + }, + { + "cell_type": "markdown", + "id": "ece47f71-c115-4f4e-ac8e-f88440fba2dc", + "metadata": {}, + "source": [ + "### Classes that convert between the database format and Python format\n", + "\n", + "There are classes prefixed with `Eql` defined in `eqlpy` which handles conversion between the format CypherStash Proxy requires and the format for Python.\n", + "\n", + "In order to encrypt and store plaintext values, CipherStash Proxy requires encrypted columns in its specific format.\n", + "In Python, this conversion is done by creating an object of `EqlText` as:\n", + "```\n", + "txt = EqlText(\"hello, world\", \"pyexamples\", \"encrypted_utf8_str\")\n", + "txt.to_db_format()\n", + "```\n", + "\n", + "The constructor for `EqlText` takes the string value, the table name (`\"pyexamples\"`) and the column name (`\"encrypted_utf8_str\"`)." + ] + }, + { + "cell_type": "markdown", + "id": "637d0665-e53b-40e5-b685-aa1aa4354e46", + "metadata": {}, + "source": [ + "### Install psycopg2 and sqlalchemy\n", + "\n", + "Install `psycopg2` and `sqlalchemy` if you have not done so yet:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bb1cc638-bd6b-4776-98a7-0e25a778be96", + "metadata": {}, + "outputs": [], + "source": ["%pip install psycopg2 sqlalchemy eqlpy"] + }, + { + "cell_type": "markdown", + "id": "1bbca0fa-4942-4e40-bffa-a3eb2027c671", + "metadata": {}, + "source": [ + "### Import class definitions\n", + "\n", + "There are some classes defined for encrytped types in this project directory.\n", + "They are in `eql_types.py` in the `eqlpy` package if you are interested in implementation details." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ebf51d89-41a7-4988-ab8b-b3cee1c35a31", + "metadata": {}, + "outputs": [], + "source": [ + "from psycopg2.extras import RealDictCursor\n", + "import psycopg2\n", + "from eqlpy.eql_types import *\n", + "\n", + "print(\"Importing done.\")" + ] + }, + { + "cell_type": "markdown", + "id": "777d2c94-4002-4e9e-bbc3-04b598f8aee4", + "metadata": {}, + "source": [ + "## Define column-function mapping\n", + "\n", + "In order to build plain-text `EqlRow`s from encrypted records, we have to provide `EqlRow` with information on which functions should be used to convert them." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f9dcfc0a-bb52-448a-961d-ee8451a23186", + "metadata": {}, + "outputs": [], + "source": [ + "mapping = {\n", + " 'encrypted_int': EqlInt.from_parsed_json,\n", + " 'encrypted_boolean': EqlBool.from_parsed_json,\n", + " 'encrypted_date': EqlDate.from_parsed_json,\n", + " 'encrypted_float': EqlFloat.from_parsed_json,\n", + " 'encrypted_utf8_str': EqlText.from_parsed_json,\n", + " 'encrypted_jsonb': EqlJsonb.from_parsed_json\n", + "}\n", + "\n", + "print(\"column-function mapping defined\")" + ] + }, + { + "cell_type": "markdown", + "id": "6ac472ff-81d1-418a-998b-975b2b3f3a05", + "metadata": {}, + "source": [ + "## Insert test record\n", + "\n", + "With the database extensions, EQL, and application specific data types installed together with the type definitions for Python, your setup is now ready to encrypt and decrypt data.\n", + "\n", + "Run the following to create a record in the `pyexamples` table:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "698c6970-62f0-4cfb-b779-7a5dbbe11d3a", + "metadata": {}, + "outputs": [], + "source": [ + "from pprint import pprint\n", + "from datetime import datetime\n", + "\n", + "conn = psycopg2.connect(\"host=localhost dbname=postgres user=postgres password=postgres port=6432\")\n", + "\n", + "cur = conn.cursor(cursor_factory=RealDictCursor)\n", + "\n", + "cur.execute(\"delete from pyexamples\") # Clear the table in case there are records from previous runs\n", + "cur.execute(\"select cs_refresh_encrypt_config()\")\n", + "\n", + "cur.execute(\"INSERT INTO pyexamples (encrypted_int, encrypted_boolean, encrypted_date, encrypted_float, encrypted_utf8_str) VALUES (%s, %s, %s, %s, %s)\",\n", + " (\n", + " EqlInt(-51, \"pyexamples\", \"encrypted_int\").to_db_format(),\n", + " EqlBool(False, \"pyexamples\", \"encrypted_boolean\").to_db_format(),\n", + " EqlDate(datetime.now().date(), \"pyexamples\", \"encrypted_date\").to_db_format(),\n", + " EqlFloat(-0.5, \"pyexamples\", \"encrypted_float\").to_db_format(),\n", + " EqlText(\"hello, world\", \"pyexamples\", \"encrypted_utf8_str\").to_db_format()\n", + " )\n", + ")\n", + "\n", + "conn.commit()\n", + "\n", + "print(\"example row created in pyexamples table\")" + ] + }, + { + "cell_type": "markdown", + "id": "2f24b9c8-8537-4f44-93f5-925a37bf65b1", + "metadata": {}, + "source": [ + "This should insert a single row in the encrypted `pyexamples` table as:\n", + "\n", + "|encrypted_int|encrypted_boolean|encrypted_date|encrypted_float|encrypted_utf8_str|\n", + "|---|-----|--------------|----|------------|\n", + "|-51|false|2024-11-01 |-0.5|hello, world|\n", + "\n", + "You can check what it looks like from both regular PostgreSQL running on port 5432 and CipherStash Proxy running on port 6432.\n", + "To look at the data through CipherStash Proxy, run the following:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9ab7973d-1d63-43cd-81c2-dd69f544deda", + "metadata": {}, + "outputs": [], + "source": [ + "# From CipherStash Proxy; you should see plaintext JSONB\n", + "%env PGPASSWORD=postgres\n", + "! psql -h localhost -p 6432 -U postgres -x -c \"select * from pyexamples limit 1;\" postgres" + ] + }, + { + "cell_type": "markdown", + "id": "802cbfb3-9869-43ce-adae-1367b23ea52e", + "metadata": {}, + "source": [ + "To look at the data directly on the PostgreSQL server, run the following:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "85597b86-c53f-4df9-87a6-d3eec1730b77", + "metadata": {}, + "outputs": [], + "source": [ + "# From PostgreSQL; you should see JSONB with encrypted values\n", + "%env PGPASSWORD=postgres\n", + "! psql -h localhost -p 5432 -U postgres -x -c \"select * from pyexamples limit 1;\" postgres" + ] + }, + { + "cell_type": "markdown", + "id": "564d2bc7-0c45-4e5f-ba0a-5919961ee9ce", + "metadata": {}, + "source": [ + "In the above example, not all fields are populated, but the populated fields contain JSONB values including the encrypted values." + ] + }, + { + "cell_type": "markdown", + "id": "e824d962-97f2-42b1-a9a2-b283576690d6", + "metadata": {}, + "source": [ + "### Converting to Python types\n", + "\n", + "By querying the proxy, you will see the JSONB values as seen above (decrypted version in the Proxy example, not the PostgreSQL example).\n", + "\n", + "The values should then be converted to types that can be used in Python using class methods for each type:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b7acd767-74ae-4717-ab2b-b71156eb9a9f", + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.display import display, Markdown\n", + "\n", + "cur.execute(\"select * from pyexamples\")\n", + "\n", + "records = cur.fetchall()\n", + "\n", + "record0 = records[0]\n", + "\n", + "# `from_parsed_json` methods convert the values into the corresponding Python types\n", + "content = f\"\"\"\n", + "### Values in the record\n", + "| | |\n", + "|--|--|\n", + "|int | {EqlInt.from_parsed_json(record0['encrypted_int'])}|\n", + "|boolean | {EqlBool.from_parsed_json(record0['encrypted_boolean'])}|\n", + "|datetime | {EqlDate.from_parsed_json(record0['encrypted_date'])}|\n", + "|float | {EqlFloat.from_parsed_json(record0['encrypted_float'])}|\n", + "|text | {EqlText.from_parsed_json(record0['encrypted_utf8_str'])}|\n", + "\"\"\"\n", + "\n", + "display(Markdown(content))\n" + ] + }, + { + "cell_type": "markdown", + "id": "94e823d5-7ee0-4e32-b820-b8fd9e5adfea", + "metadata": {}, + "source": [ + "### Querying with the encrypted fields\n", + "\n", + "You can also use the encrypted fields for queries.\n", + "\n", + "First, add some values so there are more than 1 text values and float values stored:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "36586ea6-09aa-40b0-be4a-17b3c18cfd4e", + "metadata": {}, + "outputs": [], + "source": [ + "# data for MATCH\n", + "cur.execute(\"INSERT INTO pyexamples (encrypted_utf8_str) VALUES (%s) ON CONFLICT DO NOTHING\",\n", + " (\n", + " EqlText(\"hello, python\", \"pyexamples\", \"encrypted_utf8_str\").to_db_format(),\n", + " )\n", + ")\n", + "\n", + "cur.execute(\"INSERT INTO pyexamples (encrypted_utf8_str) VALUES (%s) ON CONFLICT DO NOTHING\",\n", + " (\n", + " EqlText(\"hello, jupyter\", \"pyexamples\", \"encrypted_utf8_str\").to_db_format(),\n", + " )\n", + ")\n", + "\n", + "# data for ORE\n", + "cur.execute(\"INSERT INTO pyexamples (encrypted_float) VALUES (%s)\",\n", + " (\n", + " EqlFloat(100.1, \"pyexamples\", \"encrypted_float\").to_db_format(),\n", + " )\n", + ")\n", + "\n", + "cur.execute(\"INSERT INTO pyexamples (encrypted_float) VALUES (%s)\",\n", + " (\n", + " EqlFloat(100.2, \"pyexamples\", \"encrypted_float\").to_db_format(),\n", + " )\n", + ")\n", + "\n", + "conn.commit()\n", + "\n", + "print(\"created data for MATCH and ORE queries\")" + ] + }, + { + "cell_type": "markdown", + "id": "4e67be50-948d-4832-80e9-6411538cf974", + "metadata": {}, + "source": [ + "The example code above should insert rows like these in the pyexamples table:\n", + "\n", + "| | encrypted_utf_data | encrypted_float||\n", + "|--|---|---|---|\n", + "| |hello, python| | |\n", + "| |hello, jupyter| | |\n", + "| | | 100.1 | |\n", + "| | | 100.2 | |\n" + ] + }, + { + "cell_type": "markdown", + "id": "c2a6b812-980c-4958-a55e-45551cb474c2", + "metadata": {}, + "source": [ + "### Partial matching\n", + "\n", + "Now, a query can be run to look for a record in the `pyexamples` table where `encrypted_utf_8_str` field contains text `\"pyth\"`, which should match `\"hello, python\"`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f9455deb-ea1f-419c-ac72-90f2eb7e76c0", + "metadata": {}, + "outputs": [], + "source": [ + "# MATCH query for \"pyth\"\n", + "cur.execute(\"SELECT * FROM pyexamples WHERE cs_match_v2(encrypted_utf8_str) @> cs_match_v2(%s)\", (EqlText(\"pyth\", \"pyexamples\", \"encrypted_utf8_str\").to_db_format(),))\n", + "\n", + "found = cur.fetchall()[0]\n", + "print(f\"Record Found with MATCH query: {EqlRow(mapping, found).row}\\n\")\n", + "print(f\"Text inside the found record: {EqlText.from_parsed_json(found['encrypted_utf8_str'])}\")" + ] + }, + { + "cell_type": "markdown", + "id": "410be894-23ff-4aed-b8c4-bc283b53a36d", + "metadata": {}, + "source": [ + "### Exact matching\n", + "\n", + "Similarly, a query for the exact text of `\"hello, jupyter\"` in the `encrypted_utf_8_str` field:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "834ce829-9d12-4d14-bdac-20abb2e76148", + "metadata": {}, + "outputs": [], + "source": [ + "# UNIQUE\n", + "cur.execute(\"SELECT * FROM pyexamples WHERE cs_unique_v2(encrypted_utf8_str) = cs_unique_v2(%s)\", (EqlText(\"hello, jupyter\", \"pyexamples\", \"encrypted_utf8_str\").to_db_format(),))\n", + "found = cur.fetchall()[0]\n", + "print(f\"Record Found with UNIQUE query: {EqlRow(mapping, found).row}\\n\")\n", + "print(f\"Text inside the found record: {EqlText.from_parsed_json(found['encrypted_utf8_str'])}\")" + ] + }, + { + "cell_type": "markdown", + "id": "40037eb7-49f9-4a55-8676-4a155056a650", + "metadata": {}, + "source": [ + "### ORE queries\n", + "\n", + "With ORE, you can run a query for a record with `encrypted_float` that is larger than `100.15` which should match `100.2`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "235a8f52-e816-4bbb-8db9-be9330b4b238", + "metadata": {}, + "outputs": [], + "source": [ + "# ORE\n", + "cur.execute(\"SELECT * FROM pyexamples WHERE cs_ore_64_8_v2(encrypted_float) > cs_ore_64_8_v2(%s)\", (EqlFloat(100.15, \"pyexamples\", \"encrypted_float\").to_db_format(),))\n", + "found = cur.fetchall()[0]\n", + "print(f\"Record Found with ORE query: {EqlRow(mapping, found).row}\\n\")\n", + "print(f\"Float inside the found record: {EqlFloat.from_parsed_json(found['encrypted_float'])}\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "e9b9ee3e-6919-4a44-b98c-6964b06c3c87", + "metadata": {}, + "source": [ + "### Updating records\n", + "\n", + "Encrypted fields can be updated too.\n", + "The interface is similar to creating and querying:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9daad342-3bd7-47a0-9744-1aba1c955506", + "metadata": {}, + "outputs": [], + "source": [ + "cur.execute(\"SELECT * FROM pyexamples WHERE cs_unique_v2(encrypted_utf8_str) = cs_unique_v2(%s)\", (EqlText(\"hello, jupyter\", \"pyexamples\", \"encrypted_utf8_str\").to_db_format(),))\n", + "found = cur.fetchall()[0]\n", + "record_id = found['id']\n", + "\n", + "cur.execute(\n", + " \"UPDATE pyexamples SET encrypted_utf8_str = %s WHERE id = %s\",\n", + " (EqlText(\"UPDATED TEXT\", \"pyexamples\", \"encrypted_utf8_str\").to_db_format(), record_id) # Replace 'column_name' and 'new_value' with actual column and value\n", + ")\n", + "cur.execute(\"SELECT * FROM pyexamples WHERE id = %s\", (record_id,))\n", + "found = cur.fetchall()[0]\n", + "print(f\"Updated row: {EqlRow(mapping, found).row}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c8c82996-b91d-4536-a141-1316d293c044", + "metadata": {}, + "source": [ + "### JSONB queries and operations\n", + "\n", + "There are multiple types of JSONB queries and operations supported.\n", + "Here, we introduce:\n", + "\n", + "* Containment query\n", + "* Field extraction\n", + "* WHERE with field exctraction\n", + "* ORDER BY with field extraction\n", + "* GROUP BY with field extraction\n", + "\n", + "First, create the data for JSONB queries. The following queries will create records with JSONB values as:\n", + "\n", + "| encrypted_jsonb |\n", + "|---------|\n", + "| {\"num\": 1, \"category\": \"a\", \"top\": {\"nested\": [\"a\", \"b\", \"c\"]} |\n", + "| {\"num\": 2, \"category\": \"b\", \"top\": {\"nested\": [\"a\"]}} |\n", + "| {\"num\": 3, \"category\": \"b\", \"top\": {\"nested\": [\"z\"]}} |" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ebdf4fe5-1265-484e-8549-cbfdeab5296f", + "metadata": {}, + "outputs": [], + "source": [ + "cur.execute(\"DELETE FROM pyexamples;\")\n", + "\n", + "# data for JSONB queries\n", + "cur.execute(\"INSERT INTO pyexamples (encrypted_jsonb) VALUES (%s)\",\n", + " (\n", + " EqlJsonb({\"num\": 1, \"category\": \"a\", \"top\": {\"nested\": [\"a\", \"b\", \"c\"]}}, \"pyexamples\", \"encrypted_jsonb\").to_db_format(),\n", + " )\n", + ")\n", + "\n", + "cur.execute(\"INSERT INTO pyexamples (encrypted_jsonb) VALUES (%s)\",\n", + " (\n", + " EqlJsonb({\"num\": 2, \"category\": \"b\", \"top\": {\"nested\": [\"a\"]}}, \"pyexamples\", \"encrypted_jsonb\").to_db_format(),\n", + " )\n", + ")\n", + "\n", + "cur.execute(\"INSERT INTO pyexamples (encrypted_jsonb) VALUES (%s)\",\n", + " (\n", + " EqlJsonb({\"num\": 3, \"category\": \"b\", \"top\": {\"nested\": [\"z\"]}}, \"pyexamples\", \"encrypted_jsonb\").to_db_format(),\n", + " )\n", + ")# JSONB containment 1\n", + "\n", + "\n", + "conn.commit()\n", + "\n", + "print(\"Data for JSONB queries created\")" + ] + }, + { + "cell_type": "markdown", + "id": "c1f857e8-8fcb-46f0-b178-e17b580e7538", + "metadata": {}, + "source": [ + "### JSONB containment queries\n", + "\n", + "A record can be found using the JSONB containment.\n", + "The following matches the JSONB field containing keys `top` and `nested` with the `@>` operator:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b05bd5af-4fd7-4d74-8b41-a1c88078cbd2", + "metadata": {}, + "outputs": [], + "source": [ + "# JSONB containment 1\n", + "cur.execute(\"SELECT * from pyexamples WHERE cs_ste_vec_v2(encrypted_jsonb) @> cs_ste_vec_v2(%s)\", (EqlJsonb({'top': { 'nested': ['a', 'b'] } }, \"pyexamples\", \"encrypted_jsonb\").to_db_format(\"ste_vec\"),))\n", + "found = cur.fetchall()\n", + "for f in found:\n", + " print(f\"Record Found with JSONB query: {EqlRow(mapping, f).row}\\n\")\n", + " print(f\"JSONB inside the found record: {EqlJsonb.from_parsed_json(f['encrypted_jsonb'])}\\n\")" + ] + }, + { + "cell_type": "markdown", + "id": "97b4897b-713a-473b-9d97-25f80f263a76", + "metadata": {}, + "source": [ + "Also, the reverse operator (`<@`) is available to the above.\n", + "This query matches the JSONB field that's contained in the query:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8e68a18f-f482-473b-b05a-dd0f4291e031", + "metadata": {}, + "outputs": [], + "source": [ + "# JSONB containment 2\n", + "cur.execute(\"SELECT * from pyexamples WHERE cs_ste_vec_v2(encrypted_jsonb) <@ cs_ste_vec_v2(%s)\", (EqlJsonb({\"num\": 2, \"category\": \"b\", 'top': {'nested': ['a'] } }, \"pyexamples\", \"encrypted_jsonb\").to_db_format(\"ste_vec\"),))\n", + "found = cur.fetchall()\n", + "print(\"Record Found with JSONB query:\")\n", + "for f in found:\n", + " print(f\" {EqlRow(mapping, f).row}\")" + ] + }, + { + "cell_type": "markdown", + "id": "f8ff4434-ddad-40a1-9a28-b5be04b2d4b3", + "metadata": {}, + "source": [ + "### JSONB field extraction\n", + "\n", + "Specific JSONB fields can be accessed in queries.\n", + "\n", + "The following finds all values in `$.top.nested`:\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "909f4645-8c6a-42d2-92ea-f24181232155", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"SELECT cs_ste_vec_value_v2(encrypted_jsonb, %s) AS val FROM pyexamples\"\n", + "\n", + "results = cur.execute(query, (EqlJsonb(\"$.top.nested\", \"pyexamples\", \"encrypted_jsonb\").to_db_format(\"ejson_path\"),))\n", + "\n", + "found = cur.fetchall()\n", + "print(\"values from '$.top.nested':\")\n", + "for f in found:\n", + " print(f\" {EqlRow(mapping, f).row.get('val', {}).get('p')}\")" + ] + }, + { + "cell_type": "markdown", + "id": "ff4a571a-c00e-4e95-9554-c82a5daf9352", + "metadata": {}, + "source": ["### JSONB field in WHERE query"] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2fae30cc-c3a4-4c03-8779-4efdc0c4483c", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"SELECT * FROM pyexamples WHERE cs_ste_vec_term_v2(encrypted_jsonb, %s) > cs_ste_vec_term_v2(%s)\"\n", + "\n", + "cur.execute(\n", + " query,\n", + " (EqlJsonb(\"$.num\", \"pyexamples\", \"encrypted_jsonb\").to_db_format(\"ejson_path\"),\n", + " EqlJsonb(1, \"pyexamples\", \"encrypted_jsonb\").to_db_format(\"ste_vec\")\n", + " )\n", + ")\n", + "\n", + "found = cur.fetchall()\n", + "\n", + "for f in found:\n", + " print(f\"Record Found with JSONB query: {EqlRow(mapping, f).row}\\n\")" + ] + }, + { + "cell_type": "markdown", + "id": "4690460d-bbe8-4aaa-ae27-d013cb73b690", + "metadata": {}, + "source": [ + "### JSONB field in ORDER BY\n", + "\n", + "Field extraction can also be used to order the results.\n", + "The following will demonstrate ordering by the 'num' field in descending order:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "04afc9d6-4ff3-484d-a500-205236a2c06b", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"SELECT * FROM pyexamples ORDER BY cs_ste_vec_term_v2(encrypted_jsonb, %s) DESC\"\n", + "\n", + "cur.execute(\n", + " query,\n", + " (EqlJsonb(\"$.num\", \"pyexamples\", \"encrypted_jsonb\").to_db_format(\"ejson_path\"),)\n", + ")\n", + "found = cur.fetchall();\n", + "print(\"JSONB contents, ordered by 'num' desc:\")\n", + "for f in found:\n", + " print(f\" {EqlRow(mapping, f).row.get('encrypted_jsonb')}\")" + ] + }, + { + "cell_type": "markdown", + "id": "6c18cefe-f8a1-49cc-b10c-0a455e4908b2", + "metadata": {}, + "source": [ + "### JSONB field in GROUP BY\n", + "\n", + "Field extraction can also be used in GROUP BY.\n", + "This example demonstrates how to group the count results by the 'category' column:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6f3b1cd0-a82b-487b-bf32-6eeaf11c6cb3", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"SELECT cs_grouped_value_v2(cs_ste_vec_value_v2(encrypted_jsonb, %s)) AS category, COUNT(*) FROM pyexamples GROUP BY cs_ste_vec_term_v2(encrypted_jsonb, %s)\"\n", + "\n", + "cur.execute(\n", + " query,\n", + " (EqlJsonb(\"$.category\", \"pyexamples\", \"encrypted_jsonb\").to_db_format(\"ejson_path\"),\n", + " EqlJsonb(\"$.category\", \"pyexamples\", \"encrypted_jsonb\").to_db_format(\"ejson_path\"))\n", + ")\n", + "\n", + "found = cur.fetchall()\n", + "\n", + "print(\"count, grouped by category:\")\n", + "for f in found:\n", + " row = EqlRow(mapping, f).row\n", + " print(f\" category: {row.get('category', {}).get('p')}, count: {row.get('count')}\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "f2279c68-06ab-47cb-bd86-dfaf63a7ebb4", + "metadata": {}, + "source": [ + "### Clean up\n", + "\n", + "Clean up the data before going to the next section:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f9b74fa1-430e-49d2-868c-5acff124ceb4", + "metadata": {}, + "outputs": [], + "source": ["cur.execute(\"DELETE FROM pyexamples\")\n", "conn.commit()"] + }, + { + "cell_type": "markdown", + "id": "bf498c2e-00fa-4746-85b3-c4008b55003d", + "metadata": {}, + "source": [ + "## Using SQLAlchemy\n", + "\n", + "### Model definitions and example data\n", + "\n", + "To use SQLAlchemy with CipherStash Encrypt, it is necessary to have model classes that can handle the format conversion.\n", + "\n", + "Import the model definition and create some records as below:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24fda18e-f382-428a-9b49-d859eca04ace", + "metadata": {}, + "outputs": [], + "source": [ + "from sqlalchemy import create_engine, select, text\n", + "from sqlalchemy.orm import sessionmaker\n", + "from eqlpy.eqlalchemy import *\n", + "from datetime import date\n", + "from example_model import Example\n", + "\n", + "# Creating engine. Optionally add echo=True to see the SQL statetments dumped to stdout\n", + "engine = create_engine('postgresql://postgres:postgres@localhost:6432/postgres')\n", + "Session = sessionmaker(bind=engine)\n", + "session = Session()\n", + "\n", + "BaseModel.metadata.create_all(engine) # Create table for models if it's not created yetbelow and\n", + "\n", + "# Clear data if there is any from previous runs\n", + "session.query(Example).delete()\n", + "\n", + "ex = Example(e_utf8_str = \"example record 1\", e_jsonb = {'a': {'b': 1}}, e_int = 42, e_float = 3.14, e_date = date(2024, 10, 25), e_bool=False)\n", + "session.add(ex)\n", + "session.commit()\n", + "\n", + "ex = Example(e_utf8_str = \"example record 2\", e_jsonb = {'a': {'c': 2}}, e_int = 43, e_float = 1.41, e_date = date(2024, 10, 26), e_bool=True)\n", + "session.add(ex)\n", + "session.commit()\n", + "\n", + "ex = Example(e_utf8_str = \"example record 3\", e_jsonb = {'a': {'b': 1}}, e_int = 44, e_float = 2.718, e_date = date(2024, 10, 27), e_bool=True)\n", + "session.add(ex)\n", + "session.commit()\n", + "\n", + "print(\"Example data creation done\")" + ] + }, + { + "cell_type": "markdown", + "id": "97e4fea0-74e5-4f16-9040-1eecbefe5c14", + "metadata": {}, + "source": [ + "### Querying by exact match\n", + "\n", + "With the example data, you can query the data much like the psycopg2 examples above:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6182c491-cfbb-4348-b245-39511f55d1eb", + "metadata": {}, + "outputs": [], + "source": [ + "# UNIQUE\n", + "query_text = text('cs_unique_v2(encrypted_utf8_str) == cs_unique_v2(:term)')\n", + "query = select(Example).where(query_text).params(term=EqlText(\"example record 1\", \"pyexamples\", \"encrypted_utf8_str\").to_db_format())\n", + "results = session.execute(query).scalars().all()\n", + "\n", + "for e in results:\n", + " print(f\"UNIQUE query results: {e}\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "3ca80e10-d31c-48fe-8908-0aa98affbe3b", + "metadata": {}, + "source": ["Model attributes are available as plain text: "] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "23096c06-3136-4c1c-9c47-a6399dc754c8", + "metadata": {}, + "outputs": [], + "source": ["results[0].encrypted_utf8_str"] + }, + { + "cell_type": "markdown", + "id": "1c493bdf-25fa-49ab-a5ab-87e0fbf595e5", + "metadata": {}, + "source": [ + "### Querying by partial match\n", + "\n", + "Partial matching can also performed with SQLAlchemy:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2cfd86e9-70d2-420b-b56f-11744d5ae4bf", + "metadata": {}, + "outputs": [], + "source": [ + "# MATCH\n", + "query_text = text('cs_match_v2(encrypted_utf8_str) @> cs_match_v2(:term)')\n", + "query = select(Example).where(query_text).params(term=EqlText(\"example record\", \"pyexamples\", \"encrypted_utf8_str\").to_db_format())\n", + "results = session.execute(query).scalars().all()\n", + "\n", + "for e in results:\n", + " print(f\"MATCH query results: {e}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c35d9fd3-bfbd-4084-a00d-fee22f7e30d0", + "metadata": {}, + "source": ["### Query by ORE\n", "\n", "ORE queries can be peformed too:"] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5985efb1-1944-41d6-a912-d991095bd7d8", + "metadata": {}, + "outputs": [], + "source": [ + "# ORE\n", + "cur.execute(\"SELECT * FROM pyexamples WHERE cs_ore_64_8_v2(encrypted_float) > cs_ore_64_8_v2(%s)\", (EqlFloat(100.15, \"pyexamples\", \"encrypted_float\").to_db_format(),))\n", + "\n", + "query_text = text('cs_ore_64_8_v2(encrypted_float) > cs_ore_64_8_v2(:term)')\n", + "query = select(Example).where(query_text).params(term=EqlFloat(2.0, \"pyexamples\", \"encrypted_float\").to_db_format())\n", + "results = session.execute(query).scalars().all()\n", + "\n", + "for e in results:\n", + " print(f\"ORE query results: {e}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e93cf011-66c7-4d93-a3c8-cb6ebe4b34ff", + "metadata": {}, + "source": [ + "### Querying by JSONB containment\n", + "\n", + "These records can be queried by JSONB containment too:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5f442da8-55f4-4a62-ab57-26b844b97ea7", + "metadata": {}, + "outputs": [], + "source": [ + "# JSONB\n", + "\n", + "query_text = text(\n", + " \"cs_ste_vec_v2(encrypted_jsonb) @> cs_ste_vec_v2(:term)\"\n", + ")\n", + "\n", + "query = select(Example).where(query_text).params(\n", + " term=EqlJsonb({'a': { 'b': 1 } }, \"pyexamples\", \"encrypted_jsonb\").to_db_format(\"ste_vec\")\n", + ")\n", + "result = session.execute(query).scalars().all()\n", + "\n", + "for e in result:\n", + " print(f\"JSONB results: {e}\")\n", + " print(f\"JSONB field: {e.encrypted_jsonb}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "62bb9642-fe48-4129-a2d3-7a527833952a", + "metadata": {}, + "outputs": [], + "source": [ + "# JSONB containment 1\n", + "cur.execute(\"SELECT * from pyexamples WHERE cs_ste_vec_v2(encrypted_jsonb) @> cs_ste_vec_v2(%s)\", (EqlJsonb({'a': { 'b': 1 } }, \"pyexamples\", \"encrypted_jsonb\").to_db_format(\"ste_vec\"),))\n", + "found = cur.fetchall()\n", + "for f in found:\n", + " print(f\"Record Found with JSONB query: {EqlRow(mapping, f).row}\\n\")\n", + " print(f\"JSONB inside the found record: {EqlJsonb.from_parsed_json(f['encrypted_jsonb'])}\\n\")" + ] + }, + { + "cell_type": "markdown", + "id": "022d30f5-07a7-42e0-92a1-68f85a691a5d", + "metadata": {}, + "source": ["### Updating the records"] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "50193ce9-e717-46a0-9135-40d13b704a7b", + "metadata": {}, + "outputs": [], + "source": [ + "if len(results) > 0:\n", + " record_id = results[0].id\n", + "\n", + " results[0].encrypted_utf8_str = 'example record 1 UPDATED'\n", + " results[0].encrypted_jsonb = json.dumps({'z': {'y': 0}})\n", + " session.commit()\n", + "\n", + " updated = session.query(Example).where(Example.id == record_id).first()\n", + "\n", + " print(f\"Updated record: {updated}\")\n", + "else:\n", + " print(\"Unexpected: results are empty\")" + ] + }, + { + "cell_type": "markdown", + "id": "9a70970a-c20f-46b9-81a6-ee0b754a0210", + "metadata": {}, + "source": ["This updated record is still encrypted in the database:"] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "37e59466-3959-4c2e-ba95-219949ea9078", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "%env PGPASSWORD=postgres\n", + "! psql -h localhost -p 5432 -U postgres -x -c \"select * from pyexamples where id = {record_id};\" postgres" + ] + }, + { + "cell_type": "markdown", + "id": "77de1404-7d72-4a5d-86a0-32ca377b6d45", + "metadata": {}, + "source": [ + "## Wrapping up\n", + "\n", + "That's all for this notebook.\n", + "\n", + "There are many more features not covered in this notebook.\n", + "Refer to [EQL repository](https://github.com/cipherstash/encrypt-query-language/) and [CipherStash documentation](https://cipherstash.com/docs) for more information." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 } diff --git a/examples/python/jupyter_notebook/application_types.sql b/examples/python/jupyter_notebook/application_types.sql index 4fec5994..b27142b6 100644 --- a/examples/python/jupyter_notebook/application_types.sql +++ b/examples/python/jupyter_notebook/application_types.sql @@ -2,49 +2,49 @@ -- Application-specific types -- -CREATE DOMAIN examples__encrypted_big_int AS cs_encrypted_v1 +CREATE DOMAIN examples__encrypted_big_int AS cs_encrypted_v2 CHECK( VALUE#>>'{i,t}' = 'pyexamples' AND VALUE#>>'{i,c}' = 'encrypted_big_int' ); -CREATE DOMAIN examples__encrypted_boolean AS cs_encrypted_v1 +CREATE DOMAIN examples__encrypted_boolean AS cs_encrypted_v2 CHECK( VALUE#>>'{i,t}' = 'pyexamples' AND VALUE#>>'{i,c}' = 'encrypted_boolean' ); -CREATE DOMAIN examples__encrypted_date AS cs_encrypted_v1 +CREATE DOMAIN examples__encrypted_date AS cs_encrypted_v2 CHECK( VALUE#>>'{i,t}' = 'pyexamples' AND VALUE#>>'{i,c}' = 'encrypted_date' ); -CREATE DOMAIN examples__encrypted_float AS cs_encrypted_v1 +CREATE DOMAIN examples__encrypted_float AS cs_encrypted_v2 CHECK( VALUE#>>'{i,t}' = 'pyexamples' AND VALUE#>>'{i,c}' = 'encrypted_float' ); -CREATE DOMAIN examples__encrypted_int AS cs_encrypted_v1 +CREATE DOMAIN examples__encrypted_int AS cs_encrypted_v2 CHECK( VALUE#>>'{i,t}' = 'pyexamples' AND VALUE#>>'{i,c}' = 'encrypted_int' ); -CREATE DOMAIN examples__encrypted_small_int AS cs_encrypted_v1 +CREATE DOMAIN examples__encrypted_small_int AS cs_encrypted_v2 CHECK( VALUE#>>'{i,t}' = 'pyexamples' AND VALUE#>>'{i,c}' = 'encrypted_small_int' ); -CREATE DOMAIN examples__encrypted_utf8_str AS cs_encrypted_v1 +CREATE DOMAIN examples__encrypted_utf8_str AS cs_encrypted_v2 CHECK( VALUE#>>'{i,t}' = 'pyexamples' AND VALUE#>>'{i,c}' = 'encrypted_utf8_str' ); -CREATE DOMAIN examples__encrypted_jsonb AS cs_encrypted_v1 +CREATE DOMAIN examples__encrypted_jsonb AS cs_encrypted_v2 CHECK( VALUE#>>'{i,t}' = 'pyexamples' AND VALUE#>>'{i,c}' = 'encrypted_jsonb' diff --git a/examples/python/jupyter_notebook/create_examples_table.sql b/examples/python/jupyter_notebook/create_examples_table.sql index 110a76fa..d014b98e 100644 --- a/examples/python/jupyter_notebook/create_examples_table.sql +++ b/examples/python/jupyter_notebook/create_examples_table.sql @@ -9,26 +9,26 @@ create table pyexamples ( ); -- Add CipherStash indexes to Encrypt config -SELECT cs_add_index_v1('pyexamples', 'encrypted_boolean', 'ore', 'boolean'); -SELECT cs_add_index_v1('pyexamples', 'encrypted_date', 'ore', 'date'); -SELECT cs_add_index_v1('pyexamples', 'encrypted_float', 'ore', 'double'); -SELECT cs_add_index_v1('pyexamples', 'encrypted_int', 'ore', 'int'); -SELECT cs_add_index_v1('pyexamples', 'encrypted_utf8_str', 'unique', 'text', '{"token_filters": [{"kind": "downcase"}]}'); -SELECT cs_add_index_v1('pyexamples', 'encrypted_utf8_str', 'match', 'text'); -SELECT cs_add_index_v1('pyexamples', 'encrypted_utf8_str', 'ore', 'text'); -SELECT cs_add_index_v1('pyexamples', 'encrypted_jsonb', 'ste_vec', 'jsonb', '{"prefix": "pyexamples/encrypted_jsonb"}'); +SELECT cs_add_index_v2('pyexamples', 'encrypted_boolean', 'ore', 'boolean'); +SELECT cs_add_index_v2('pyexamples', 'encrypted_date', 'ore', 'date'); +SELECT cs_add_index_v2('pyexamples', 'encrypted_float', 'ore', 'double'); +SELECT cs_add_index_v2('pyexamples', 'encrypted_int', 'ore', 'int'); +SELECT cs_add_index_v2('pyexamples', 'encrypted_utf8_str', 'unique', 'text', '{"token_filters": [{"kind": "downcase"}]}'); +SELECT cs_add_index_v2('pyexamples', 'encrypted_utf8_str', 'match', 'text'); +SELECT cs_add_index_v2('pyexamples', 'encrypted_utf8_str', 'ore', 'text'); +SELECT cs_add_index_v2('pyexamples', 'encrypted_jsonb', 'ste_vec', 'jsonb', '{"prefix": "pyexamples/encrypted_jsonb"}'); -- Add corresponding PG indexes for each CipherStash index -CREATE INDEX ON pyexamples (cs_ore_64_8_v1(encrypted_boolean)); -CREATE INDEX ON pyexamples (cs_ore_64_8_v1(encrypted_date)); -CREATE INDEX ON pyexamples (cs_ore_64_8_v1(encrypted_float)); -CREATE INDEX ON pyexamples (cs_ore_64_8_v1(encrypted_int)); -CREATE UNIQUE INDEX ON pyexamples(cs_unique_v1(encrypted_utf8_str)); -CREATE INDEX ON pyexamples USING GIN (cs_match_v1(encrypted_utf8_str)); -CREATE INDEX ON pyexamples (cs_ore_64_8_v1(encrypted_utf8_str)); --- CREATE INDEX ON pyexamples USING GIN (cs_ste_vec_v1(encrypted_jsonb)); +CREATE INDEX ON pyexamples (cs_ore_64_8_v2(encrypted_boolean)); +CREATE INDEX ON pyexamples (cs_ore_64_8_v2(encrypted_date)); +CREATE INDEX ON pyexamples (cs_ore_64_8_v2(encrypted_float)); +CREATE INDEX ON pyexamples (cs_ore_64_8_v2(encrypted_int)); +CREATE UNIQUE INDEX ON pyexamples(cs_unique_v2(encrypted_utf8_str)); +CREATE INDEX ON pyexamples USING GIN (cs_match_v2(encrypted_utf8_str)); +CREATE INDEX ON pyexamples (cs_ore_64_8_v2(encrypted_utf8_str)); +-- CREATE INDEX ON pyexamples USING GIN (cs_ste_vec_v2(encrypted_jsonb)); -- Transition the Encrypt config state from "pending", to "encrypting", and then "active". -- The Encrypt config must be "active" for Proxy to use it. -SELECT cs_encrypt_v1(true); -SELECT cs_activate_v1(); +SELECT cs_encrypt_v2(true); +SELECT cs_activate_v2(); diff --git a/mise.toml b/mise.toml index ccfd9ef0..6b10ab7e 100644 --- a/mise.toml +++ b/mise.toml @@ -7,10 +7,7 @@ # "./tests/mise.tls.toml", # ] [task_config] -includes = [ - "tasks", - "tasks/postgres.toml" -] +includes = ["tasks", "tasks/postgres.toml"] [env] POSTGRES_DB = "cipherstash" @@ -18,3 +15,11 @@ POSTGRES_USER = "cipherstash" POSTGRES_PASSWORD = "password" POSTGRES_HOST = "localhost" POSTGRES_PORT = "7432" + +[tasks."clean"] +alias = 'k' +description = "Clean release" +run = """ + rm -f release/cipherstash-encrypt-uninstall.sql + rm -f release/cipherstash-encrypt.sql +""" diff --git a/sql/001-ore.sql b/sql/001-ore.sql deleted file mode 100644 index 84735758..00000000 --- a/sql/001-ore.sql +++ /dev/null @@ -1,427 +0,0 @@ -CREATE EXTENSION IF NOT EXISTS pgcrypto; - -CREATE TYPE ore_64_8_v1_term AS ( - bytes bytea -); - -CREATE TYPE ore_64_8_v1 AS ( - terms ore_64_8_v1_term[] -); - -DROP FUNCTION IF EXISTS compare_ore_64_8_v1_term(a ore_64_8_v1_term, b ore_64_8_v1_term); - -CREATE FUNCTION compare_ore_64_8_v1_term(a ore_64_8_v1_term, b ore_64_8_v1_term) returns integer AS $$ - DECLARE - eq boolean := true; - unequal_block smallint := 0; - hash_key bytea; - target_block bytea; - - left_block_size CONSTANT smallint := 16; - right_block_size CONSTANT smallint := 32; - right_offset CONSTANT smallint := 136; -- 8 * 17 - - indicator smallint := 0; - BEGIN - IF a IS NULL AND b IS NULL THEN - RETURN 0; - END IF; - - IF a IS NULL THEN - RETURN -1; - END IF; - - IF b IS NULL THEN - RETURN 1; - END IF; - - IF bit_length(a.bytes) != bit_length(b.bytes) THEN - RAISE EXCEPTION 'Ciphertexts are different lengths'; - END IF; - - FOR block IN 0..7 LOOP - -- Compare each PRP (byte from the first 8 bytes) and PRF block (8 byte - -- chunks of the rest of the value). - -- NOTE: - -- * Substr is ordinally indexed (hence 1 and not 0, and 9 and not 8). - -- * We are not worrying about timing attacks here; don't fret about - -- the OR or !=. - IF - substr(a.bytes, 1 + block, 1) != substr(b.bytes, 1 + block, 1) - OR substr(a.bytes, 9 + left_block_size * block, left_block_size) != substr(b.bytes, 9 + left_block_size * BLOCK, left_block_size) - THEN - -- set the first unequal block we find - IF eq THEN - unequal_block := block; - END IF; - eq = false; - END IF; - END LOOP; - - IF eq THEN - RETURN 0::integer; - END IF; - - -- Hash key is the IV from the right CT of b - hash_key := substr(b.bytes, right_offset + 1, 16); - - -- first right block is at right offset + nonce_size (ordinally indexed) - target_block := substr(b.bytes, right_offset + 17 + (unequal_block * right_block_size), right_block_size); - - indicator := ( - get_bit( - encrypt( - substr(a.bytes, 9 + (left_block_size * unequal_block), left_block_size), - hash_key, - 'aes-ecb' - ), - 0 - ) + get_bit(target_block, get_byte(a.bytes, unequal_block))) % 2; - - IF indicator = 1 THEN - RETURN 1::integer; - ELSE - RETURN -1::integer; - END IF; - END; -$$ LANGUAGE plpgsql; - - -DROP FUNCTION IF EXISTS ore_64_8_v1_term_eq(a ore_64_8_v1_term, b ore_64_8_v1_term); - -CREATE FUNCTION ore_64_8_v1_term_eq(a ore_64_8_v1_term, b ore_64_8_v1_term) -RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1_term(a, b) = 0 -$$ LANGUAGE SQL; - - -DROP FUNCTION IF EXISTS ore_64_8_v1_term_neq(a ore_64_8_v1_term, b ore_64_8_v1_term); - -CREATE FUNCTION ore_64_8_v1_term_neq(a ore_64_8_v1_term, b ore_64_8_v1_term) -RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1_term(a, b) <> 0 -$$ LANGUAGE SQL; - - -DROP FUNCTION IF EXISTS ore_64_8_v1_term_lt(a ore_64_8_v1_term, b ore_64_8_v1_term); - -CREATE FUNCTION ore_64_8_v1_term_lt(a ore_64_8_v1_term, b ore_64_8_v1_term) -RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1_term(a, b) = -1 -$$ LANGUAGE SQL; - - -DROP FUNCTION IF EXISTS ore_64_8_v1_term_lte(a ore_64_8_v1_term, b ore_64_8_v1_term); - -CREATE FUNCTION ore_64_8_v1_term_lte(a ore_64_8_v1_term, b ore_64_8_v1_term) -RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1_term(a, b) != 1 -$$ LANGUAGE SQL; - - -DROP FUNCTION IF EXISTS ore_64_8_v1_term_gt(a ore_64_8_v1_term, b ore_64_8_v1_term); - -CREATE FUNCTION ore_64_8_v1_term_gt(a ore_64_8_v1_term, b ore_64_8_v1_term) -RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1_term(a, b) = 1 -$$ LANGUAGE SQL; - - -DROP FUNCTION IF EXISTS ore_64_8_v1_term_gte(a ore_64_8_v1_term, b ore_64_8_v1_term); - -CREATE FUNCTION ore_64_8_v1_term_gte(a ore_64_8_v1_term, b ore_64_8_v1_term) -RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1_term(a, b) != -1 -$$ LANGUAGE SQL; - - -DROP OPERATOR IF EXISTS = (ore_64_8_v1_term, ore_64_8_v1_term); - -CREATE OPERATOR = ( - PROCEDURE="ore_64_8_v1_term_eq", - LEFTARG=ore_64_8_v1_term, - RIGHTARG=ore_64_8_v1_term, - NEGATOR = <>, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - - -DROP OPERATOR IF EXISTS <> (ore_64_8_v1_term, ore_64_8_v1_term); - -CREATE OPERATOR <> ( - PROCEDURE="ore_64_8_v1_term_neq", - LEFTARG=ore_64_8_v1_term, - RIGHTARG=ore_64_8_v1_term, - NEGATOR = =, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - -DROP OPERATOR IF EXISTS > (ore_64_8_v1_term, ore_64_8_v1_term); - -CREATE OPERATOR > ( - PROCEDURE="ore_64_8_v1_term_gt", - LEFTARG=ore_64_8_v1_term, - RIGHTARG=ore_64_8_v1_term, - COMMUTATOR = <, - NEGATOR = <=, - RESTRICT = scalargtsel, - JOIN = scalargtjoinsel -); - -DROP OPERATOR IF EXISTS < (ore_64_8_v1_term, ore_64_8_v1_term); - -CREATE OPERATOR < ( - PROCEDURE="ore_64_8_v1_term_lt", - LEFTARG=ore_64_8_v1_term, - RIGHTARG=ore_64_8_v1_term, - COMMUTATOR = >, - NEGATOR = >=, - RESTRICT = scalarltsel, - JOIN = scalarltjoinsel -); - -DROP OPERATOR IF EXISTS <= (ore_64_8_v1_term, ore_64_8_v1_term); - -CREATE OPERATOR <= ( - PROCEDURE="ore_64_8_v1_term_lte", - LEFTARG=ore_64_8_v1_term, - RIGHTARG=ore_64_8_v1_term, - COMMUTATOR = >=, - NEGATOR = >, - RESTRICT = scalarlesel, - JOIN = scalarlejoinsel -); - -DROP OPERATOR IF EXISTS >= (ore_64_8_v1_term, ore_64_8_v1_term); - -CREATE OPERATOR >= ( - PROCEDURE="ore_64_8_v1_term_gte", - LEFTARG=ore_64_8_v1_term, - RIGHTARG=ore_64_8_v1_term, - COMMUTATOR = <=, - NEGATOR = <, - RESTRICT = scalarlesel, - JOIN = scalarlejoinsel -); - -DROP OPERATOR FAMILY IF EXISTS ore_64_8_v1_term_btree_ops USING btree; - -CREATE OPERATOR FAMILY ore_64_8_v1_term_btree_ops USING btree; - - -DROP OPERATOR CLASS IF EXISTS ore_64_8_v1_term_btree_ops USING btree; - -CREATE OPERATOR CLASS ore_64_8_v1_term_btree_ops DEFAULT FOR TYPE ore_64_8_v1_term USING btree FAMILY ore_64_8_v1_term_btree_ops AS - OPERATOR 1 <, - OPERATOR 2 <=, - OPERATOR 3 =, - OPERATOR 4 >=, - OPERATOR 5 >, - FUNCTION 1 compare_ore_64_8_v1_term(a ore_64_8_v1_term, b ore_64_8_v1_term); - --- Compare the "head" of each array and recurse if necessary --- This function assumes an empty string is "less than" everything else --- so if a is empty we return -1, if be is empty and a isn't, we return 1. --- If both are empty we return 0. This cases probably isn't necessary as equality --- doesn't always make sense but it's here for completeness. --- If both are non-empty, we compare the first element. If they are equal --- we need to consider the next block so we recurse, otherwise we return the comparison result. -DROP FUNCTION IF EXISTS compare_ore_array(a ore_64_8_v1_term[], b ore_64_8_v1_term[]); - -CREATE FUNCTION compare_ore_array(a ore_64_8_v1_term[], b ore_64_8_v1_term[]) -RETURNS integer AS $$ - DECLARE - cmp_result integer; - BEGIN - - -- NULLs are NULL - IF a IS NULL OR b IS NULL THEN - RETURN NULL; - END IF; - - -- empty a and b - IF cardinality(a) = 0 AND cardinality(b) = 0 THEN - RETURN 0; - END IF; - - -- empty a and some b - IF (cardinality(a) = 0) AND cardinality(b) > 0 THEN - RETURN -1; - END IF; - - -- some a and empty b - IF cardinality(a) > 0 AND (cardinality(b) = 0) THEN - RETURN 1; - END IF; - - cmp_result := compare_ore_64_8_v1_term(a[1], b[1]); - IF cmp_result = 0 THEN - -- Removes the first element in the array, and calls this fn again to compare the next element/s in the array. - RETURN compare_ore_array(a[2:array_length(a,1)], b[2:array_length(b,1)]); - END IF; - - RETURN cmp_result; - END -$$ LANGUAGE plpgsql; - --- This function uses lexicographic comparison -DROP FUNCTION IF EXISTS compare_ore_64_8_v1(a ore_64_8_v1, b ore_64_8_v1); - -CREATE FUNCTION compare_ore_64_8_v1(a ore_64_8_v1, b ore_64_8_v1) -RETURNS integer AS $$ - DECLARE - cmp_result integer; - BEGIN - -- Recursively compare blocks bailing as soon as we can make a decision - RETURN compare_ore_array(a.terms, b.terms); - END -$$ LANGUAGE plpgsql; - - -DROP FUNCTION IF EXISTS ore_64_8_v1_eq(a ore_64_8_v1, b ore_64_8_v1); - -CREATE FUNCTION ore_64_8_v1_eq(a ore_64_8_v1, b ore_64_8_v1) -RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1(a, b) = 0 -$$ LANGUAGE SQL; - - -DROP FUNCTION IF EXISTS ore_64_8_v1_neq(a ore_64_8_v1, b ore_64_8_v1); - -CREATE FUNCTION ore_64_8_v1_neq(a ore_64_8_v1, b ore_64_8_v1) -RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1(a, b) <> 0 -$$ LANGUAGE SQL; - - -DROP FUNCTION IF EXISTS ore_64_8_v1_lt(a ore_64_8_v1, b ore_64_8_v1); - -CREATE FUNCTION ore_64_8_v1_lt(a ore_64_8_v1, b ore_64_8_v1) -RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1(a, b) = -1 -$$ LANGUAGE SQL; - - -DROP FUNCTION IF EXISTS ore_64_8_v1_lte(a ore_64_8_v1, b ore_64_8_v1); - -CREATE FUNCTION ore_64_8_v1_lte(a ore_64_8_v1, b ore_64_8_v1) -RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1(a, b) != 1 -$$ LANGUAGE SQL; - - -DROP FUNCTION IF EXISTS ore_64_8_v1_gt(a ore_64_8_v1, b ore_64_8_v1); - -CREATE FUNCTION ore_64_8_v1_gt(a ore_64_8_v1, b ore_64_8_v1) -RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1(a, b) = 1 -$$ LANGUAGE SQL; - - -DROP FUNCTION IF EXISTS ore_64_8_v1_gte(a ore_64_8_v1, b ore_64_8_v1); - -CREATE FUNCTION ore_64_8_v1_gte(a ore_64_8_v1, b ore_64_8_v1) -RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1(a, b) != -1 -$$ LANGUAGE SQL; - - -DROP OPERATOR IF EXISTS = (ore_64_8_v1, ore_64_8_v1); - -CREATE OPERATOR = ( - PROCEDURE="ore_64_8_v1_eq", - LEFTARG=ore_64_8_v1, - RIGHTARG=ore_64_8_v1, - NEGATOR = <>, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - - -DROP OPERATOR IF EXISTS <> (ore_64_8_v1, ore_64_8_v1); - -CREATE OPERATOR <> ( - PROCEDURE="ore_64_8_v1_neq", - LEFTARG=ore_64_8_v1, - RIGHTARG=ore_64_8_v1, - NEGATOR = =, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - -DROP OPERATOR IF EXISTS > (ore_64_8_v1, ore_64_8_v1); - -CREATE OPERATOR > ( - PROCEDURE="ore_64_8_v1_gt", - LEFTARG=ore_64_8_v1, - RIGHTARG=ore_64_8_v1, - COMMUTATOR = <, - NEGATOR = <=, - RESTRICT = scalargtsel, - JOIN = scalargtjoinsel -); - - -DROP OPERATOR IF EXISTS < (ore_64_8_v1, ore_64_8_v1); - -CREATE OPERATOR < ( - PROCEDURE="ore_64_8_v1_lt", - LEFTARG=ore_64_8_v1, - RIGHTARG=ore_64_8_v1, - COMMUTATOR = >, - NEGATOR = >=, - RESTRICT = scalarltsel, - JOIN = scalarltjoinsel -); - - -DROP OPERATOR IF EXISTS <= (ore_64_8_v1, ore_64_8_v1); - -CREATE OPERATOR <= ( - PROCEDURE="ore_64_8_v1_lte", - LEFTARG=ore_64_8_v1, - RIGHTARG=ore_64_8_v1, - COMMUTATOR = >=, - NEGATOR = >, - RESTRICT = scalarlesel, - JOIN = scalarlejoinsel -); - - -DROP OPERATOR IF EXISTS >= (ore_64_8_v1, ore_64_8_v1); - -CREATE OPERATOR >= ( - PROCEDURE="ore_64_8_v1_gte", - LEFTARG=ore_64_8_v1, - RIGHTARG=ore_64_8_v1, - COMMUTATOR = <=, - NEGATOR = <, - RESTRICT = scalarlesel, - JOIN = scalarlejoinsel -); - - -DROP OPERATOR FAMILY IF EXISTS ore_64_8_v1_btree_ops USING btree; - -CREATE OPERATOR FAMILY ore_64_8_v1_btree_ops USING btree; - - -DROP OPERATOR CLASS IF EXISTS ore_64_8_v1_btree_ops USING btree; - -CREATE OPERATOR CLASS ore_64_8_v1_btree_ops DEFAULT FOR TYPE ore_64_8_v1 USING btree FAMILY ore_64_8_v1_btree_ops AS - OPERATOR 1 <, - OPERATOR 2 <=, - OPERATOR 3 =, - OPERATOR 4 >=, - OPERATOR 5 >, - FUNCTION 1 compare_ore_64_8_v1(a ore_64_8_v1, b ore_64_8_v1); diff --git a/sql/002-ore-cllw.sql b/sql/002-ore-cllw.sql deleted file mode 100644 index 4330b145..00000000 --- a/sql/002-ore-cllw.sql +++ /dev/null @@ -1,439 +0,0 @@ - ---- ---- ORE CLLW types, functions, and operators ---- - --- Represents a ciphertext encrypted with the CLLW ORE scheme for a fixed output size --- Each output block is 8-bits -CREATE TYPE ore_cllw_8_v1 AS ( - bytes bytea -); - --- Represents a ciphertext encrypted with the CLLW ORE scheme for a variable output size --- Each output block is 8-bits -CREATE TYPE ore_cllw_8_variable_v1 AS ( - bytes bytea -); - -DROP FUNCTION IF EXISTS __bytea_ct_eq(a bytea, b bytea); - --- Constant time comparison of 2 bytea values -CREATE FUNCTION __bytea_ct_eq(a bytea, b bytea) RETURNS boolean AS $$ -DECLARE - result boolean; - differing bytea; -BEGIN - -- Check if the bytea values are the same length - IF LENGTH(a) != LENGTH(b) THEN - RETURN false; - END IF; - - -- Compare each byte in the bytea values - result := true; - FOR i IN 1..LENGTH(a) LOOP - IF SUBSTRING(a FROM i FOR 1) != SUBSTRING(b FROM i FOR 1) THEN - result := result AND false; - END IF; - END LOOP; - - RETURN result; -END; -$$ LANGUAGE plpgsql; - -DROP FUNCTION IF EXISTS __compare_inner_ore_cllw_8_v1(a bytea, b bytea); - -CREATE FUNCTION __compare_inner_ore_cllw_8_v1(a bytea, b bytea) -RETURNS int AS $$ -DECLARE - len_a INT; - x BYTEA; - y BYTEA; - i INT; - differing RECORD; -BEGIN - len_a := LENGTH(a); - - -- Iterate over each byte and compare them - FOR i IN 1..len_a LOOP - x := SUBSTRING(a FROM i FOR 1); - y := SUBSTRING(b FROM i FOR 1); - - -- Check if there's a difference - IF x != y THEN - differing := (x, y); - EXIT; - END IF; - END LOOP; - - -- If a difference is found, compare the bytes as in Rust logic - IF differing IS NOT NULL THEN - IF (get_byte(y, 0) + 1) % 256 = get_byte(x, 0) THEN - RETURN 1; - ELSE - RETURN -1; - END IF; - ELSE - RETURN 0; - END IF; -END; -$$ LANGUAGE plpgsql; - - -DROP FUNCTION IF EXISTS compare_ore_cllw_8_v1(a ore_cllw_8_v1, b ore_cllw_8_v1); - -CREATE FUNCTION compare_ore_cllw_8_v1(a ore_cllw_8_v1, b ore_cllw_8_v1) -RETURNS int AS $$ -DECLARE - len_a INT; - len_b INT; - x BYTEA; - y BYTEA; - i INT; - differing RECORD; -BEGIN - -- Check if the lengths of the two bytea arguments are the same - len_a := LENGTH(a.bytes); - len_b := LENGTH(b.bytes); - - IF len_a != len_b THEN - RAISE EXCEPTION 'Numeric ORE comparison requires bytea values of the same length'; - END IF; - - RETURN __compare_inner_ore_cllw_8_v1(a.bytes, b.bytes); -END; -$$ LANGUAGE plpgsql; - - -DROP FUNCTION IF EXISTS compare_lex_ore_cllw_8_v1(a ore_cllw_8_variable_v1, b ore_cllw_8_variable_v1); - -CREATE FUNCTION compare_lex_ore_cllw_8_v1(a ore_cllw_8_variable_v1, b ore_cllw_8_variable_v1) -RETURNS int AS $$ -DECLARE - len_a INT; - len_b INT; - -- length of the common part of the two bytea values - common_len INT; - cmp_result INT; -BEGIN - -- Get the lengths of both bytea inputs - len_a := LENGTH(a.bytes); - len_b := LENGTH(b.bytes); - - -- Handle empty cases - IF len_a = 0 AND len_b = 0 THEN - RETURN 0; - ELSIF len_a = 0 THEN - RETURN -1; - ELSIF len_b = 0 THEN - RETURN 1; - END IF; - - -- Find the length of the shorter bytea - IF len_a < len_b THEN - common_len := len_a; - ELSE - common_len := len_b; - END IF; - - -- Use the compare_bytea function to compare byte by byte - cmp_result := __compare_inner_ore_cllw_8_v1( - SUBSTRING(a.bytes FROM 1 FOR common_len), - SUBSTRING(b.bytes FROM 1 FOR common_len) - ); - - -- If the comparison returns 'less' or 'greater', return that result - IF cmp_result = -1 THEN - RETURN -1; - ELSIF cmp_result = 1 THEN - RETURN 1; - END IF; - - -- If the bytea comparison is 'equal', compare lengths - IF len_a < len_b THEN - RETURN -1; - ELSIF len_a > len_b THEN - RETURN 1; - ELSE - RETURN 0; - END IF; -END; -$$ LANGUAGE plpgsql; - - -DROP FUNCTION IF EXISTS ore_cllw_8_v1_eq(a ore_cllw_8_v1, b ore_cllw_8_v1); - -CREATE FUNCTION ore_cllw_8_v1_eq(a ore_cllw_8_v1, b ore_cllw_8_v1) -RETURNS boolean AS $$ - SELECT __bytea_ct_eq(a.bytes, b.bytes) -$$ LANGUAGE SQL; - - -DROP FUNCTION IF EXISTS ore_cllw_8_v1_neq(a ore_cllw_8_v1, b ore_cllw_8_v1); - -CREATE FUNCTION ore_cllw_8_v1_neq(a ore_cllw_8_v1, b ore_cllw_8_v1) -RETURNS boolean AS $$ - SELECT not __bytea_ct_eq(a.bytes, b.bytes) -$$ LANGUAGE SQL; - - -DROP FUNCTION IF EXISTS ore_cllw_8_v1_lt(a ore_cllw_8_v1, b ore_cllw_8_v1); - -CREATE FUNCTION ore_cllw_8_v1_lt(a ore_cllw_8_v1, b ore_cllw_8_v1) -RETURNS boolean AS $$ - SELECT compare_ore_cllw_8_v1(a, b) = -1 -$$ LANGUAGE SQL; - - -DROP FUNCTION IF EXISTS ore_cllw_8_v1_lte(a ore_cllw_8_v1, b ore_cllw_8_v1); - -CREATE FUNCTION ore_cllw_8_v1_lte(a ore_cllw_8_v1, b ore_cllw_8_v1) -RETURNS boolean AS $$ - SELECT compare_ore_cllw_8_v1(a, b) != 1 -$$ LANGUAGE SQL; - - -DROP FUNCTION IF EXISTS ore_cllw_8_v1_gt(a ore_cllw_8_v1, b ore_cllw_8_v1); - -CREATE FUNCTION ore_cllw_8_v1_gt(a ore_cllw_8_v1, b ore_cllw_8_v1) -RETURNS boolean AS $$ - SELECT compare_ore_cllw_8_v1(a, b) = 1 -$$ LANGUAGE SQL; - - -DROP FUNCTION IF EXISTS ore_cllw_8_v1_gte(a ore_cllw_8_v1, b ore_cllw_8_v1); - -CREATE FUNCTION ore_cllw_8_v1_gte(a ore_cllw_8_v1, b ore_cllw_8_v1) -RETURNS boolean AS $$ - SELECT compare_ore_cllw_8_v1(a, b) != -1 -$$ LANGUAGE SQL; - - -DROP OPERATOR IF EXISTS = (ore_cllw_8_v1, ore_cllw_8_v1); - -CREATE OPERATOR = ( - PROCEDURE="ore_cllw_8_v1_eq", - LEFTARG=ore_cllw_8_v1, - RIGHTARG=ore_cllw_8_v1, - NEGATOR = <>, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - - -DROP OPERATOR IF EXISTS <> (ore_cllw_8_v1, ore_cllw_8_v1); - -CREATE OPERATOR <> ( - PROCEDURE="ore_cllw_8_v1_neq", - LEFTARG=ore_cllw_8_v1, - RIGHTARG=ore_cllw_8_v1, - NEGATOR = =, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - - -DROP OPERATOR IF EXISTS > (ore_cllw_8_v1, ore_cllw_8_v1); - -CREATE OPERATOR > ( - PROCEDURE="ore_cllw_8_v1_gt", - LEFTARG=ore_cllw_8_v1, - RIGHTARG=ore_cllw_8_v1, - NEGATOR = <=, - RESTRICT = scalarltsel, - JOIN = scalarltjoinsel, - HASHES, - MERGES -); - - -DROP OPERATOR IF EXISTS < (ore_cllw_8_v1, ore_cllw_8_v1); - -CREATE OPERATOR < ( - PROCEDURE="ore_cllw_8_v1_lt", - LEFTARG=ore_cllw_8_v1, - RIGHTARG=ore_cllw_8_v1, - NEGATOR = >=, - RESTRICT = scalargtsel, - JOIN = scalargtjoinsel, - HASHES, - MERGES -); - - -DROP OPERATOR IF EXISTS >= (ore_cllw_8_v1, ore_cllw_8_v1); - -CREATE OPERATOR >= ( - PROCEDURE="ore_cllw_8_v1_gte", - LEFTARG=ore_cllw_8_v1, - RIGHTARG=ore_cllw_8_v1, - NEGATOR = <, - RESTRICT = scalarltsel, - JOIN = scalarltjoinsel, - HASHES, - MERGES -); - - -DROP OPERATOR IF EXISTS <= (ore_cllw_8_v1, ore_cllw_8_v1); - -CREATE OPERATOR <= ( - PROCEDURE="ore_cllw_8_v1_lte", - LEFTARG=ore_cllw_8_v1, - RIGHTARG=ore_cllw_8_v1, - NEGATOR = >, - RESTRICT = scalargtsel, - JOIN = scalargtjoinsel, - HASHES, - MERGES -); - --- Lexical comparison operators - -DROP FUNCTION IF EXISTS ore_cllw_8_variable_v1_eq(a ore_cllw_8_variable_v1, b ore_cllw_8_variable_v1); - -CREATE OR REPLACE FUNCTION ore_cllw_8_variable_v1_eq(a ore_cllw_8_variable_v1, b ore_cllw_8_variable_v1) RETURNS boolean AS $$ - SELECT __bytea_ct_eq(a.bytes, b.bytes) -$$ LANGUAGE SQL; - -DROP FUNCTION IF EXISTS ore_cllw_8_variable_v1_neq(a ore_cllw_8_variable_v1, b ore_cllw_8_variable_v1); - -CREATE OR REPLACE FUNCTION ore_cllw_8_variable_v1_neq(a ore_cllw_8_variable_v1, b ore_cllw_8_variable_v1) RETURNS boolean AS $$ - SELECT not __bytea_ct_eq(a.bytes, b.bytes) -$$ LANGUAGE SQL; - -DROP FUNCTION IF EXISTS ore_cllw_8_v1_lt_lex(a ore_cllw_8_variable_v1, b ore_cllw_8_variable_v1); - -CREATE OR REPLACE FUNCTION ore_cllw_8_v1_lt_lex(a ore_cllw_8_variable_v1, b ore_cllw_8_variable_v1) RETURNS boolean AS $$ - SELECT compare_lex_ore_cllw_8_v1(a, b) = -1 -$$ LANGUAGE SQL; - -DROP FUNCTION IF EXISTS ore_cllw_8_v1_lte_lex(a ore_cllw_8_variable_v1, b ore_cllw_8_variable_v1); - -CREATE OR REPLACE FUNCTION ore_cllw_8_v1_lte_lex(a ore_cllw_8_variable_v1, b ore_cllw_8_variable_v1) RETURNS boolean AS $$ - SELECT compare_lex_ore_cllw_8_v1(a, b) != 1 -$$ LANGUAGE SQL; - -DROP FUNCTION IF EXISTS ore_cllw_8_v1_gt_lex(a ore_cllw_8_variable_v1, b ore_cllw_8_variable_v1); - -CREATE OR REPLACE FUNCTION ore_cllw_8_v1_gt_lex(a ore_cllw_8_variable_v1, b ore_cllw_8_variable_v1) RETURNS boolean AS $$ - SELECT compare_lex_ore_cllw_8_v1(a, b) = 1 -$$ LANGUAGE SQL; - -DROP FUNCTION IF EXISTS ore_cllw_8_v1_gte_lex(a ore_cllw_8_variable_v1, b ore_cllw_8_variable_v1); - -CREATE OR REPLACE FUNCTION ore_cllw_8_v1_gte_lex(a ore_cllw_8_variable_v1, b ore_cllw_8_variable_v1) RETURNS boolean AS $$ - SELECT compare_lex_ore_cllw_8_v1(a, b) != -1 -$$ LANGUAGE SQL; - -DROP OPERATOR IF EXISTS = (ore_cllw_8_variable_v1, ore_cllw_8_variable_v1); - -CREATE OPERATOR = ( - PROCEDURE="ore_cllw_8_variable_v1_eq", - LEFTARG=ore_cllw_8_variable_v1, - RIGHTARG=ore_cllw_8_variable_v1, - NEGATOR = <>, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - -DROP OPERATOR IF EXISTS <> (ore_cllw_8_variable_v1, ore_cllw_8_variable_v1); - -CREATE OPERATOR <> ( - PROCEDURE="ore_cllw_8_variable_v1_neq", - LEFTARG=ore_cllw_8_variable_v1, - RIGHTARG=ore_cllw_8_variable_v1, - NEGATOR = =, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - -DROP OPERATOR IF EXISTS > (ore_cllw_8_variable_v1, ore_cllw_8_variable_v1); - -CREATE OPERATOR > ( - PROCEDURE="ore_cllw_8_v1_gt_lex", - LEFTARG=ore_cllw_8_variable_v1, - RIGHTARG=ore_cllw_8_variable_v1, - NEGATOR = <=, - RESTRICT = scalarltsel, - JOIN = scalarltjoinsel, - HASHES, - MERGES -); - -DROP OPERATOR IF EXISTS < (ore_cllw_8_variable_v1, ore_cllw_8_variable_v1); - -CREATE OPERATOR < ( - PROCEDURE="ore_cllw_8_v1_lt_lex", - LEFTARG=ore_cllw_8_variable_v1, - RIGHTARG=ore_cllw_8_variable_v1, - NEGATOR = >=, - RESTRICT = scalargtsel, - JOIN = scalargtjoinsel, - HASHES, - MERGES -); - -DROP OPERATOR IF EXISTS >= (ore_cllw_8_variable_v1, ore_cllw_8_variable_v1); - -CREATE OPERATOR >= ( - PROCEDURE="ore_cllw_8_v1_gte_lex", - LEFTARG=ore_cllw_8_variable_v1, - RIGHTARG=ore_cllw_8_variable_v1, - NEGATOR = <, - RESTRICT = scalarltsel, - JOIN = scalarltjoinsel, - HASHES, - MERGES -); - -DROP OPERATOR IF EXISTS <= (ore_cllw_8_variable_v1, ore_cllw_8_variable_v1); - -CREATE OPERATOR <= ( - PROCEDURE="ore_cllw_8_v1_lte_lex", - LEFTARG=ore_cllw_8_variable_v1, - RIGHTARG=ore_cllw_8_variable_v1, - NEGATOR = >, - RESTRICT = scalargtsel, - JOIN = scalargtjoinsel, - HASHES, - MERGES -); - - -DROP OPERATOR FAMILY IF EXISTS ore_cllw_8_v1_btree_ops USING btree; - -CREATE OPERATOR FAMILY ore_cllw_8_v1_btree_ops USING btree; - -DROP OPERATOR CLASS IF EXISTS ore_cllw_8_v1_btree_ops USING btree; - -CREATE OPERATOR CLASS ore_cllw_8_v1_btree_ops DEFAULT FOR TYPE ore_cllw_8_v1 USING btree FAMILY ore_cllw_8_v1_btree_ops AS - OPERATOR 1 <, - OPERATOR 2 <=, - OPERATOR 3 =, - OPERATOR 4 >=, - OPERATOR 5 >, - FUNCTION 1 compare_ore_cllw_8_v1(a ore_cllw_8_v1, b ore_cllw_8_v1); - --- Lexical comparison operator class - -DROP OPERATOR FAMILY IF EXISTS ore_cllw_8_v1_variable_btree_ops USING btree; - -CREATE OPERATOR FAMILY ore_cllw_8_v1_variable_btree_ops USING btree; - -DROP OPERATOR CLASS IF EXISTS ore_cllw_8_v1_variable_btree_ops USING btree; - -CREATE OPERATOR CLASS ore_cllw_8_v1_variable_btree_ops DEFAULT FOR TYPE ore_cllw_8_variable_v1 USING btree FAMILY ore_cllw_8_v1_variable_btree_ops AS - OPERATOR 1 <, - OPERATOR 2 <=, - OPERATOR 3 =, - OPERATOR 4 >=, - OPERATOR 5 >, - FUNCTION 1 compare_lex_ore_cllw_8_v1(a ore_cllw_8_variable_v1, b ore_cllw_8_variable_v1); diff --git a/sql/003-ste-vec.sql b/sql/003-ste-vec.sql deleted file mode 100644 index bbe8c473..00000000 --- a/sql/003-ste-vec.sql +++ /dev/null @@ -1,444 +0,0 @@ - ---- ---- SteVec types, functions, and operators ---- - -CREATE TYPE cs_ste_vec_encrypted_term_v1 AS ( - bytes bytea -); - -DROP FUNCTION IF EXISTS compare_ste_vec_encrypted_term_v1(a cs_ste_vec_encrypted_term_v1, b cs_ste_vec_encrypted_term_v1); - -CREATE FUNCTION compare_ste_vec_encrypted_term_v1(a cs_ste_vec_encrypted_term_v1, b cs_ste_vec_encrypted_term_v1) -RETURNS INT AS $$ -DECLARE - header_a INT; - header_b INT; - body_a BYTEA; - body_b BYTEA; -BEGIN - -- `get_byte` is 0-indexed - header_a := get_byte(a.bytes, 0); - header_b := get_byte(b.bytes, 0); - - IF header_a != header_b THEN - RAISE EXCEPTION 'compare_ste_vec_encrypted_term_v1: expected equal header bytes'; - END IF; - - -- `substr` is 1-indexed (yes, `subtr` starts at 1 and `get_byte` starts at 0). - body_a := substr(a.bytes, 2); - body_b := substr(b.bytes, 2); - - CASE header_a - WHEN 0 THEN - RAISE EXCEPTION 'compare_ste_vec_encrypted_term_v1: can not compare MAC terms'; - WHEN 1 THEN - RETURN compare_ore_cllw_8_v1(ROW(body_a)::ore_cllw_8_v1, ROW(body_b)::ore_cllw_8_v1); - WHEN 2 THEN - RETURN compare_lex_ore_cllw_8_v1(ROW(body_a)::ore_cllw_8_variable_v1, ROW(body_b)::ore_cllw_8_variable_v1); - ELSE - RAISE EXCEPTION 'compare_ste_vec_encrypted_term_v1: invalid header for cs_ste_vec_encrypted_term_v1: header "%", body "%', header_a, body_a; - END CASE; -END; -$$ LANGUAGE plpgsql; - -DROP FUNCTION IF EXISTS cs_ste_vec_encrypted_term_eq(a cs_ste_vec_encrypted_term_v1, b cs_ste_vec_encrypted_term_v1); - -CREATE FUNCTION cs_ste_vec_encrypted_term_eq(a cs_ste_vec_encrypted_term_v1, b cs_ste_vec_encrypted_term_v1) RETURNS boolean AS $$ - SELECT __bytea_ct_eq(a.bytes, b.bytes) -$$ LANGUAGE SQL; - -DROP FUNCTION IF EXISTS cs_ste_vec_encrypted_term_neq(a cs_ste_vec_encrypted_term_v1, b cs_ste_vec_encrypted_term_v1); - -CREATE FUNCTION cs_ste_vec_encrypted_term_neq(a cs_ste_vec_encrypted_term_v1, b cs_ste_vec_encrypted_term_v1) RETURNS boolean AS $$ - SELECT not __bytea_ct_eq(a.bytes, b.bytes) -$$ LANGUAGE SQL; - -DROP FUNCTION IF EXISTS cs_ste_vec_encrypted_term_lt(a cs_ste_vec_encrypted_term_v1, b cs_ste_vec_encrypted_term_v1); - -CREATE FUNCTION cs_ste_vec_encrypted_term_lt(a cs_ste_vec_encrypted_term_v1, b cs_ste_vec_encrypted_term_v1) RETURNS boolean AS $$ - SELECT compare_ste_vec_encrypted_term_v1(a, b) = -1 -$$ LANGUAGE SQL; - -DROP FUNCTION IF EXISTS cs_ste_vec_encrypted_term_lte(a cs_ste_vec_encrypted_term_v1, b cs_ste_vec_encrypted_term_v1); - -CREATE FUNCTION cs_ste_vec_encrypted_term_lte(a cs_ste_vec_encrypted_term_v1, b cs_ste_vec_encrypted_term_v1) RETURNS boolean AS $$ - SELECT compare_ste_vec_encrypted_term_v1(a, b) != 1 -$$ LANGUAGE SQL; - -DROP FUNCTION IF EXISTS cs_ste_vec_encrypted_term_gt(a cs_ste_vec_encrypted_term_v1, b cs_ste_vec_encrypted_term_v1); - -CREATE FUNCTION cs_ste_vec_encrypted_term_gt(a cs_ste_vec_encrypted_term_v1, b cs_ste_vec_encrypted_term_v1) RETURNS boolean AS $$ - SELECT compare_ste_vec_encrypted_term_v1(a, b) = 1 -$$ LANGUAGE SQL; - -DROP FUNCTION IF EXISTS cs_ste_vec_encrypted_term_gte(a cs_ste_vec_encrypted_term_v1, b cs_ste_vec_encrypted_term_v1); - -CREATE FUNCTION cs_ste_vec_encrypted_term_gte(a cs_ste_vec_encrypted_term_v1, b cs_ste_vec_encrypted_term_v1) RETURNS boolean AS $$ - SELECT compare_ste_vec_encrypted_term_v1(a, b) != -1 -$$ LANGUAGE SQL; - -DROP OPERATOR IF EXISTS = (cs_ste_vec_encrypted_term_v1, cs_ste_vec_encrypted_term_v1); - -CREATE OPERATOR = ( - PROCEDURE="cs_ste_vec_encrypted_term_eq", - LEFTARG=cs_ste_vec_encrypted_term_v1, - RIGHTARG=cs_ste_vec_encrypted_term_v1, - NEGATOR = <>, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - -DROP OPERATOR IF EXISTS <> (cs_ste_vec_encrypted_term_v1, cs_ste_vec_encrypted_term_v1); - -CREATE OPERATOR <> ( - PROCEDURE="cs_ste_vec_encrypted_term_neq", - LEFTARG=cs_ste_vec_encrypted_term_v1, - RIGHTARG=cs_ste_vec_encrypted_term_v1, - NEGATOR = =, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - -DROP OPERATOR IF EXISTS > (cs_ste_vec_encrypted_term_v1, cs_ste_vec_encrypted_term_v1); - -CREATE OPERATOR > ( - PROCEDURE="cs_ste_vec_encrypted_term_gt", - LEFTARG=cs_ste_vec_encrypted_term_v1, - RIGHTARG=cs_ste_vec_encrypted_term_v1, - NEGATOR = <=, - RESTRICT = scalarltsel, - JOIN = scalarltjoinsel, - HASHES, - MERGES -); - -DROP OPERATOR IF EXISTS < (cs_ste_vec_encrypted_term_v1, cs_ste_vec_encrypted_term_v1); - -CREATE OPERATOR < ( - PROCEDURE="cs_ste_vec_encrypted_term_lt", - LEFTARG=cs_ste_vec_encrypted_term_v1, - RIGHTARG=cs_ste_vec_encrypted_term_v1, - NEGATOR = >=, - RESTRICT = scalargtsel, - JOIN = scalargtjoinsel, - HASHES, - MERGES -); - -DROP OPERATOR IF EXISTS >= (cs_ste_vec_encrypted_term_v1, cs_ste_vec_encrypted_term_v1); - -CREATE OPERATOR >= ( - PROCEDURE="cs_ste_vec_encrypted_term_gte", - LEFTARG=cs_ste_vec_encrypted_term_v1, - RIGHTARG=cs_ste_vec_encrypted_term_v1, - NEGATOR = <, - RESTRICT = scalarltsel, - JOIN = scalarltjoinsel, - HASHES, - MERGES -); - -DROP OPERATOR IF EXISTS <= (cs_ste_vec_encrypted_term_v1, cs_ste_vec_encrypted_term_v1); - -CREATE OPERATOR <= ( - PROCEDURE="cs_ste_vec_encrypted_term_lte", - LEFTARG=cs_ste_vec_encrypted_term_v1, - RIGHTARG=cs_ste_vec_encrypted_term_v1, - NEGATOR = >, - RESTRICT = scalargtsel, - JOIN = scalargtjoinsel, - HASHES, - MERGES -); - -DROP OPERATOR FAMILY IF EXISTS cs_ste_vec_encrypted_term_v1_btree_ops USING btree; - -CREATE OPERATOR FAMILY cs_ste_vec_encrypted_term_v1_btree_ops USING btree; - -DROP OPERATOR CLASS IF EXISTS cs_ste_vec_encrypted_term_v1_btree_ops USING btree; - -CREATE OPERATOR CLASS cs_ste_vec_encrypted_term_v1_btree_ops DEFAULT FOR TYPE cs_ste_vec_encrypted_term_v1 USING btree FAMILY cs_ste_vec_encrypted_term_v1_btree_ops AS - OPERATOR 1 <, - OPERATOR 2 <=, - OPERATOR 3 =, - OPERATOR 4 >=, - OPERATOR 5 >, - FUNCTION 1 compare_ste_vec_encrypted_term_v1(a cs_ste_vec_encrypted_term_v1, b cs_ste_vec_encrypted_term_v1); - -CREATE TYPE cs_ste_vec_v1_entry AS ( - tokenized_selector text, - term cs_ste_vec_encrypted_term_v1, - ciphertext text -); - -CREATE TYPE cs_ste_vec_index_v1 AS ( - entries cs_ste_vec_v1_entry[] -); - -DROP FUNCTION IF EXISTS cs_ste_vec_value_v1(col jsonb, selector jsonb); - --- col: already encrypted payload --- selector: already encrypted payload --- returns a value in the format of our custom jsonb schema that will be decrypted -CREATE FUNCTION cs_ste_vec_value_v1(col jsonb, selector jsonb) -RETURNS jsonb AS $$ -DECLARE - ste_vec_index cs_ste_vec_index_v1; - target_selector text; - found text; - ignored text; - i integer; -BEGIN - ste_vec_index := cs_ste_vec_v1(col); - - IF ste_vec_index IS NULL THEN - RETURN NULL; - END IF; - - target_selector := selector->>'svs'; - - FOR i IN 1..array_length(ste_vec_index.entries, 1) LOOP - -- The ELSE part is to help ensure constant time operation. - -- The result is thrown away. - IF ste_vec_index.entries[i].tokenized_selector = target_selector THEN - found := ste_vec_index.entries[i].ciphertext; - ELSE - ignored := ste_vec_index.entries[i].ciphertext; - END IF; - END LOOP; - - IF found IS NOT NULL THEN - RETURN jsonb_build_object( - 'k', 'ct', - 'c', found, - 'o', NULL, - 'm', NULL, - 'u', NULL, - 'i', col->'i', - 'v', 1 - ); - ELSE - RETURN NULL; - END IF; -END; -$$ LANGUAGE plpgsql; - -DROP FUNCTION IF EXISTS cs_ste_vec_terms_v1(col jsonb, selector jsonb); - -CREATE FUNCTION cs_ste_vec_terms_v1(col jsonb, selector jsonb) -RETURNS cs_ste_vec_encrypted_term_v1[] AS $$ -DECLARE - ste_vec_index cs_ste_vec_index_v1; - target_selector text; - found cs_ste_vec_encrypted_term_v1; - ignored cs_ste_vec_encrypted_term_v1; - i integer; - term_array cs_ste_vec_encrypted_term_v1[]; -BEGIN - ste_vec_index := cs_ste_vec_v1(col); - - IF ste_vec_index IS NULL THEN - RETURN NULL; - END IF; - - target_selector := selector->>'svs'; - - FOR i IN 1..array_length(ste_vec_index.entries, 1) LOOP - -- The ELSE part is to help ensure constant time operation. - -- The result is thrown away. - IF ste_vec_index.entries[i].tokenized_selector = target_selector THEN - found := ste_vec_index.entries[i].term; - term_array := array_append(term_array, found); - ELSE - ignored := ste_vec_index.entries[i].term; - END IF; - END LOOP; - - RETURN term_array; -END; -$$ LANGUAGE plpgsql; - -DROP FUNCTION IF EXISTS cs_ste_vec_term_v1(col jsonb, selector jsonb); - --- col: already encrypted payload --- selector: already encrypted payload --- returns a value that can be used for comparison operations -CREATE OR REPLACE FUNCTION cs_ste_vec_term_v1(col jsonb, selector jsonb) -RETURNS cs_ste_vec_encrypted_term_v1 AS $$ -DECLARE - ste_vec_index cs_ste_vec_index_v1; - target_selector text; - found cs_ste_vec_encrypted_term_v1; - ignored cs_ste_vec_encrypted_term_v1; - i integer; -BEGIN - ste_vec_index := cs_ste_vec_v1(col); - - IF ste_vec_index IS NULL THEN - RETURN NULL; - END IF; - - target_selector := selector->>'svs'; - - FOR i IN 1..array_length(ste_vec_index.entries, 1) LOOP - -- The ELSE part is to help ensure constant time operation. - -- The result is thrown away. - IF ste_vec_index.entries[i].tokenized_selector = target_selector THEN - found := ste_vec_index.entries[i].term; - ELSE - ignored := ste_vec_index.entries[i].term; - END IF; - END LOOP; - - RETURN found; -END; -$$ LANGUAGE plpgsql; - -DROP FUNCTION IF EXISTS cs_ste_vec_term_v1(col jsonb); - -CREATE FUNCTION cs_ste_vec_term_v1(col jsonb) -RETURNS cs_ste_vec_encrypted_term_v1 AS $$ -DECLARE - ste_vec_index cs_ste_vec_index_v1; -BEGIN - ste_vec_index := cs_ste_vec_v1(col); - - IF ste_vec_index IS NULL THEN - RETURN NULL; - END IF; - - RETURN ste_vec_index.entries[1].term; -END; -$$ LANGUAGE plpgsql; - --- Determine if a == b (ignoring ciphertext values) -DROP FUNCTION IF EXISTS cs_ste_vec_v1_entry_eq(a cs_ste_vec_v1_entry, b cs_ste_vec_v1_entry); - -CREATE FUNCTION cs_ste_vec_v1_entry_eq(a cs_ste_vec_v1_entry, b cs_ste_vec_v1_entry) -RETURNS boolean AS $$ -DECLARE - sel_cmp int; - term_cmp int; -BEGIN - -- Constant time comparison - IF a.tokenized_selector = b.tokenized_selector THEN - sel_cmp := 1; - ELSE - sel_cmp := 0; - END IF; - IF a.term = b.term THEN - term_cmp := 1; - ELSE - term_cmp := 0; - END IF; - RETURN (sel_cmp # term_cmp) = 0; -END; -$$ LANGUAGE plpgsql; - --- Determine if a contains b (ignoring ciphertext values) -DROP FUNCTION IF EXISTS ste_vec_v1_logical_contains(a cs_ste_vec_index_v1, b cs_ste_vec_index_v1); - -CREATE FUNCTION ste_vec_v1_logical_contains(a cs_ste_vec_index_v1, b cs_ste_vec_index_v1) -RETURNS boolean AS $$ -DECLARE - result boolean; - intermediate_result boolean; -BEGIN - result := true; - IF array_length(b.entries, 1) IS NULL THEN - RETURN result; - END IF; - FOR i IN 1..array_length(b.entries, 1) LOOP - intermediate_result := cs_ste_vec_v1_entry_array_contains_entry(a.entries, b.entries[i]); - result := result AND intermediate_result; - END LOOP; - RETURN result; -END; -$$ LANGUAGE plpgsql; - --- Determine if a contains b (ignoring ciphertext values) -DROP FUNCTION IF EXISTS cs_ste_vec_v1_entry_array_contains_entry(a cs_ste_vec_v1_entry[], b cs_ste_vec_v1_entry); - -CREATE FUNCTION cs_ste_vec_v1_entry_array_contains_entry(a cs_ste_vec_v1_entry[], b cs_ste_vec_v1_entry) -RETURNS boolean AS $$ -DECLARE - result boolean; - intermediate_result boolean; -BEGIN - IF array_length(a, 1) IS NULL THEN - RETURN false; - END IF; - - result := false; - FOR i IN 1..array_length(a, 1) LOOP - intermediate_result := a[i].tokenized_selector = b.tokenized_selector AND a[i].term = b.term; - result := result OR intermediate_result; - END LOOP; - RETURN result; -END; -$$ LANGUAGE plpgsql; - --- Determine if a is contained by b (ignoring ciphertext values) -DROP FUNCTION IF EXISTS ste_vec_v1_logical_is_contained(a cs_ste_vec_index_v1, b cs_ste_vec_index_v1); - -CREATE FUNCTION ste_vec_v1_logical_is_contained(a cs_ste_vec_index_v1, b cs_ste_vec_index_v1) -RETURNS boolean AS $$ -BEGIN - RETURN ste_vec_v1_logical_contains(b, a); -END; -$$ LANGUAGE plpgsql; - - -DROP FUNCTION IF EXISTS jsonb_to_cs_ste_vec_index_v1(input jsonb); - -CREATE FUNCTION jsonb_to_cs_ste_vec_index_v1(input jsonb) -RETURNS cs_ste_vec_index_v1 AS $$ -DECLARE - vec_entry cs_ste_vec_v1_entry; - entry_array cs_ste_vec_v1_entry[]; - entry_json jsonb; - entry_json_array jsonb[]; - entry_array_length int; - i int; -BEGIN - FOR entry_json IN SELECT * FROM jsonb_array_elements(input) - LOOP - vec_entry := ROW( - entry_json->>0, - ROW(decode(entry_json->>1, 'hex'))::cs_ste_vec_encrypted_term_v1, - entry_json->>2 - )::cs_ste_vec_v1_entry; - entry_array := array_append(entry_array, vec_entry); - END LOOP; - - RETURN ROW(entry_array)::cs_ste_vec_index_v1; -END; -$$ LANGUAGE plpgsql; - -DROP CAST IF EXISTS (jsonb AS cs_ste_vec_index_v1); - -CREATE CAST (jsonb AS cs_ste_vec_index_v1) - WITH FUNCTION jsonb_to_cs_ste_vec_index_v1(jsonb) AS IMPLICIT; - -DROP OPERATOR IF EXISTS @> (cs_ste_vec_index_v1, cs_ste_vec_index_v1); - -CREATE OPERATOR @> ( - PROCEDURE="ste_vec_v1_logical_contains", - LEFTARG=cs_ste_vec_index_v1, - RIGHTARG=cs_ste_vec_index_v1, - COMMUTATOR = <@ -); - -DROP OPERATOR IF EXISTS <@ (cs_ste_vec_index_v1, cs_ste_vec_index_v1); - -CREATE OPERATOR <@ ( - PROCEDURE="ste_vec_v1_logical_is_contained", - LEFTARG=cs_ste_vec_index_v1, - RIGHTARG=cs_ste_vec_index_v1, - COMMUTATOR = @> -); diff --git a/sql/010-core-domain-types.sql b/sql/010-core-domain-types.sql deleted file mode 100644 index 8d9ba783..00000000 --- a/sql/010-core-domain-types.sql +++ /dev/null @@ -1,159 +0,0 @@ -DROP DOMAIN IF EXISTS cs_match_index_v1; -CREATE DOMAIN cs_match_index_v1 AS smallint[]; - -DROP DOMAIN IF EXISTS cs_unique_index_v1; -CREATE DOMAIN cs_unique_index_v1 AS text; - - --- cs_encrypted_v1 is a column type and cannot be dropped if in use -DO $$ -BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'cs_encrypted_v1') THEN - CREATE DOMAIN cs_encrypted_v1 AS JSONB; - END IF; -END -$$; - - --- Should include a kind field -DROP FUNCTION IF EXISTS _cs_encrypted_check_k(jsonb); -CREATE FUNCTION _cs_encrypted_check_k(val jsonb) - RETURNS boolean -AS $$ - BEGIN - IF (val->>'k' = ANY('{ct, sv}')) THEN - RETURN true; - END IF; - RAISE 'Invalid kind (%) in Encrypted column. Kind should be one of {ct, sv}', val; - END; -$$ LANGUAGE plpgsql; - - --- --- CT payload should include a c field --- -DROP FUNCTION IF EXISTS _cs_encrypted_check_k_ct(jsonb); -CREATE FUNCTION _cs_encrypted_check_k_ct(val jsonb) - RETURNS boolean -AS $$ - BEGIN - IF (val->>'k' = 'ct') THEN - IF (val ? 'c') THEN - RETURN true; - END IF; - RAISE 'Encrypted column kind (k) of "ct" missing data field (c): %', val; - END IF; - RETURN true; - END; -$$ LANGUAGE plpgsql; - - --- --- SV payload should include an sv field --- -DROP FUNCTION IF EXISTS _cs_encrypted_check_k_sv(jsonb); -CREATE FUNCTION _cs_encrypted_check_k_sv(val jsonb) - RETURNS boolean -AS $$ - BEGIN - IF (val->>'k' = 'sv') THEN - IF (val ? 'sv') THEN - RETURN true; - END IF; - RAISE 'Encrypted column kind (k) of "sv" missing data field (sv): %', val; - END IF; - RETURN true; - END; -$$ LANGUAGE plpgsql; - - --- Plaintext field should never be present in an encrypted column -DROP FUNCTION IF EXISTS _cs_encrypted_check_p(jsonb); -CREATE FUNCTION _cs_encrypted_check_p(val jsonb) - RETURNS boolean -AS $$ - BEGIN - IF NOT val ? 'p' THEN - RETURN true; - END IF; - RAISE 'Encrypted column includes plaintext (p) field: %', val; - END; -$$ LANGUAGE plpgsql; - --- Should include an ident field -DROP FUNCTION IF EXISTS _cs_encrypted_check_i(jsonb); -CREATE FUNCTION _cs_encrypted_check_i(val jsonb) - RETURNS boolean -AS $$ - BEGIN - IF val ? 'i' THEN - RETURN true; - END IF; - RAISE 'Encrypted column missing ident (i) field: %', val; - END; -$$ LANGUAGE plpgsql; - --- Query field should never be present in an encrypted column -DROP FUNCTION IF EXISTS _cs_encrypted_check_q(jsonb); -CREATE FUNCTION _cs_encrypted_check_q(val jsonb) - RETURNS boolean -AS $$ - BEGIN - IF val ? 'q' THEN - RAISE 'Encrypted column includes query (q) field: %', val; - END IF; - RETURN true; - END; -$$ LANGUAGE plpgsql; - --- Ident field should include table and column -DROP FUNCTION IF EXISTS _cs_encrypted_check_i_ct(jsonb); -CREATE FUNCTION _cs_encrypted_check_i_ct(val jsonb) - RETURNS boolean -AS $$ - BEGIN - IF (val->'i' ?& array['t', 'c']) THEN - RETURN true; - END IF; - RAISE 'Encrypted column ident (i) missing table (t) or column (c) fields: %', val; - END; -$$ LANGUAGE plpgsql; - --- Should include a version field -DROP FUNCTION IF EXISTS _cs_encrypted_check_v(jsonb); -CREATE FUNCTION _cs_encrypted_check_v(val jsonb) - RETURNS boolean -AS $$ - BEGIN - IF (val ? 'v') THEN - RETURN true; - END IF; - RAISE 'Encrypted column missing version (v) field: %', val; - END; -$$ LANGUAGE plpgsql; - - -DROP FUNCTION IF EXISTS cs_check_encrypted_v1(val jsonb); - -CREATE FUNCTION cs_check_encrypted_v1(val jsonb) - RETURNS BOOLEAN -LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -BEGIN ATOMIC - RETURN ( - _cs_encrypted_check_v(val) AND - _cs_encrypted_check_i(val) AND - _cs_encrypted_check_k(val) AND - _cs_encrypted_check_k_ct(val) AND - _cs_encrypted_check_k_sv(val) AND - _cs_encrypted_check_q(val) AND - _cs_encrypted_check_p(val) - ); -END; - -ALTER DOMAIN cs_encrypted_v1 DROP CONSTRAINT IF EXISTS cs_encrypted_v1_check; - -ALTER DOMAIN cs_encrypted_v1 - ADD CONSTRAINT cs_encrypted_v1_check CHECK ( - cs_check_encrypted_v1(VALUE) -); - diff --git a/sql/011-core-functions.sql b/sql/011-core-functions.sql deleted file mode 100644 index 48d36953..00000000 --- a/sql/011-core-functions.sql +++ /dev/null @@ -1,228 +0,0 @@ -DROP FUNCTION IF EXISTS cs_ciphertext_v1_v0_0(val jsonb); - -CREATE FUNCTION cs_ciphertext_v1_v0_0(val jsonb) - RETURNS text - IMMUTABLE STRICT PARALLEL SAFE -AS $$ - BEGIN - IF val ? 'c' THEN - RETURN val->>'c'; - END IF; - RAISE 'Expected a ciphertext (c) value in json: %', val; - END; -$$ LANGUAGE plpgsql; - - -DROP FUNCTION IF EXISTS cs_ciphertext_v1_v0(val jsonb); - -CREATE FUNCTION cs_ciphertext_v1_v0(val jsonb) - RETURNS text - LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -BEGIN ATOMIC - RETURN cs_ciphertext_v1_v0_0(val); -END; - - -DROP FUNCTION IF EXISTS cs_ciphertext_v1(val jsonb); - -CREATE FUNCTION cs_ciphertext_v1(val jsonb) - RETURNS text - LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -BEGIN ATOMIC - RETURN cs_ciphertext_v1_v0_0(val); -END; - - --- extracts match index from an emcrypted column -DROP FUNCTION IF EXISTS cs_match_v1_v0_0(val jsonb); - -CREATE FUNCTION cs_match_v1_v0_0(val jsonb) - RETURNS cs_match_index_v1 - IMMUTABLE STRICT PARALLEL SAFE -AS $$ - BEGIN - IF val ? 'm' THEN - RETURN ARRAY(SELECT jsonb_array_elements(val->'m'))::cs_match_index_v1; - END IF; - RAISE 'Expected a match index (m) value in json: %', val; - END; -$$ LANGUAGE plpgsql; - - -DROP FUNCTION IF EXISTS cs_match_v1_v0(val jsonb); - -CREATE FUNCTION cs_match_v1_v0(val jsonb) - RETURNS cs_match_index_v1 - LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -BEGIN ATOMIC - RETURN cs_match_v1_v0_0(val); -END; - - -DROP FUNCTION IF EXISTS cs_match_v1(val jsonb); - -CREATE FUNCTION cs_match_v1(val jsonb) - RETURNS cs_match_index_v1 - LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -BEGIN ATOMIC - RETURN cs_match_v1_v0_0(val); -END; - - --- extracts unique index from an encrypted column -DROP FUNCTION IF EXISTS cs_unique_v1_v0_0(val jsonb); - -CREATE FUNCTION cs_unique_v1_v0_0(val jsonb) - RETURNS cs_unique_index_v1 - IMMUTABLE STRICT PARALLEL SAFE -AS $$ - BEGIN - IF val ? 'u' THEN - RETURN val->>'u'; - END IF; - RAISE 'Expected a unique index (u) value in json: %', val; - END; -$$ LANGUAGE plpgsql; - - -DROP FUNCTION IF EXISTS cs_unique_v1_v0(val jsonb); - -CREATE FUNCTION cs_unique_v1_v0(val jsonb) - RETURNS cs_unique_index_v1 - LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -BEGIN ATOMIC - RETURN cs_unique_v1_v0_0(val); -END; - - -DROP FUNCTION IF EXISTS cs_unique_v1(val jsonb); - -CREATE FUNCTION cs_unique_v1(val jsonb) - RETURNS cs_unique_index_v1 - LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -BEGIN ATOMIC - RETURN cs_unique_v1_v0_0(val); -END; - --- extracts json ste_vec index from an encrypted column -DROP FUNCTION IF EXISTS cs_ste_vec_v1_v0_0(val jsonb); - -CREATE FUNCTION cs_ste_vec_v1_v0_0(val jsonb) - RETURNS cs_ste_vec_index_v1 - IMMUTABLE STRICT PARALLEL SAFE -AS $$ - BEGIN - IF val ? 'sv' THEN - RETURN (val->'sv')::cs_ste_vec_index_v1; - END IF; - RAISE 'Expected a structured vector index (sv) value in json: %', val; - END; -$$ LANGUAGE plpgsql; - - -DROP FUNCTION IF EXISTS cs_ste_vec_v1_v0(val jsonb); - -CREATE FUNCTION cs_ste_vec_v1_v0(val jsonb) - RETURNS cs_ste_vec_index_v1 - LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -BEGIN ATOMIC - RETURN cs_ste_vec_v1_v0_0(val); -END; - - -DROP FUNCTION IF EXISTS cs_ste_vec_v1(val jsonb); - -CREATE FUNCTION cs_ste_vec_v1(val jsonb) - RETURNS cs_ste_vec_index_v1 - LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -BEGIN ATOMIC - RETURN cs_ste_vec_v1_v0_0(val); -END; - - --- casts text to ore_64_8_v1_term (bytea) -DROP FUNCTION IF EXISTS _cs_text_to_ore_64_8_v1_term_v1_0(t text); - -CREATE FUNCTION _cs_text_to_ore_64_8_v1_term_v1_0(t text) - RETURNS ore_64_8_v1_term - LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -BEGIN ATOMIC - RETURN t::bytea; -END; - --- cast to cleanup ore_64_8_v1 extraction -DROP CAST IF EXISTS (text AS ore_64_8_v1_term); - -CREATE CAST (text AS ore_64_8_v1_term) - WITH FUNCTION _cs_text_to_ore_64_8_v1_term_v1_0(text) AS IMPLICIT; - -DROP FUNCTION IF EXISTS jsonb_array_to_ore_64_8_v1(val jsonb); - --- Casts a jsonb array of hex-encoded strings to the `ore_64_8_v1` composite type. --- In other words, this function takes the ORE index format sent through in the --- EQL payload from Proxy and decodes it as the composite type that we use for --- ORE operations on the Postgres side. -CREATE FUNCTION jsonb_array_to_ore_64_8_v1(val jsonb) -RETURNS ore_64_8_v1 AS $$ -DECLARE - terms_arr ore_64_8_v1_term[]; -BEGIN - IF jsonb_typeof(val) = 'null' THEN - RETURN NULL; - END IF; - - SELECT array_agg(ROW(decode(value::text, 'hex'))::ore_64_8_v1_term) - INTO terms_arr - FROM jsonb_array_elements_text(val) AS value; - - RETURN ROW(terms_arr)::ore_64_8_v1; -END; -$$ LANGUAGE plpgsql; - --- extracts ore index from an encrypted column -DROP FUNCTION IF EXISTS cs_ore_64_8_v1_v0_0(val jsonb); - -CREATE FUNCTION cs_ore_64_8_v1_v0_0(val jsonb) - RETURNS ore_64_8_v1 - IMMUTABLE STRICT PARALLEL SAFE -AS $$ - BEGIN - IF val ? 'o' THEN - RETURN jsonb_array_to_ore_64_8_v1(val->'o'); - END IF; - RAISE 'Expected an ore index (o) value in json: %', val; - END; -$$ LANGUAGE plpgsql; - - -DROP FUNCTION IF EXISTS cs_ore_64_8_v1_v0(val jsonb); - -CREATE FUNCTION cs_ore_64_8_v1_v0(val jsonb) - RETURNS ore_64_8_v1 - LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -BEGIN ATOMIC - RETURN cs_ore_64_8_v1_v0_0(val); -END; - -DROP FUNCTION IF EXISTS cs_ore_64_8_v1(val jsonb); - -CREATE FUNCTION cs_ore_64_8_v1(val jsonb) - RETURNS ore_64_8_v1 - LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -BEGIN ATOMIC - RETURN cs_ore_64_8_v1_v0_0(val); -END; - -DROP FUNCTION IF EXISTS _cs_first_grouped_value(jsonb, jsonb); - -CREATE FUNCTION _cs_first_grouped_value(jsonb, jsonb) -RETURNS jsonb AS $$ - SELECT COALESCE($1, $2); -$$ LANGUAGE sql IMMUTABLE; - -DROP AGGREGATE IF EXISTS cs_grouped_value_v1(jsonb); - -CREATE AGGREGATE cs_grouped_value_v1(jsonb) ( - SFUNC = _cs_first_grouped_value, - STYPE = jsonb -); diff --git a/sql/015-operators-eq.sql b/sql/015-operators-eq.sql deleted file mode 100644 index 5b79543e..00000000 --- a/sql/015-operators-eq.sql +++ /dev/null @@ -1,525 +0,0 @@ --- Operators for unique comparisons of cs_encrypted_v1 types --- --- Support for the following comparisons: --- --- cs_encrypted_v1 = cs_encrypted_v1 --- cs_encrypted_v1 <> cs_encrypted_v1 --- cs_encrypted_v1 = jsonb --- cs_encrypted_v1 <> jsonb --- cs_encrypted_v1 = text --- cs_encrypted_v1 <> text --- - -DROP OPERATOR IF EXISTS = (cs_encrypted_v1, cs_encrypted_v1); -DROP FUNCTION IF EXISTS cs_encrypted_eq_v1(a cs_encrypted_v1, b cs_encrypted_v1); - -CREATE FUNCTION cs_encrypted_eq_v1(a cs_encrypted_v1, b cs_encrypted_v1) - RETURNS boolean - IMMUTABLE STRICT PARALLEL SAFE -AS $$ - DECLARE - u boolean; - o boolean; - BEGIN - BEGIN - u := (SELECT cs_unique_v1(a) = cs_unique_v1(b)); - EXCEPTION WHEN OTHERS THEN - u := false; - END; - - BEGIN - o := (SELECT cs_ore_64_8_v1(a) = cs_ore_64_8_v1(b)); - EXCEPTION WHEN OTHERS THEN - o := false; - END; - - RETURN u OR o; - END; -$$ LANGUAGE plpgsql; - - -CREATE OPERATOR = ( - PROCEDURE="cs_encrypted_eq_v1", - LEFTARG=cs_encrypted_v1, - RIGHTARG=cs_encrypted_v1, - NEGATOR = <>, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - -DROP OPERATOR IF EXISTS = (cs_encrypted_v1, jsonb); -DROP FUNCTION IF EXISTS cs_encrypted_eq_v1(a cs_encrypted_v1, b jsonb); - -CREATE FUNCTION cs_encrypted_eq_v1(a cs_encrypted_v1, b jsonb) - RETURNS boolean - IMMUTABLE STRICT PARALLEL SAFE -AS $$ - DECLARE - u boolean; - o boolean; - BEGIN - BEGIN - u := (SELECT cs_unique_v1(a) = cs_unique_v1(b)); - EXCEPTION WHEN OTHERS THEN - u := false; - END; - - BEGIN - o := (SELECT cs_ore_64_8_v1(a) = cs_ore_64_8_v1(b)); - EXCEPTION WHEN OTHERS THEN - o := false; - END; - - RETURN u OR o; - END; -$$ LANGUAGE plpgsql; - -CREATE OPERATOR = ( - PROCEDURE="cs_encrypted_eq_v1", - LEFTARG=cs_encrypted_v1, - RIGHTARG=jsonb, - NEGATOR = <>, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - - -DROP OPERATOR IF EXISTS = (jsonb, cs_encrypted_v1); -DROP FUNCTION IF EXISTS cs_encrypted_eq_v1(a jsonb, b cs_encrypted_v1); - -CREATE FUNCTION cs_encrypted_eq_v1(a jsonb, b cs_encrypted_v1) - RETURNS boolean - IMMUTABLE STRICT PARALLEL SAFE -AS $$ - DECLARE - u boolean; - o boolean; - BEGIN - BEGIN - u := (SELECT cs_unique_v1(a) = cs_unique_v1(b)); - EXCEPTION WHEN OTHERS THEN - u := false; - END; - - BEGIN - o := (SELECT cs_ore_64_8_v1(a) = cs_ore_64_8_v1(b)); - EXCEPTION WHEN OTHERS THEN - o := false; - END; - - RETURN u OR o; - END; -$$ LANGUAGE plpgsql; - - -CREATE OPERATOR = ( - PROCEDURE="cs_encrypted_eq_v1", - LEFTARG=jsonb, - RIGHTARG=cs_encrypted_v1, - NEGATOR = <>, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - - -DROP OPERATOR IF EXISTS = (cs_encrypted_v1, cs_unique_index_v1); -DROP FUNCTION IF EXISTS cs_encrypted_eq_v1(a cs_encrypted_v1, b cs_unique_index_v1); - -CREATE FUNCTION cs_encrypted_eq_v1(a cs_encrypted_v1, b cs_unique_index_v1) - RETURNS boolean - IMMUTABLE STRICT PARALLEL SAFE -AS $$ - DECLARE - u boolean; - BEGIN - BEGIN - u := (SELECT cs_unique_v1(a) = b); - EXCEPTION WHEN OTHERS THEN - u := false; - END; - - RETURN u; - END; -$$ LANGUAGE plpgsql; - - -CREATE OPERATOR = ( - PROCEDURE="cs_encrypted_eq_v1", - LEFTARG=cs_encrypted_v1, - RIGHTARG=cs_unique_index_v1, - NEGATOR = <>, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - -DROP OPERATOR IF EXISTS = (cs_unique_index_v1, cs_encrypted_v1); -DROP FUNCTION IF EXISTS cs_encrypted_eq_v1(a cs_unique_index_v1, b cs_encrypted_v1); - -CREATE FUNCTION cs_encrypted_eq_v1(a cs_unique_index_v1, b cs_encrypted_v1) - RETURNS boolean - IMMUTABLE STRICT PARALLEL SAFE -AS $$ - DECLARE - u boolean; - BEGIN - BEGIN - u := (SELECT a = cs_unique_v1(b)); - EXCEPTION WHEN OTHERS THEN - u := false; - END; - - RETURN u; - END; -$$ LANGUAGE plpgsql; - - -CREATE OPERATOR =( - PROCEDURE="cs_encrypted_eq_v1", - LEFTARG=cs_unique_index_v1, - RIGHTARG=cs_encrypted_v1, - NEGATOR = <>, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - - - -DROP OPERATOR IF EXISTS = (cs_encrypted_v1, ore_64_8_v1); -DROP FUNCTION IF EXISTS cs_encrypted_eq_v1(a cs_encrypted_v1, b ore_64_8_v1); - -CREATE FUNCTION cs_encrypted_eq_v1(a cs_encrypted_v1, b ore_64_8_v1) - RETURNS boolean - IMMUTABLE STRICT PARALLEL SAFE -AS $$ - DECLARE - o boolean; - BEGIN - - BEGIN - o := (SELECT cs_ore_64_8_v1(a) = b); - EXCEPTION WHEN OTHERS THEN - o := false; - END; - - RETURN o; - END; -$$ LANGUAGE plpgsql; - - -CREATE OPERATOR = ( - PROCEDURE="cs_encrypted_eq_v1", - LEFTARG=cs_encrypted_v1, - RIGHTARG=ore_64_8_v1, - NEGATOR = <>, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - -DROP OPERATOR IF EXISTS = (ore_64_8_v1, cs_encrypted_v1); -DROP FUNCTION IF EXISTS cs_encrypted_eq_v1(a ore_64_8_v1, b cs_encrypted_v1); - -CREATE FUNCTION cs_encrypted_eq_v1(a ore_64_8_v1, b cs_encrypted_v1) - RETURNS boolean - IMMUTABLE STRICT PARALLEL SAFE -AS $$ - DECLARE - o boolean; - BEGIN - - BEGIN - o := (SELECT a = cs_ore_64_8_v1(b)); - EXCEPTION WHEN OTHERS THEN - o := false; - END; - - RETURN o; - END; -$$ LANGUAGE plpgsql; - - -CREATE OPERATOR =( - PROCEDURE="cs_encrypted_eq_v1", - LEFTARG=ore_64_8_v1, - RIGHTARG=cs_encrypted_v1, - NEGATOR = <>, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - - - ---- ------------------------------------------------------------ - -DROP OPERATOR IF EXISTS <> (cs_encrypted_v1, cs_encrypted_v1); -DROP FUNCTION IF EXISTS cs_encrypted_neq_v1(a cs_encrypted_v1, b cs_encrypted_v1); - -CREATE FUNCTION cs_encrypted_neq_v1(a cs_encrypted_v1, b cs_encrypted_v1) - RETURNS boolean - IMMUTABLE STRICT PARALLEL SAFE -AS $$ - DECLARE - u boolean; - o boolean; - BEGIN - BEGIN - u := (SELECT cs_unique_v1(a) <> cs_unique_v1(b)); - EXCEPTION WHEN OTHERS THEN - u := false; - END; - - BEGIN - o := (SELECT cs_ore_64_8_v1(a) <> cs_ore_64_8_v1(b)); - EXCEPTION WHEN OTHERS THEN - o := false; - END; - - RETURN u OR o; - END; -$$ LANGUAGE plpgsql; - -CREATE OPERATOR <> ( - PROCEDURE="cs_encrypted_neq_v1", - LEFTARG=cs_encrypted_v1, - RIGHTARG=cs_encrypted_v1, - NEGATOR = =, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - - -DROP OPERATOR IF EXISTS <> (cs_encrypted_v1, jsonb); -DROP FUNCTION IF EXISTS cs_encrypted_neq_v1(a cs_encrypted_v1, b jsonb); - -CREATE FUNCTION cs_encrypted_neq_v1(a cs_encrypted_v1, b jsonb) - RETURNS boolean - IMMUTABLE STRICT PARALLEL SAFE -AS $$ - DECLARE - u boolean; - o boolean; - BEGIN - BEGIN - u := (SELECT cs_unique_v1(a) <> cs_unique_v1(b)); - EXCEPTION WHEN OTHERS THEN - u := false; - END; - - BEGIN - o := (SELECT cs_ore_64_8_v1(a) <> cs_ore_64_8_v1(b)); - EXCEPTION WHEN OTHERS THEN - o := false; - END; - - RETURN u OR o; - END; -$$ LANGUAGE plpgsql; - -CREATE OPERATOR <> ( - PROCEDURE="cs_encrypted_neq_v1", - LEFTARG=cs_encrypted_v1, - RIGHTARG=jsonb, - NEGATOR = =, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - - -DROP OPERATOR IF EXISTS <> (jsonb, cs_encrypted_v1); -DROP FUNCTION IF EXISTS cs_encrypted_neq_v1(a jsonb, b cs_encrypted_v1); - -CREATE FUNCTION cs_encrypted_neq_v1(a jsonb, b cs_encrypted_v1) - RETURNS boolean - IMMUTABLE STRICT PARALLEL SAFE -AS $$ - DECLARE - u boolean; - o boolean; - BEGIN - BEGIN - u := (SELECT cs_unique_v1(a) <> cs_unique_v1(b)); - EXCEPTION WHEN OTHERS THEN - u := false; - END; - - BEGIN - o := (SELECT cs_ore_64_8_v1(a) <> cs_ore_64_8_v1(b)); - EXCEPTION WHEN OTHERS THEN - o := false; - END; - - RETURN u OR o; - END; -$$ LANGUAGE plpgsql; - -CREATE OPERATOR <> ( - PROCEDURE="cs_encrypted_neq_v1", - LEFTARG=jsonb, - RIGHTARG=cs_encrypted_v1, - NEGATOR = =, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - - -DROP OPERATOR IF EXISTS <> (cs_encrypted_v1, cs_unique_index_v1); -DROP FUNCTION IF EXISTS cs_encrypted_neq_v1(a cs_encrypted_v1, b cs_unique_index_v1); - --- --- Compare the cs_unique_index_v1 or return FALSE --- cs_unique_index_v1 cannot be ore_64_8_v1 --- -CREATE FUNCTION cs_encrypted_neq_v1(a cs_encrypted_v1, b cs_unique_index_v1) - RETURNS boolean - IMMUTABLE STRICT PARALLEL SAFE -AS $$ - DECLARE - u boolean; - BEGIN - BEGIN - u := (SELECT cs_unique_v1(a) <> b); - EXCEPTION WHEN OTHERS THEN - u := false; - END; - - RETURN u; - END; -$$ LANGUAGE plpgsql; - - -CREATE OPERATOR <> ( - PROCEDURE="cs_encrypted_neq_v1", - LEFTARG=cs_encrypted_v1, - RIGHTARG=cs_unique_index_v1, - NEGATOR = =, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - - --- --- Compare the cs_unique_index_v1 or return FALSE --- cs_unique_index_v1 cannot be ore_64_8_v1 --- -DROP OPERATOR IF EXISTS <> (cs_unique_index_v1, cs_encrypted_v1); -DROP FUNCTION IF EXISTS cs_encrypted_neq_v1(a cs_unique_index_v1, b cs_encrypted_v1); - -CREATE FUNCTION cs_encrypted_neq_v1(a cs_unique_index_v1, b cs_encrypted_v1) - RETURNS boolean - IMMUTABLE STRICT PARALLEL SAFE -AS $$ - DECLARE - u boolean; - BEGIN - BEGIN - u := (SELECT a <> cs_unique_v1(b)); - EXCEPTION WHEN OTHERS THEN - u := false; - END; - - RETURN u; - END; -$$ LANGUAGE plpgsql; - - -CREATE OPERATOR <> ( - PROCEDURE="cs_encrypted_neq_v1", - LEFTARG=cs_unique_index_v1, - RIGHTARG=cs_encrypted_v1, - NEGATOR = =, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - - - - -DROP OPERATOR IF EXISTS <> (cs_encrypted_v1, ore_64_8_v1); -DROP FUNCTION IF EXISTS cs_encrypted_neq_v1(a cs_encrypted_v1, b ore_64_8_v1); - -CREATE FUNCTION cs_encrypted_neq_v1(a cs_encrypted_v1, b ore_64_8_v1) - RETURNS boolean - IMMUTABLE STRICT PARALLEL SAFE -AS $$ - DECLARE - o boolean; - BEGIN - BEGIN - o := (SELECT cs_ore_64_8_v1(a) <> b); - EXCEPTION WHEN OTHERS THEN - o := false; - END; - - RETURN o; - END; -$$ LANGUAGE plpgsql; - -CREATE OPERATOR <> ( - PROCEDURE="cs_encrypted_neq_v1", - LEFTARG=cs_encrypted_v1, - RIGHTARG=ore_64_8_v1, - NEGATOR = =, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - - -DROP OPERATOR IF EXISTS <> (ore_64_8_v1, cs_encrypted_v1); -DROP FUNCTION IF EXISTS cs_encrypted_neq_v1(a ore_64_8_v1, b cs_encrypted_v1); - -CREATE FUNCTION cs_encrypted_neq_v1(a ore_64_8_v1, b cs_encrypted_v1) - RETURNS boolean - IMMUTABLE STRICT PARALLEL SAFE -AS $$ - DECLARE - o boolean; - BEGIN - - BEGIN - o := (SELECT a <> cs_ore_64_8_v1(b)); - EXCEPTION WHEN OTHERS THEN - o := false; - END; - - RETURN o; - END; -$$ LANGUAGE plpgsql; - -CREATE OPERATOR <> ( - PROCEDURE="cs_encrypted_neq_v1", - LEFTARG=ore_64_8_v1, - RIGHTARG=cs_encrypted_v1, - NEGATOR = =, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - - - diff --git a/sql/016-operators-match.sql b/sql/016-operators-match.sql deleted file mode 100644 index 4c3b8b50..00000000 --- a/sql/016-operators-match.sql +++ /dev/null @@ -1,187 +0,0 @@ --- Operators for match comparisons of cs_encrypted_v1 types --- --- Support for the following comparisons: --- --- cs_encrypted_v1 ~~ cs_encrypted_v1 --- cs_encrypted_v1 ~~ jsonb --- cs_encrypted_v1 ~~ cs_match_index_v1 --- - -DROP OPERATOR IF EXISTS ~~ (cs_encrypted_v1, cs_encrypted_v1); -DROP FUNCTION IF EXISTS cs_encrypted_match_v1(a cs_encrypted_v1, b cs_encrypted_v1); - -CREATE FUNCTION cs_encrypted_match_v1(a cs_encrypted_v1, b cs_encrypted_v1) -RETURNS boolean AS $$ - SELECT cs_match_v1(a) @> cs_match_v1(b); -$$ LANGUAGE SQL; - -CREATE OPERATOR ~~( - PROCEDURE="cs_encrypted_match_v1", - LEFTARG=cs_encrypted_v1, - RIGHTARG=cs_encrypted_v1, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - -CREATE OPERATOR ~~*( - PROCEDURE="cs_encrypted_match_v1", - LEFTARG=cs_encrypted_v1, - RIGHTARG=cs_encrypted_v1, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - - -DROP OPERATOR IF EXISTS ~~ (cs_encrypted_v1, cs_match_index_v1); -DROP FUNCTION IF EXISTS cs_encrypted_match_v1(a cs_encrypted_v1, b cs_match_index_v1); - -CREATE FUNCTION cs_encrypted_match_v1(a cs_encrypted_v1, b cs_match_index_v1) -RETURNS boolean AS $$ - SELECT cs_match_v1(a) @> b; -$$ LANGUAGE SQL; - -CREATE OPERATOR ~~( - PROCEDURE="cs_encrypted_match_v1", - LEFTARG=cs_encrypted_v1, - RIGHTARG=cs_match_index_v1, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - -CREATE OPERATOR ~~*( - PROCEDURE="cs_encrypted_match_v1", - LEFTARG=cs_encrypted_v1, - RIGHTARG=cs_match_index_v1, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - - - -DROP OPERATOR IF EXISTS ~~ (cs_match_index_v1, cs_encrypted_v1); -DROP FUNCTION IF EXISTS cs_encrypted_match_v1(a cs_match_index_v1, b cs_encrypted_v1); - -CREATE FUNCTION cs_encrypted_match_v1(a cs_match_index_v1, b cs_encrypted_v1) -RETURNS boolean AS $$ - SELECT a @> cs_match_v1(b); -$$ LANGUAGE SQL; - -CREATE OPERATOR ~~( - PROCEDURE="cs_encrypted_match_v1", - LEFTARG=cs_match_index_v1, - RIGHTARG=cs_encrypted_v1, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - -CREATE OPERATOR ~~*( - PROCEDURE="cs_encrypted_match_v1", - LEFTARG=cs_match_index_v1, - RIGHTARG=cs_encrypted_v1, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - - -DROP OPERATOR IF EXISTS ~~ (cs_match_index_v1, cs_match_index_v1); -DROP FUNCTION IF EXISTS cs_encrypted_match_v1(a cs_match_index_v1, b cs_match_index_v1); - -CREATE FUNCTION cs_encrypted_match_v1(a cs_match_index_v1, b cs_match_index_v1) -RETURNS boolean AS $$ - SELECT a @> b; -$$ LANGUAGE SQL; - -CREATE OPERATOR ~~( - PROCEDURE="cs_encrypted_match_v1", - LEFTARG=cs_match_index_v1, - RIGHTARG=cs_match_index_v1, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - -CREATE OPERATOR ~~*( - PROCEDURE="cs_encrypted_match_v1", - LEFTARG=cs_match_index_v1, - RIGHTARG=cs_match_index_v1, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - - -DROP OPERATOR IF EXISTS ~~ (cs_encrypted_v1, jsonb); -DROP FUNCTION IF EXISTS cs_encrypted_match_v1(a cs_encrypted_v1, b jsonb); - -CREATE FUNCTION cs_encrypted_match_v1(a cs_encrypted_v1, b jsonb) -RETURNS boolean AS $$ - SELECT cs_match_v1(a) @> cs_match_v1(b); -$$ LANGUAGE SQL; - -CREATE OPERATOR ~~( - PROCEDURE="cs_encrypted_match_v1", - LEFTARG=cs_encrypted_v1, - RIGHTARG=jsonb, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - -CREATE OPERATOR ~~*( - PROCEDURE="cs_encrypted_match_v1", - LEFTARG=cs_encrypted_v1, - RIGHTARG=jsonb, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - - - -DROP OPERATOR IF EXISTS ~~ (jsonb, cs_encrypted_v1); -DROP FUNCTION IF EXISTS cs_encrypted_match_v1(a jsonb, b cs_encrypted_v1); - -CREATE FUNCTION cs_encrypted_match_v1(a jsonb, b cs_encrypted_v1) -RETURNS boolean AS $$ - SELECT cs_match_v1(a) @> cs_match_v1(b); -$$ LANGUAGE SQL; - -CREATE OPERATOR ~~( - PROCEDURE="cs_encrypted_match_v1", - LEFTARG=jsonb, - RIGHTARG=cs_encrypted_v1, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - -CREATE OPERATOR ~~*( - PROCEDURE="cs_encrypted_match_v1", - LEFTARG=jsonb, - RIGHTARG=cs_encrypted_v1, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - - --- ----------------------------------------------------------------------------- - diff --git a/sql/017-operators-ore.sql b/sql/017-operators-ore.sql deleted file mode 100644 index 15f306ab..00000000 --- a/sql/017-operators-ore.sql +++ /dev/null @@ -1,449 +0,0 @@ --- Operators for match comparisons of cs_encrypted_v1 types --- --- Support for the following comparisons: --- --- cs_encrypted_v1 > >= < <= cs_encrypted_v1 --- cs_encrypted_v1 > jsonb --- cs_encrypted_v1 > ore_64_8_v1 --- - -DROP OPERATOR IF EXISTS > (cs_encrypted_v1, cs_encrypted_v1); -DROP FUNCTION IF EXISTS cs_encrypted_ore_64_8_v1_gt_v1(a cs_encrypted_v1, b cs_encrypted_v1); - -CREATE FUNCTION cs_encrypted_ore_64_8_v1_gt_v1(a cs_encrypted_v1, b cs_encrypted_v1) -RETURNS boolean AS $$ - SELECT cs_ore_64_8_v1(a) > cs_ore_64_8_v1(b); -$$ LANGUAGE SQL; - -CREATE OPERATOR >( - PROCEDURE="cs_encrypted_ore_64_8_v1_gt_v1", - LEFTARG=cs_encrypted_v1, - RIGHTARG=cs_encrypted_v1, - COMMUTATOR = <, - NEGATOR = <=, - RESTRICT = scalargtsel, - JOIN = scalargtjoinsel -); - - -DROP OPERATOR IF EXISTS > (cs_encrypted_v1, jsonb); -DROP FUNCTION IF EXISTS cs_encrypted_ore_64_8_v1_gt_v1(a cs_encrypted_v1, b jsonb); - -CREATE FUNCTION cs_encrypted_ore_64_8_v1_gt_v1(a cs_encrypted_v1, b jsonb) -RETURNS boolean AS $$ - SELECT cs_ore_64_8_v1(a) > cs_ore_64_8_v1(b); -$$ LANGUAGE SQL; - -CREATE OPERATOR >( - PROCEDURE="cs_encrypted_ore_64_8_v1_gt_v1", - LEFTARG=cs_encrypted_v1, - RIGHTARG=jsonb, - COMMUTATOR = <, - NEGATOR = <=, - RESTRICT = scalargtsel, - JOIN = scalargtjoinsel -); - - -DROP OPERATOR IF EXISTS > (cs_encrypted_v1, ore_64_8_v1); -DROP FUNCTION IF EXISTS cs_encrypted_ore_64_8_v1_gt_v1(a cs_encrypted_v1, b ore_64_8_v1); - -CREATE FUNCTION cs_encrypted_ore_64_8_v1_gt_v1(a cs_encrypted_v1, b ore_64_8_v1) -RETURNS boolean AS $$ - SELECT cs_ore_64_8_v1(a) > b; -$$ LANGUAGE SQL; - -CREATE OPERATOR >( - PROCEDURE="cs_encrypted_ore_64_8_v1_gt_v1", - LEFTARG=cs_encrypted_v1, - RIGHTARG=ore_64_8_v1, - COMMUTATOR = <, - NEGATOR = <=, - RESTRICT = scalargtsel, - JOIN = scalargtjoinsel -); - - - -DROP OPERATOR IF EXISTS > (jsonb, cs_encrypted_v1); -DROP FUNCTION IF EXISTS cs_encrypted_ore_64_8_v1_gt_v1(a jsonb, b cs_encrypted_v1); - -CREATE FUNCTION cs_encrypted_ore_64_8_v1_gt_v1(a jsonb, b cs_encrypted_v1) -RETURNS boolean AS $$ - SELECT cs_ore_64_8_v1(a) > cs_ore_64_8_v1(b); -$$ LANGUAGE SQL; - -CREATE OPERATOR >( - PROCEDURE="cs_encrypted_ore_64_8_v1_gt_v1", - LEFTARG=jsonb, - RIGHTARG=cs_encrypted_v1, - COMMUTATOR = <, - NEGATOR = <=, - RESTRICT = scalargtsel, - JOIN = scalargtjoinsel -); - - -DROP OPERATOR IF EXISTS > (ore_64_8_v1, cs_encrypted_v1); -DROP FUNCTION IF EXISTS cs_encrypted_ore_64_8_v1_gt_v1(a ore_64_8_v1, b cs_encrypted_v1); - -CREATE FUNCTION cs_encrypted_ore_64_8_v1_gt_v1(a ore_64_8_v1, b cs_encrypted_v1) -RETURNS boolean AS $$ - SELECT a > cs_ore_64_8_v1(b); -$$ LANGUAGE SQL; - -CREATE OPERATOR >( - PROCEDURE="cs_encrypted_ore_64_8_v1_gt_v1", - LEFTARG=ore_64_8_v1, - RIGHTARG=cs_encrypted_v1, - COMMUTATOR = <, - NEGATOR = <=, - RESTRICT = scalargtsel, - JOIN = scalargtjoinsel -); - - ------------------------------------------------------------------------------------------ --- LT - - -DROP OPERATOR IF EXISTS < (cs_encrypted_v1, cs_encrypted_v1); -DROP FUNCTION IF EXISTS cs_encrypted_ore_64_8_v1_lt_v1(a cs_encrypted_v1, b cs_encrypted_v1); - -CREATE FUNCTION cs_encrypted_ore_64_8_v1_lt_v1(a cs_encrypted_v1, b cs_encrypted_v1) -RETURNS boolean AS $$ - SELECT cs_ore_64_8_v1(a) < cs_ore_64_8_v1(b); -$$ LANGUAGE SQL; - -CREATE OPERATOR <( - PROCEDURE="cs_encrypted_ore_64_8_v1_lt_v1", - LEFTARG=cs_encrypted_v1, - RIGHTARG=cs_encrypted_v1, - COMMUTATOR = >, - NEGATOR = >=, - RESTRICT = scalarltsel, - JOIN = scalarltjoinsel -); - - -DROP OPERATOR IF EXISTS < (cs_encrypted_v1, jsonb); -DROP FUNCTION IF EXISTS cs_encrypted_ore_64_8_v1_lt_v1(a cs_encrypted_v1, b jsonb); - -CREATE FUNCTION cs_encrypted_ore_64_8_v1_lt_v1(a cs_encrypted_v1, b jsonb) -RETURNS boolean AS $$ - SELECT cs_ore_64_8_v1(a) < cs_ore_64_8_v1(b); -$$ LANGUAGE SQL; - -CREATE OPERATOR <( - PROCEDURE="cs_encrypted_ore_64_8_v1_lt_v1", - LEFTARG=cs_encrypted_v1, - RIGHTARG=jsonb, - COMMUTATOR = >, - NEGATOR = >=, - RESTRICT = scalarltsel, - JOIN = scalarltjoinsel -); - - -DROP OPERATOR IF EXISTS < (jsonb, cs_encrypted_v1); -DROP FUNCTION IF EXISTS cs_encrypted_ore_64_8_v1_lt_v1(a jsonb, b cs_encrypted_v1); - -CREATE FUNCTION cs_encrypted_ore_64_8_v1_lt_v1(a jsonb, b cs_encrypted_v1) -RETURNS boolean AS $$ - SELECT cs_ore_64_8_v1(a) < cs_ore_64_8_v1(b); -$$ LANGUAGE SQL; - -CREATE OPERATOR <( - PROCEDURE="cs_encrypted_ore_64_8_v1_lt_v1", - LEFTARG=jsonb, - RIGHTARG=cs_encrypted_v1, - COMMUTATOR = >, - NEGATOR = >=, - RESTRICT = scalarltsel, - JOIN = scalarltjoinsel -); - - - -DROP OPERATOR IF EXISTS <(cs_encrypted_v1, ore_64_8_v1); -DROP FUNCTION IF EXISTS cs_encrypted_ore_64_8_v1_lt_v1(a cs_encrypted_v1, b ore_64_8_v1); - -CREATE FUNCTION cs_encrypted_ore_64_8_v1_lt_v1(a cs_encrypted_v1, b ore_64_8_v1) -RETURNS boolean AS $$ - SELECT cs_ore_64_8_v1(a) < b; -$$ LANGUAGE SQL; - -CREATE OPERATOR <( - PROCEDURE="cs_encrypted_ore_64_8_v1_lt_v1", - LEFTARG=cs_encrypted_v1, - RIGHTARG=ore_64_8_v1, - COMMUTATOR = >, - NEGATOR = >=, - RESTRICT = scalarltsel, - JOIN = scalarltjoinsel -); - - -DROP OPERATOR IF EXISTS <(ore_64_8_v1, cs_encrypted_v1); -DROP FUNCTION IF EXISTS cs_encrypted_ore_64_8_v1_lt_v1(a ore_64_8_v1, b cs_encrypted_v1); - -CREATE FUNCTION cs_encrypted_ore_64_8_v1_lt_v1(a ore_64_8_v1, b cs_encrypted_v1) -RETURNS boolean AS $$ - SELECT a < cs_ore_64_8_v1(b); -$$ LANGUAGE SQL; - -CREATE OPERATOR <( - PROCEDURE="cs_encrypted_ore_64_8_v1_lt_v1", - LEFTARG=ore_64_8_v1, - RIGHTARG=cs_encrypted_v1, - COMMUTATOR = >, - NEGATOR = >=, - RESTRICT = scalarltsel, - JOIN = scalarltjoinsel -); - - ------------------------------------------------------------------------------------------ - - -DROP OPERATOR IF EXISTS >=(cs_encrypted_v1, cs_encrypted_v1); -DROP FUNCTION IF EXISTS cs_encrypted_ore_64_8_v1_gte_v1(a cs_encrypted_v1, b cs_encrypted_v1); - -CREATE FUNCTION cs_encrypted_ore_64_8_v1_gte_v1(a cs_encrypted_v1, b cs_encrypted_v1) -RETURNS boolean AS $$ - SELECT cs_ore_64_8_v1(a) >= cs_ore_64_8_v1(b); -$$ LANGUAGE SQL; - -CREATE OPERATOR >=( - PROCEDURE="cs_encrypted_ore_64_8_v1_gte_v1", - LEFTARG=cs_encrypted_v1, - RIGHTARG=cs_encrypted_v1, - COMMUTATOR = <=, - NEGATOR = <, - RESTRICT = scalarlesel, - JOIN = scalarlejoinsel -); - - -DROP OPERATOR IF EXISTS >= (cs_encrypted_v1, jsonb); -DROP FUNCTION IF EXISTS cs_encrypted_ore_64_8_v1_gte_v1(a cs_encrypted_v1, b jsonb); - -CREATE FUNCTION cs_encrypted_ore_64_8_v1_gte_v1(a cs_encrypted_v1, b jsonb) -RETURNS boolean AS $$ - SELECT cs_ore_64_8_v1(a) >= cs_ore_64_8_v1(b); -$$ LANGUAGE SQL; - -CREATE OPERATOR >=( - PROCEDURE="cs_encrypted_ore_64_8_v1_gte_v1", - LEFTARG=cs_encrypted_v1, - RIGHTARG=jsonb, - COMMUTATOR = <=, - NEGATOR = <, - RESTRICT = scalarlesel, - JOIN = scalarlejoinsel -); - - -DROP OPERATOR IF EXISTS >= (cs_encrypted_v1, ore_64_8_v1); -DROP FUNCTION IF EXISTS cs_encrypted_ore_64_8_v1_gte_v1(a cs_encrypted_v1, b ore_64_8_v1); - -CREATE FUNCTION cs_encrypted_ore_64_8_v1_gte_v1(a cs_encrypted_v1, b ore_64_8_v1) -RETURNS boolean AS $$ - SELECT cs_ore_64_8_v1(a) >= b; -$$ LANGUAGE SQL; - -CREATE OPERATOR >=( - PROCEDURE="cs_encrypted_ore_64_8_v1_gte_v1", - LEFTARG=cs_encrypted_v1, - RIGHTARG=ore_64_8_v1, - COMMUTATOR = <=, - NEGATOR = <, - RESTRICT = scalarlesel, - JOIN = scalarlejoinsel -); - - -DROP OPERATOR IF EXISTS >= (jsonb, cs_encrypted_v1); -DROP FUNCTION IF EXISTS cs_encrypted_ore_64_8_v1_gte_v1(a jsonb, b cs_encrypted_v1); - -CREATE FUNCTION cs_encrypted_ore_64_8_v1_gte_v1(a jsonb, b cs_encrypted_v1) -RETURNS boolean AS $$ - SELECT cs_ore_64_8_v1(a) >= cs_ore_64_8_v1(b); -$$ LANGUAGE SQL; - -CREATE OPERATOR >=( - PROCEDURE="cs_encrypted_ore_64_8_v1_gte_v1", - LEFTARG=jsonb, - RIGHTARG=cs_encrypted_v1, - COMMUTATOR = <=, - NEGATOR = <, - RESTRICT = scalarlesel, - JOIN = scalarlejoinsel -); - - -DROP OPERATOR IF EXISTS >=(ore_64_8_v1, cs_encrypted_v1); -DROP FUNCTION IF EXISTS cs_encrypted_ore_64_8_v1_gte_v1(a ore_64_8_v1, b cs_encrypted_v1); - -CREATE FUNCTION cs_encrypted_ore_64_8_v1_gte_v1(a ore_64_8_v1, b cs_encrypted_v1) -RETURNS boolean AS $$ - SELECT a >= cs_ore_64_8_v1(b); -$$ LANGUAGE SQL; - -CREATE OPERATOR >=( - PROCEDURE="cs_encrypted_ore_64_8_v1_gte_v1", - LEFTARG=ore_64_8_v1, - RIGHTARG=cs_encrypted_v1, - COMMUTATOR = <=, - NEGATOR = <, - RESTRICT = scalarlesel, - JOIN = scalarlejoinsel -); - - ------------------------------------------------------------------------------------------ - - -DROP OPERATOR IF EXISTS <= (cs_encrypted_v1, cs_encrypted_v1); -DROP FUNCTION IF EXISTS cs_encrypted_ore_64_8_v1_lte_v1(a cs_encrypted_v1, b cs_encrypted_v1); - -CREATE FUNCTION cs_encrypted_ore_64_8_v1_lte_v1(a cs_encrypted_v1, b cs_encrypted_v1) -RETURNS boolean AS $$ - SELECT cs_ore_64_8_v1(a) <= cs_ore_64_8_v1(b); -$$ LANGUAGE SQL; - -CREATE OPERATOR <=( - PROCEDURE="cs_encrypted_ore_64_8_v1_lte_v1", - LEFTARG=cs_encrypted_v1, - RIGHTARG=cs_encrypted_v1, - COMMUTATOR = >=, - NEGATOR = >, - RESTRICT = scalarlesel, - JOIN = scalarlejoinsel -); - - -DROP OPERATOR IF EXISTS <= (cs_encrypted_v1, jsonb); -DROP FUNCTION IF EXISTS cs_encrypted_ore_64_8_v1_lte_v1(a cs_encrypted_v1, b jsonb); - -CREATE FUNCTION cs_encrypted_ore_64_8_v1_lte_v1(a cs_encrypted_v1, b jsonb) -RETURNS boolean AS $$ - SELECT cs_ore_64_8_v1(a) <= cs_ore_64_8_v1(b); -$$ LANGUAGE SQL; - -CREATE OPERATOR <=( - PROCEDURE="cs_encrypted_ore_64_8_v1_lte_v1", - LEFTARG=cs_encrypted_v1, - RIGHTARG=jsonb, - COMMUTATOR = >=, - NEGATOR = >, - RESTRICT = scalarlesel, - JOIN = scalarlejoinsel -); - - -DROP OPERATOR IF EXISTS <= (jsonb, cs_encrypted_v1); -DROP FUNCTION IF EXISTS cs_encrypted_ore_64_8_v1_lte_v1(a jsonb, b cs_encrypted_v1); - -CREATE FUNCTION cs_encrypted_ore_64_8_v1_lte_v1(a jsonb, b cs_encrypted_v1) -RETURNS boolean AS $$ - SELECT cs_ore_64_8_v1(a) <= cs_ore_64_8_v1(b); -$$ LANGUAGE SQL; - -CREATE OPERATOR <=( - PROCEDURE="cs_encrypted_ore_64_8_v1_lte_v1", - LEFTARG=jsonb, - RIGHTARG=cs_encrypted_v1, - COMMUTATOR = >=, - NEGATOR = >, - RESTRICT = scalarlesel, - JOIN = scalarlejoinsel -); - - -DROP OPERATOR IF EXISTS <= (cs_encrypted_v1, ore_64_8_v1); -DROP FUNCTION IF EXISTS cs_encrypted_ore_64_8_v1_lte_v1(a cs_encrypted_v1, b ore_64_8_v1); - -CREATE FUNCTION cs_encrypted_ore_64_8_v1_lte_v1(a cs_encrypted_v1, b ore_64_8_v1) -RETURNS boolean AS $$ - SELECT cs_ore_64_8_v1(a) <= b; -$$ LANGUAGE SQL; - -CREATE OPERATOR <=( - PROCEDURE="cs_encrypted_ore_64_8_v1_lte_v1", - LEFTARG=cs_encrypted_v1, - RIGHTARG=ore_64_8_v1, - COMMUTATOR = >=, - NEGATOR = >, - RESTRICT = scalarlesel, - JOIN = scalarlejoinsel -); - - -DROP OPERATOR IF EXISTS <= (ore_64_8_v1, cs_encrypted_v1); -DROP FUNCTION IF EXISTS cs_encrypted_ore_64_8_v1_lte_v1(a ore_64_8_v1, b cs_encrypted_v1); - -CREATE FUNCTION cs_encrypted_ore_64_8_v1_lte_v1(a ore_64_8_v1, b cs_encrypted_v1) -RETURNS boolean AS $$ - SELECT a <= cs_ore_64_8_v1(b); -$$ LANGUAGE SQL; - -CREATE OPERATOR <=( - PROCEDURE="cs_encrypted_ore_64_8_v1_lte_v1", - LEFTARG=ore_64_8_v1, - RIGHTARG=cs_encrypted_v1, - COMMUTATOR = >=, - NEGATOR = >, - RESTRICT = scalarlesel, - JOIN = scalarlejoinsel -); - - ------------------------------------------------------------------------------------------ - - -DROP FUNCTION IF EXISTS cs_encrypted_ore_64_8_compare(a cs_encrypted_v1, b cs_encrypted_v1); - -CREATE FUNCTION cs_encrypted_ore_64_8_compare(a cs_encrypted_v1, b cs_encrypted_v1) - RETURNS integer AS $$ - BEGIN - RETURN compare_ore_64_8_v1(cs_ore_64_8_v1(a), cs_ore_64_8_v1(b)); - END; -$$ LANGUAGE plpgsql; - -DROP FUNCTION IF EXISTS cs_encrypted_ore_64_8_compare(a cs_encrypted_v1, b jsonb); - -CREATE FUNCTION cs_encrypted_ore_64_8_compare(a cs_encrypted_v1, b jsonb) - RETURNS integer AS $$ - BEGIN - RETURN compare_ore_64_8_v1(cs_ore_64_8_v1(a), cs_ore_64_8_v1(b)); - END; -$$ LANGUAGE plpgsql; - - -CREATE FUNCTION cs_encrypted_ore_64_8_compare(a cs_encrypted_v1, b jsonb) - RETURNS integer AS $$ - BEGIN - RETURN compare_ore_64_8_v1(cs_ore_64_8_v1(a), cs_ore_64_8_v1(b)); - END; -$$ LANGUAGE plpgsql; - ------------------------------------------------------------------------------------------ - - -DROP OPERATOR FAMILY IF EXISTS cs_encrypted_ore_64_8_v1_btree_ops_v1 USING btree; - -CREATE OPERATOR FAMILY cs_encrypted_ore_64_8_v1_btree_ops_v1 USING btree; - - -DROP OPERATOR CLASS IF EXISTS ore_64_8_v1_btree_ops USING btree; - -CREATE OPERATOR CLASS cs_encrypted_ore_64_8_v1_btree_ops_v1 DEFAULT -FOR TYPE cs_encrypted_v1 USING btree - FAMILY cs_encrypted_ore_64_8_v1_btree_ops_v1 AS - OPERATOR 1 <, - OPERATOR 2 <=, - OPERATOR 3 =, - OPERATOR 4 >=, - OPERATOR 5 >, - FUNCTION 1 cs_encrypted_ore_64_8_compare(a cs_encrypted_v1, b cs_encrypted_v1); diff --git a/sql/020-config-schema.sql b/sql/020-config-schema.sql deleted file mode 100644 index a54b3dc2..00000000 --- a/sql/020-config-schema.sql +++ /dev/null @@ -1,142 +0,0 @@ --- --- Configuration Schema --- --- Defines core config state and storage types --- Creates the cs_configuration_v1 table with constraint and unique indexes --- --- - - --- --- cs_configuration_data_v1 is a jsonb column that stores the actuak configuration --- --- For some reason CREATE DFOMAIN and CREATE TYPE do not support IF NOT EXISTS --- Types cannot be dropped if used by a table, and we never drop the configuration table --- DOMAIN constraints are added separately and not tied to DOMAIN creation --- -DO $$ - BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'cs_configuration_data_v1') THEN - CREATE DOMAIN cs_configuration_data_v1 AS JSONB; - END IF; - END -$$; - --- --- cs_configuration_state_v1 is an ENUM that defines the valid configuration states --- -DO $$ - BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'cs_configuration_state_v1') THEN - CREATE TYPE cs_configuration_state_v1 AS ENUM ('active', 'inactive', 'encrypting', 'pending'); - END IF; - END -$$; - - - --- --- Extracts index keys/names from configuration json --- --- Used by the _cs_config_check_indexes as part of the cs_configuration_data_v1_check constraint --- -DROP FUNCTION IF EXISTS _cs_extract_indexes(jsonb); -CREATE FUNCTION _cs_extract_indexes(val jsonb) - RETURNS SETOF text - LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE -BEGIN ATOMIC - SELECT jsonb_object_keys(jsonb_path_query(val,'$.tables.*.*.indexes')); -END; - --- --- _cs_check_config_indexes returns true if the table configuration only includes valid index types --- --- Used by the cs_configuration_data_v1_check constraint --- -DROP FUNCTION IF EXISTS _cs_config_check_indexes(jsonb); -CREATE FUNCTION _cs_config_check_indexes(val jsonb) - RETURNS BOOLEAN -AS $$ - BEGIN - IF (SELECT EXISTS (SELECT _cs_extract_indexes(val))) THEN - IF (SELECT bool_and(index = ANY('{match, ore, unique, ste_vec}')) FROM _cs_extract_indexes(val) AS index) THEN - RETURN true; - END IF; - RAISE 'Configuration has an invalid index (%). Index should be one of {match, ore, unique, ste_vec}', val; - END IF; - RETURN true; - END; -$$ LANGUAGE plpgsql; - - -DROP FUNCTION IF EXISTS _cs_config_check_cast(jsonb); - -CREATE FUNCTION _cs_config_check_cast(val jsonb) - RETURNS BOOLEAN -AS $$ - BEGIN - IF EXISTS (SELECT jsonb_array_elements_text(jsonb_path_query_array(val, '$.tables.*.*.cast_as')) = ANY('{text, int, small_int, big_int, real, double, boolean, date, jsonb}')) THEN - RETURN true; - END IF; - RAISE 'Configuration has an invalid cast_as (%). Cast should be one of {text, int, small_int, big_int, real, double, boolean, date, jsonb}', val; - END; -$$ LANGUAGE plpgsql; - --- --- Should include a tables field --- Tables should not be empty -DROP FUNCTION IF EXISTS _cs_config_check_tables(jsonb); -CREATE FUNCTION _cs_config_check_tables(val jsonb) - RETURNS boolean -AS $$ - BEGIN - IF (val ? 'tables') AND (val->'tables' <> '{}'::jsonb) THEN - RETURN true; - END IF; - RAISE 'Configuration missing tables (tables) field: %', val; - END; -$$ LANGUAGE plpgsql; - --- Should include a version field -DROP FUNCTION IF EXISTS _cs_config_check_v(jsonb); -CREATE FUNCTION _cs_config_check_v(val jsonb) - RETURNS boolean -AS $$ - BEGIN - IF (val ? 'v') THEN - RETURN true; - END IF; - RAISE 'Configuration missing version (v) field: %', val; - END; -$$ LANGUAGE plpgsql; - - -ALTER DOMAIN cs_configuration_data_v1 DROP CONSTRAINT IF EXISTS cs_configuration_data_v1_check; - -ALTER DOMAIN cs_configuration_data_v1 - ADD CONSTRAINT cs_configuration_data_v1_check CHECK ( - _cs_config_check_v(VALUE) AND - _cs_config_check_tables(VALUE) AND - _cs_config_check_cast(VALUE) AND - _cs_config_check_indexes(VALUE) -); - - --- --- CREATE the cs_configuration_v1 TABLE --- -CREATE TABLE IF NOT EXISTS cs_configuration_v1 -( - id bigint GENERATED ALWAYS AS IDENTITY, - state cs_configuration_state_v1 NOT NULL DEFAULT 'pending', - data cs_configuration_data_v1, - created_at timestamptz not null default current_timestamp, - PRIMARY KEY(id) -); - --- --- Define partial indexes to ensure that there is only one active, pending and encrypting config at a time --- -CREATE UNIQUE INDEX IF NOT EXISTS cs_configuration_v1_index_active ON cs_configuration_v1 (state) WHERE state = 'active'; -CREATE UNIQUE INDEX IF NOT EXISTS cs_configuration_v1_index_pending ON cs_configuration_v1 (state) WHERE state = 'pending'; -CREATE UNIQUE INDEX IF NOT EXISTS cs_configuration_v1_index_encrypting ON cs_configuration_v1 (state) WHERE state = 'encrypting'; diff --git a/sql/040-aggregate-ore.sql b/sql/040-aggregate-ore.sql deleted file mode 100644 index d6afadef..00000000 --- a/sql/040-aggregate-ore.sql +++ /dev/null @@ -1,42 +0,0 @@ --- Aggregate functions for ORE -DROP FUNCTION IF EXISTS cs_min_encrypted_v1; -CREATE FUNCTION cs_min_encrypted_v1(a cs_encrypted_v1, b cs_encrypted_v1) -RETURNS cs_encrypted_v1 -LANGUAGE plpgsql -STRICT -AS $$ - BEGIN - IF cs_ore_64_8_v1(a) < cs_ore_64_8_v1(b) THEN - RETURN a; - ELSE - RETURN b; - END IF; - END; -$$; - -CREATE AGGREGATE cs_min_v1(cs_encrypted_v1) -( - sfunc = cs_min_encrypted_v1, - stype = cs_encrypted_v1 -); - -DROP FUNCTION IF EXISTS cs_max_encrypted_v1; -CREATE FUNCTION cs_max_encrypted_v1(a cs_encrypted_v1, b cs_encrypted_v1) -RETURNS cs_encrypted_v1 -LANGUAGE plpgsql -STRICT -AS $$ - BEGIN - IF cs_ore_64_8_v1(a) > cs_ore_64_8_v1(b) THEN - RETURN a; - ELSE - RETURN b; - END IF; - END; -$$; - -CREATE AGGREGATE cs_max_v1(cs_encrypted_v1) -( - sfunc = cs_max_encrypted_v1, - stype = cs_encrypted_v1 -); diff --git a/sql/666-drop-operators.sql b/sql/666-drop-operators.sql deleted file mode 100644 index 8bd9fe3a..00000000 --- a/sql/666-drop-operators.sql +++ /dev/null @@ -1,49 +0,0 @@ -DROP OPERATOR FAMILY IF EXISTS cs_encrypted_ore_64_8_v1_btree_ops_v1 USING btree; -DROP OPERATOR CLASS IF EXISTS ore_64_8_v1_btree_ops USING btree; -DROP OPERATOR IF EXISTS @> (cs_encrypted_v1, cs_encrypted_v1); -DROP OPERATOR IF EXISTS @> (cs_encrypted_v1, cs_match_index_v1); -DROP OPERATOR IF EXISTS @> (cs_match_index_v1, cs_encrypted_v1); - -DROP OPERATOR IF EXISTS <@ (cs_encrypted_v1, cs_encrypted_v1); -DROP OPERATOR IF EXISTS <@ (cs_encrypted_v1, cs_match_index_v1); -DROP OPERATOR IF EXISTS <@ (cs_match_index_v1, cs_encrypted_v1); - -DROP OPERATOR IF EXISTS <= (ore_64_8_v1, cs_encrypted_v1); -DROP OPERATOR IF EXISTS <= (cs_encrypted_v1, ore_64_8_v1); -DROP OPERATOR IF EXISTS <= (jsonb, cs_encrypted_v1); -DROP OPERATOR IF EXISTS <= (cs_encrypted_v1, jsonb); -DROP OPERATOR IF EXISTS <= (cs_encrypted_v1, cs_encrypted_v1); - -DROP OPERATOR IF EXISTS >= (ore_64_8_v1, cs_encrypted_v1); -DROP OPERATOR IF EXISTS >= (jsonb, cs_encrypted_v1); -DROP OPERATOR IF EXISTS >= (cs_encrypted_v1, ore_64_8_v1); -DROP OPERATOR IF EXISTS >= (cs_encrypted_v1, jsonb); -DROP OPERATOR IF EXISTS >= (cs_encrypted_v1, cs_encrypted_v1); - -DROP OPERATOR IF EXISTS < (ore_64_8_v1, cs_encrypted_v1); -DROP OPERATOR IF EXISTS < (cs_encrypted_v1, ore_64_8_v1); -DROP OPERATOR IF EXISTS < (jsonb, cs_encrypted_v1); -DROP OPERATOR IF EXISTS < (cs_encrypted_v1, jsonb); -DROP OPERATOR IF EXISTS < (cs_encrypted_v1, cs_encrypted_v1); - -DROP OPERATOR IF EXISTS > (ore_64_8_v1, cs_encrypted_v1); -DROP OPERATOR IF EXISTS > (jsonb, cs_encrypted_v1); -DROP OPERATOR IF EXISTS > (cs_encrypted_v1, ore_64_8_v1); -DROP OPERATOR IF EXISTS > (cs_encrypted_v1, jsonb); -DROP OPERATOR IF EXISTS > (cs_encrypted_v1, cs_encrypted_v1); - -DROP OPERATOR IF EXISTS = (cs_encrypted_v1, cs_encrypted_v1); -DROP OPERATOR IF EXISTS = (cs_encrypted_v1, jsonb); -DROP OPERATOR IF EXISTS = (jsonb, cs_encrypted_v1); -DROP OPERATOR IF EXISTS = (cs_encrypted_v1, cs_unique_index_v1); -DROP OPERATOR IF EXISTS = (cs_unique_index_v1, cs_encrypted_v1); -DROP OPERATOR IF EXISTS = (cs_encrypted_v1, ore_64_8_v1); -DROP OPERATOR IF EXISTS = (ore_64_8_v1, cs_encrypted_v1); - -DROP OPERATOR IF EXISTS <> (cs_encrypted_v1, cs_encrypted_v1); -DROP OPERATOR IF EXISTS <> (cs_encrypted_v1, jsonb); -DROP OPERATOR IF EXISTS <> (jsonb, cs_encrypted_v1); -DROP OPERATOR IF EXISTS <> (cs_encrypted_v1, cs_unique_index_v1); -DROP OPERATOR IF EXISTS <> (cs_unique_index_v1, cs_encrypted_v1); -DROP OPERATOR IF EXISTS <> (ore_64_8_v1, cs_encrypted_v1); -DROP OPERATOR IF EXISTS <> (cs_encrypted_v1, ore_64_8_v1); diff --git a/sql/666-drop_types.sql b/sql/666-drop_types.sql deleted file mode 100644 index 8de0cb64..00000000 --- a/sql/666-drop_types.sql +++ /dev/null @@ -1,8 +0,0 @@ --- ANYTHING THAT NEEDS TO BE DROPPED LAST -DROP TYPE IF EXISTS ore_64_8_v1; -DROP TYPE IF EXISTS ore_64_8_v1_term; -DROP TYPE IF EXISTS cs_ste_vec_index_v1; -DROP TYPE IF EXISTS cs_ste_vec_v1_entry; -DROP TYPE IF EXISTS ore_cllw_8_v1; -DROP TYPE IF EXISTS ore_cllw_8_variable_v1; -DROP TYPE IF EXISTS cs_ste_vec_encrypted_term_v1; diff --git a/sql/666-rename_configuration_table.sql b/sql/666-rename_configuration_table.sql deleted file mode 100644 index 00b1bbef..00000000 --- a/sql/666-rename_configuration_table.sql +++ /dev/null @@ -1,10 +0,0 @@ --- DANGEROUS --- DROP TABLE IF EXISTS cs_configuration_v1 CASCADE; --- ALTER TABLE cs_configuration_v1 RENAME TO cs_configuration_v1_; - -DO $$ -BEGIN - EXECUTE format('ALTER TABLE IF EXISTS %I RENAME TO %I_%s', 'cs_configuration_v1','cs_configuration_v1_', to_char(current_date,'YYYYMMDD')::TEXT); -END -$$; - diff --git a/sql/database-extensions/postgresql/install.sql b/sql/database-extensions/postgresql/install.sql deleted file mode 100644 index 9f2a6617..00000000 --- a/sql/database-extensions/postgresql/install.sql +++ /dev/null @@ -1,317 +0,0 @@ -CREATE EXTENSION IF NOT EXISTS pgcrypto; - -CREATE TYPE ore_64_8_v1_term AS ( - bytes bytea -); - -CREATE TYPE ore_64_8_v1 AS ( - terms ore_64_8_v1_term[] -); - -CREATE OR REPLACE FUNCTION compare_ore_64_8_v1_term(a ore_64_8_v1_term, b ore_64_8_v1_term) returns integer AS $$ - DECLARE - eq boolean := true; - unequal_block smallint := 0; - hash_key bytea; - target_block bytea; - - left_block_size CONSTANT smallint := 16; - right_block_size CONSTANT smallint := 32; - right_offset CONSTANT smallint := 136; -- 8 * 17 - - indicator smallint := 0; - BEGIN - IF a IS NULL AND b IS NULL THEN - RETURN 0; - END IF; - - IF a IS NULL THEN - RETURN -1; - END IF; - - IF b IS NULL THEN - RETURN 1; - END IF; - - IF bit_length(a.bytes) != bit_length(b.bytes) THEN - RAISE EXCEPTION 'Ciphertexts are different lengths'; - END IF; - - FOR block IN 0..7 LOOP - -- Compare each PRP (byte from the first 8 bytes) and PRF block (8 byte - -- chunks of the rest of the value). - -- NOTE: - -- * Substr is ordinally indexed (hence 1 and not 0, and 9 and not 8). - -- * We are not worrying about timing attacks here; don't fret about - -- the OR or !=. - IF - substr(a.bytes, 1 + block, 1) != substr(b.bytes, 1 + block, 1) - OR substr(a.bytes, 9 + left_block_size * block, left_block_size) != substr(b.bytes, 9 + left_block_size * BLOCK, left_block_size) - THEN - -- set the first unequal block we find - IF eq THEN - unequal_block := block; - END IF; - eq = false; - END IF; - END LOOP; - - IF eq THEN - RETURN 0::integer; - END IF; - - -- Hash key is the IV from the right CT of b - hash_key := substr(b.bytes, right_offset + 1, 16); - - -- first right block is at right offset + nonce_size (ordinally indexed) - target_block := substr(b.bytes, right_offset + 17 + (unequal_block * right_block_size), right_block_size); - - indicator := ( - get_bit( - encrypt( - substr(a.bytes, 9 + (left_block_size * unequal_block), left_block_size), - hash_key, - 'aes-ecb' - ), - 0 - ) + get_bit(target_block, get_byte(a.bytes, unequal_block))) % 2; - - IF indicator = 1 THEN - RETURN 1::integer; - ELSE - RETURN -1::integer; - END IF; - END; -$$ LANGUAGE plpgsql; - - -CREATE OR REPLACE FUNCTION ore_64_8_v1_term_eq(a ore_64_8_v1_term, b ore_64_8_v1_term) RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1_term(a, b) = 0 -$$ LANGUAGE SQL; - -CREATE OR REPLACE FUNCTION ore_64_8_v1_term_neq(a ore_64_8_v1_term, b ore_64_8_v1_term) RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1_term(a, b) <> 0 -$$ LANGUAGE SQL; - -CREATE OR REPLACE FUNCTION ore_64_8_v1_term_lt(a ore_64_8_v1_term, b ore_64_8_v1_term) RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1_term(a, b) = -1 -$$ LANGUAGE SQL; - -CREATE OR REPLACE FUNCTION ore_64_8_v1_term_lte(a ore_64_8_v1_term, b ore_64_8_v1_term) RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1_term(a, b) != 1 -$$ LANGUAGE SQL; - -CREATE OR REPLACE FUNCTION ore_64_8_v1_term_gt(a ore_64_8_v1_term, b ore_64_8_v1_term) RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1_term(a, b) = 1 -$$ LANGUAGE SQL; - -CREATE OR REPLACE FUNCTION ore_64_8_v1_term_gte(a ore_64_8_v1_term, b ore_64_8_v1_term) RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1_term(a, b) != -1 -$$ LANGUAGE SQL; - -CREATE OPERATOR = ( - PROCEDURE="ore_64_8_v1_term_eq", - LEFTARG=ore_64_8_v1_term, - RIGHTARG=ore_64_8_v1_term, - NEGATOR = <>, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - -CREATE OPERATOR <> ( - PROCEDURE="ore_64_8_v1_term_neq", - LEFTARG=ore_64_8_v1_term, - RIGHTARG=ore_64_8_v1_term, - NEGATOR = =, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - -CREATE OPERATOR > ( - PROCEDURE="ore_64_8_v1_term_gt", - LEFTARG=ore_64_8_v1_term, - RIGHTARG=ore_64_8_v1_term, - COMMUTATOR = <, - NEGATOR = <=, - RESTRICT = scalargtsel, - JOIN = scalargtjoinsel -); - -CREATE OPERATOR < ( - PROCEDURE="ore_64_8_v1_term_lt", - LEFTARG=ore_64_8_v1_term, - RIGHTARG=ore_64_8_v1_term, - COMMUTATOR = >, - NEGATOR = >=, - RESTRICT = scalarltsel, - JOIN = scalarltjoinsel -); - -CREATE OPERATOR <= ( - PROCEDURE="ore_64_8_v1_term_lte", - LEFTARG=ore_64_8_v1_term, - RIGHTARG=ore_64_8_v1_term, - COMMUTATOR = >=, - NEGATOR = >, - RESTRICT = scalarlesel, - JOIN = scalarlejoinsel -); - -CREATE OPERATOR >= ( - PROCEDURE="ore_64_8_v1_term_gte", - LEFTARG=ore_64_8_v1_term, - RIGHTARG=ore_64_8_v1_term, - COMMUTATOR = <=, - NEGATOR = <, - RESTRICT = scalarlesel, - JOIN = scalarlejoinsel -); - -CREATE OPERATOR FAMILY ore_64_8_v1_term_btree_ops USING btree; -CREATE OPERATOR CLASS ore_64_8_v1_term_btree_ops DEFAULT FOR TYPE ore_64_8_v1_term USING btree FAMILY ore_64_8_v1_term_btree_ops AS - OPERATOR 1 <, - OPERATOR 2 <=, - OPERATOR 3 =, - OPERATOR 4 >=, - OPERATOR 5 >, - FUNCTION 1 compare_ore_64_8_v1_term(a ore_64_8_v1_term, b ore_64_8_v1_term); - --- Compare the "head" of each array and recurse if necessary --- This function assumes an empty string is "less than" everything else --- so if a is empty we return -1, if be is empty and a isn't, we return 1. --- If both are empty we return 0. This cases probably isn't necessary as equality --- doesn't always make sense but it's here for completeness. --- If both are non-empty, we compare the first element. If they are equal --- we need to consider the next block so we recurse, otherwise we return the comparison result. -CREATE OR REPLACE FUNCTION compare_ore_array(a ore_64_8_v1_term[], b ore_64_8_v1_term[]) returns integer AS $$ - DECLARE - cmp_result integer; - BEGIN - IF (array_length(a, 1) = 0 OR a IS NULL) AND (array_length(b, 1) = 0 OR b IS NULL) THEN - RETURN 0; - END IF; - IF array_length(a, 1) = 0 OR a IS NULL THEN - RETURN -1; - END IF; - IF array_length(b, 1) = 0 OR a IS NULL THEN - RETURN 1; - END IF; - - cmp_result := compare_ore_64_8_v1_term(a[1], b[1]); - IF cmp_result = 0 THEN - -- Removes the first element in the array, and calls this fn again to compare the next element/s in the array. - RETURN compare_ore_array(a[2:array_length(a,1)], b[2:array_length(b,1)]); - END IF; - - RETURN cmp_result; - END -$$ LANGUAGE plpgsql; - --- This function uses lexicographic comparison -CREATE OR REPLACE FUNCTION compare_ore_64_8_v1(a ore_64_8_v1, b ore_64_8_v1) returns integer AS $$ - DECLARE - cmp_result integer; - BEGIN - -- Recursively compare blocks bailing as soon as we can make a decision - RETURN compare_ore_array(a.terms, b.terms); - END -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION ore_64_8_v1_eq(a ore_64_8_v1, b ore_64_8_v1) RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1(a, b) = 0 -$$ LANGUAGE SQL; - -CREATE OR REPLACE FUNCTION ore_64_8_v1_neq(a ore_64_8_v1, b ore_64_8_v1) RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1(a, b) <> 0 -$$ LANGUAGE SQL; - -CREATE OR REPLACE FUNCTION ore_64_8_v1_lt(a ore_64_8_v1, b ore_64_8_v1) RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1(a, b) = -1 -$$ LANGUAGE SQL; - -CREATE OR REPLACE FUNCTION ore_64_8_v1_lte(a ore_64_8_v1, b ore_64_8_v1) RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1(a, b) != 1 -$$ LANGUAGE SQL; - -CREATE OR REPLACE FUNCTION ore_64_8_v1_gt(a ore_64_8_v1, b ore_64_8_v1) RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1(a, b) = 1 -$$ LANGUAGE SQL; - -CREATE OR REPLACE FUNCTION ore_64_8_v1_gte(a ore_64_8_v1, b ore_64_8_v1) RETURNS boolean AS $$ - SELECT compare_ore_64_8_v1(a, b) != -1 -$$ LANGUAGE SQL; - -CREATE OPERATOR = ( - PROCEDURE="ore_64_8_v1_eq", - LEFTARG=ore_64_8_v1, - RIGHTARG=ore_64_8_v1, - NEGATOR = <>, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - -CREATE OPERATOR <> ( - PROCEDURE="ore_64_8_v1_neq", - LEFTARG=ore_64_8_v1, - RIGHTARG=ore_64_8_v1, - NEGATOR = =, - RESTRICT = eqsel, - JOIN = eqjoinsel, - HASHES, - MERGES -); - -CREATE OPERATOR > ( - PROCEDURE="ore_64_8_v1_gt", - LEFTARG=ore_64_8_v1, - RIGHTARG=ore_64_8_v1, - COMMUTATOR = <, - NEGATOR = <=, - RESTRICT = scalargtsel, - JOIN = scalargtjoinsel -); - -CREATE OPERATOR < ( - PROCEDURE="ore_64_8_v1_lt", - LEFTARG=ore_64_8_v1, - RIGHTARG=ore_64_8_v1, - COMMUTATOR = >, - NEGATOR = >=, - RESTRICT = scalarltsel, - JOIN = scalarltjoinsel -); - -CREATE OPERATOR <= ( - PROCEDURE="ore_64_8_v1_lte", - LEFTARG=ore_64_8_v1, - RIGHTARG=ore_64_8_v1, - COMMUTATOR = >=, - NEGATOR = >, - RESTRICT = scalarlesel, - JOIN = scalarlejoinsel -); - -CREATE OPERATOR >= ( - PROCEDURE="ore_64_8_v1_gte", - LEFTARG=ore_64_8_v1, - RIGHTARG=ore_64_8_v1, - COMMUTATOR = <=, - NEGATOR = <, - RESTRICT = scalarlesel, - JOIN = scalarlejoinsel -); - -CREATE OPERATOR FAMILY ore_64_8_v1_btree_ops USING btree; -CREATE OPERATOR CLASS ore_64_8_v1_btree_ops DEFAULT FOR TYPE ore_64_8_v1 USING btree FAMILY ore_64_8_v1_btree_ops AS - OPERATOR 1 <, - OPERATOR 2 <=, - OPERATOR 3 =, - OPERATOR 4 >=, - OPERATOR 5 >, - FUNCTION 1 compare_ore_64_8_v1(a ore_64_8_v1, b ore_64_8_v1); diff --git a/sql/database-extensions/postgresql/uninstall.sql b/sql/database-extensions/postgresql/uninstall.sql deleted file mode 100644 index 453ae68b..00000000 --- a/sql/database-extensions/postgresql/uninstall.sql +++ /dev/null @@ -1,20 +0,0 @@ --- TODO: what happens if we try to uninstall a type which is in use? -DROP OPERATOR IF EXISTS = (ore_64_8_v1_term, ore_64_8_v1_term) CASCADE; -DROP OPERATOR IF EXISTS <> (ore_64_8_v1_term, ore_64_8_v1_term) CASCADE; -DROP OPERATOR IF EXISTS > (ore_64_8_v1_term, ore_64_8_v1_term) CASCADE; -DROP OPERATOR IF EXISTS < (ore_64_8_v1_term, ore_64_8_v1_term) CASCADE; -DROP OPERATOR IF EXISTS <= (ore_64_8_v1_term, ore_64_8_v1_term) CASCADE; -DROP OPERATOR IF EXISTS >= (ore_64_8_v1_term, ore_64_8_v1_term) CASCADE; -DROP OPERATOR CLASS IF EXISTS ore_64_8_v1_term_btree_ops USING btree CASCADE; -DROP OPERATOR FAMILY IF EXISTS ore_64_8_v1_term_btree_ops USING btree CASCADE; -DROP TYPE IF EXISTS ore_64_8_v1_term CASCADE; - -DROP OPERATOR IF EXISTS = (ore_64_8_v1, ore_64_8_v1) CASCADE; -DROP OPERATOR IF EXISTS <> (ore_64_8_v1, ore_64_8_v1) CASCADE; -DROP OPERATOR IF EXISTS > (ore_64_8_v1, ore_64_8_v1) CASCADE; -DROP OPERATOR IF EXISTS < (ore_64_8_v1, ore_64_8_v1) CASCADE; -DROP OPERATOR IF EXISTS <= (ore_64_8_v1, ore_64_8_v1) CASCADE; -DROP OPERATOR IF EXISTS >= (ore_64_8_v1, ore_64_8_v1) CASCADE; -DROP OPERATOR CLASS IF EXISTS ore_64_8_v1_btree_ops USING btree CASCADE; -DROP OPERATOR FAMILY IF EXISTS ore_64_8_v1_btree_ops USING btree CASCADE; -DROP TYPE IF EXISTS ore_64_8_v1 CASCADE; diff --git a/sql/schemas/cs_configuration_data_v1.schema.json b/sql/schemas/cs_configuration_data_v1.schema.json deleted file mode 100644 index 42c86f44..00000000 --- a/sql/schemas/cs_configuration_data_v1.schema.json +++ /dev/null @@ -1,116 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "s": { - "title": "Schema version", - "description": "The schema version of this json document ", - "type": "integer", - "enum": [1] - }, - "tables": { - "type": "object", - "additionalProperties": { - "$ref": "#/$defs/table" - } - } - }, - "required": ["s", "tables"], - "$defs": { - "table": { - "title": "Table configuration", - "type": "object", - "additionalProperties": { - "$ref": "#/$defs/column" - } - }, - "column": { - "title": "Column configuration", - "type": "object", - "properties": { - "cast_as": { - "title": "cast as type", - "description": "The type the decrypted column value will be cast as", - "type": "string", - "enum": ["text", "int"] - }, - "indexes": { - "title": "Table configuration", - "type": "object", - "properties": { - "match_1": { - "$ref": "#/$defs/match_1" - }, - "ore_1": { - "$ref": "#/$defs/ore_1" - }, - "unique_1": { - "$ref": "#/$defs/unique_1" - } - }, - "additionalProperties": false - } - } - }, - "token_filters": { - "title": "Token filters", - "token_filters": { - "type": "object", - "properties": { - "kind": { - "type": "string", - "enum": ["downcase", "upcase"] - } - } - } - }, - "ore_1": { - "title": "Index configuration", - "type": "object", - "additionalProperties": false - }, - "unique_1": { - "title": "Unique index v1", - "type": "object", - "properties": { - "token_filters": { - "$ref": "#/$defs/token_filters" - } - } - }, - "match_1": { - "title": "Index configuration", - "type": "object", - "properties": { - "k": { - "type": "integer" - }, - "m": { - "type": "integer" - }, - "tokenizer": { - "type": "object", - "properties": { - "kind": { - "type": "string", - "enum": ["edge-ngram", "ngram"] - }, - "min_gram": { - "type": "integer" - }, - "max_gram": { - "type": "integer" - }, - "include_original": { - "type": "boolean" - } - } - }, - "token_filters": { - "$ref": "#/$defs/token_filters" - } - }, - "required": ["k", "m"] - } - } -} diff --git a/sql/schemas/cs_encrypted_query_v1.schema.json b/sql/schemas/cs_encrypted_query_v1.schema.json deleted file mode 100644 index b3be518d..00000000 --- a/sql/schemas/cs_encrypted_query_v1.schema.json +++ /dev/null @@ -1,94 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "description": "The EQL encrypted JSON payload used for queries.", - "type": "object", - "properties": { - "k": { - "title": "kind", - "type": "string", - "enum": ["qm", "qo", "qu", "qsv", "qsvs"] - } - }, - "oneOf": [ - { - "description": "match query", - "properties": { - "k": { - "const": "qm" - }, - "m": { - "title": "match index", - "type": "array", - "minItems": 1, - "items": { - "type": "number" - } - } - }, - "required": ["m"] - }, - { - "description": "ore query", - "properties": { - "k": { - "const": "qo" - }, - "o": { - "title": "ore index", - "type": "array", - "minItems": 1, - "items": { - "type": "string" - } - } - }, - "required": ["o"] - }, - { - "description": "unique query", - "properties": { - "k": { - "const": "qu" - }, - "u": { - "title": "unique index", - "type": "string" - } - }, - "required": ["u"] - }, - { - "description": "Structured Encryption vector query", - "properties": { - "k": { - "const": "qsv" - }, - "sv": { - "type": "array", - "items": { - "type": "array", - "items": { - "type": "string", - "minItems": 2, - "maxItems": 2 - } - } - } - }, - "required": ["sv"] - }, - { - "description": "Structured Encryption vector selector query", - "properties": { - "k": { - "const": "qsvs" - }, - "svs": { - "type": "string" - } - }, - "required": ["svs"] - } - ], - "required": ["k"] -} diff --git a/sql/schemas/cs_encrypted_storage_v1.schema.json b/sql/schemas/cs_encrypted_storage_v1.schema.json deleted file mode 100644 index 8cefef4e..00000000 --- a/sql/schemas/cs_encrypted_storage_v1.schema.json +++ /dev/null @@ -1,88 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "description": "The EQL encrypted JSON payload used for storage.", - "type": "object", - "properties": { - "v": { - "title": "Schema version", - "type": "integer" - }, - "k": { - "title": "kind", - "type": "string", - "enum": ["ct", "sv"] - }, - "i": { - "title": "ident", - "type": "object", - "properties": { - "t": { - "title": "table", - "type": "string", - "pattern": "^[a-zA-Z_]{1}[0-9a-zA-Z_]*$" - }, - "c": { - "title": "column", - "type": "string", - "pattern": "^[a-zA-Z_]{1}[0-9a-zA-Z_]*$" - } - }, - "required": ["t", "c"] - } - }, - "oneOf": [ - { - "properties": { - "k": { - "const": "ct" - }, - "c": { - "title": "ciphertext", - "type": "string" - }, - "u": { - "title": "unique index", - "type": "string" - }, - "o": { - "title": "ore index", - "type": "array", - "minItems": 1, - "items": { - "type": "string" - } - }, - "m": { - "title": "match index", - "type": "array", - "minItems": 1, - "items": { - "type": "number" - } - } - }, - "required": ["c"] - }, - { - "properties": { - "k": { - "const": "sv" - }, - "sv": { - "title": "Structured Encryption vector", - "type": "array", - "items": { - "type": "array", - "items": { - "type": "string", - "minItems": 3, - "maxItems": 3 - } - } - } - }, - "required": ["sv"] - } - ], - "required": ["v", "k", "i"] -} diff --git a/sql/schemas/cs_plaintext_v1.schema.json b/sql/schemas/cs_plaintext_v1.schema.json deleted file mode 100644 index 56bec1b7..00000000 --- a/sql/schemas/cs_plaintext_v1.schema.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "description": "The EQL plaintext JSON payload sent by a client (such as an application) to CipherStash Proxy.", - "type": "object", - "properties": { - "v": { - "title": "Schema version", - "type": "integer" - }, - "k": { - "title": "kind", - "type": "string", - "const": "pt" - }, - "i": { - "title": "ident", - "type": "object", - "properties": { - "t": { - "title": "table", - "type": "string", - "pattern": "^[a-zA-Z_]{1}[0-9a-zA-Z_]*$" - }, - "c": { - "title": "column", - "type": "string", - "pattern": "^[a-zA-Z_]{1}[0-9a-zA-Z_]*$" - } - }, - "required": ["t", "c"] - }, - "p": { - "title": "plaintext", - "type": "string" - }, - "q": { - "title": "for query", - "description": "Specifies that the plaintext should be encrypted for a specific query operation. If null, source encryption and encryption for all indexes will be performed.", - "type": "string", - "enum": ["match", "ore", "unique", "ste_vec", "ejson_path"] - } - }, - "required": ["v", "k", "i", "p"] -} diff --git a/sql/schemas/payload-examples.md b/sql/schemas/payload-examples.md deleted file mode 100644 index c9fb450a..00000000 --- a/sql/schemas/payload-examples.md +++ /dev/null @@ -1,87 +0,0 @@ - -## Minimal Plaintext - -``` -{ - "v": 1, - "s": { - "k": "pt", - "p": "plaintext string", - "e": { - "t": "users", - "c": "name_encrypted" - } - } -} -``` - - -INSERT INTO users (name_encrypted) VALUES (' -{ - "v": 1, - "s": { - "k": "ct", - "c": "ciphertext", - "e": { - "t": "table", - "c": "column" - }, - "m": [42], - "u": "unique", - "o": ["a","b","c"] - }, - "t": { - "k": "pt", - "p": "plaintext", - "e": { - "t": "table", - "c": "column" - }, - "m": [42], - "u": "unique" - } -}'::cs_encrypted_v1); - - - - -## Minimal Ciphertext - -``` -{ - "v": 1, - "s": { - "k": "ct", - "c": "XvfWQUrSxKNhkOxiMXvgvkwxIYFfnYTb", - "e": { - "t": "users", - "c": "name_encrypted" - } - } -} -``` - - -## Embedded - -``` -{ - "v": 1, - "s": { - "k": "ct", - "c": "XvfWQUrSxKNhkOxiMXvgvkwxIYFfnYTb", - "e": { - "t": "users", - "c": "name_encrypted" - } - } - "t": { - "k": "pt", - "p": "plaintext string", - "e": { - "t": "users", - "c": "name" - } - } -} -``` \ No newline at end of file diff --git a/sql/README.md b/src/README.md similarity index 100% rename from sql/README.md rename to src/README.md diff --git a/src/blake3/functions.sql b/src/blake3/functions.sql new file mode 100644 index 00000000..2b1241b7 --- /dev/null +++ b/src/blake3/functions.sql @@ -0,0 +1,35 @@ +-- REQUIRE: src/schema.sql + +-- extracts ste_vec index from a jsonb value + +-- extracts blake3 index from a jsonb value + +CREATE FUNCTION eql_v2.blake3(val jsonb) + RETURNS eql_v2.blake3 + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + + IF NOT (val ? 'b') NULL THEN + RAISE 'Expected a blake3 index (b) value in json: %', val; + END IF; + + IF val->>'b' IS NULL THEN + RETURN NULL; + END IF; + + RETURN val->>'b'; + END; +$$ LANGUAGE plpgsql; + + +-- extracts blake3 index from an eql_v2_encrypted value + +CREATE FUNCTION eql_v2.blake3(val eql_v2_encrypted) + RETURNS eql_v2.blake3 + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN (SELECT eql_v2.blake3(val.data)); + END; +$$ LANGUAGE plpgsql; diff --git a/src/blake3/types.sql b/src/blake3/types.sql new file mode 100644 index 00000000..b13c4f9d --- /dev/null +++ b/src/blake3/types.sql @@ -0,0 +1,3 @@ +-- REQUIRE: src/schema.sql + +CREATE DOMAIN eql_v2.blake3 AS text; diff --git a/src/common.sql b/src/common.sql new file mode 100644 index 00000000..f47d917e --- /dev/null +++ b/src/common.sql @@ -0,0 +1,71 @@ +-- AUTOMATICALLY GENERATED FILE +-- REQUIRE: src/schema.sql + +-- Constant time comparison of 2 bytea values + + +CREATE FUNCTION eql_v2.bytea_eq(a bytea, b bytea) RETURNS boolean AS $$ +DECLARE + result boolean; + differing bytea; +BEGIN + + -- Check if the bytea values are the same length + IF LENGTH(a) != LENGTH(b) THEN + RETURN false; + END IF; + + -- Compare each byte in the bytea values + result := true; + FOR i IN 1..LENGTH(a) LOOP + IF SUBSTRING(a FROM i FOR 1) != SUBSTRING(b FROM i FOR 1) THEN + result := result AND false; + END IF; + END LOOP; + + RETURN result; +END; +$$ LANGUAGE plpgsql; + +-- Casts a jsonb array of hex-encoded strings to an array of bytea. +CREATE FUNCTION eql_v2.jsonb_array_to_bytea_array(val jsonb) +RETURNS bytea[] AS $$ +DECLARE + terms_arr bytea[]; +BEGIN + IF jsonb_typeof(val) = 'null' THEN + RETURN NULL; + END IF; + + SELECT array_agg(decode(value::text, 'hex')::bytea) + INTO terms_arr + FROM jsonb_array_elements_text(val) AS value; + + RETURN terms_arr; +END; +$$ LANGUAGE plpgsql; + + + +-- +-- Convenience function to log a message +-- +CREATE FUNCTION eql_v2.log(s text) + RETURNS void +AS $$ + BEGIN + RAISE NOTICE '[LOG] %', s; +END; +$$ LANGUAGE plpgsql; + + +-- +-- Convenience function to describe a test +-- +CREATE FUNCTION eql_v2.log(ctx text, s text) + RETURNS void +AS $$ + BEGIN + RAISE NOTICE '[LOG] % %', ctx, s; +END; +$$ LANGUAGE plpgsql; diff --git a/tests/config.sql b/src/config/config_test.sql similarity index 59% rename from tests/config.sql rename to src/config/config_test.sql index e165c76b..bf3ac407 100644 --- a/tests/config.sql +++ b/src/config/config_test.sql @@ -1,8 +1,6 @@ \set ON_ERROR_STOP on - - -- -- Helper function for assertions -- @@ -11,7 +9,7 @@ CREATE FUNCTION _index_exists(table_name text, column_name text, index_name text RETURNS boolean LANGUAGE sql STRICT PARALLEL SAFE BEGIN ATOMIC - SELECT EXISTS (SELECT id FROM cs_configuration_v1 c + SELECT EXISTS (SELECT id FROM eql_v2_configuration c WHERE c.state = state AND c.data #> array['tables', table_name, column_name, 'indexes'] ? index_name); END; @@ -21,31 +19,30 @@ END; -- Add and remove multiple indexes -- -- ----------------------------------------------- -TRUNCATE TABLE cs_configuration_v1; - +TRUNCATE TABLE eql_v2_configuration; DO $$ BEGIN -- Add indexes - PERFORM cs_add_index_v1('users', 'name', 'match'); + PERFORM eql_v2.add_index('users', 'name', 'match'); ASSERT (SELECT _index_exists('users', 'name', 'match')); -- Add index with cast - PERFORM cs_add_index_v1('users', 'name', 'unique', 'int'); + PERFORM eql_v2.add_index('users', 'name', 'unique', 'int'); ASSERT (SELECT _index_exists('users', 'name', 'unique')); - ASSERT (SELECT EXISTS (SELECT id FROM cs_configuration_v1 c + ASSERT (SELECT EXISTS (SELECT id FROM eql_v2_configuration c WHERE c.state = 'pending' AND c.data #> array['tables', 'users', 'name'] ? 'cast_as')); -- Match index removed - PERFORM cs_remove_index_v1('users', 'name', 'match'); + PERFORM eql_v2.remove_index('users', 'name', 'match'); ASSERT NOT (SELECT _index_exists('users', 'name', 'match')); -- All indexes removed, delete the emtpty pending config - PERFORM cs_remove_index_v1('users', 'name', 'unique'); - ASSERT (SELECT NOT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'pending')); + PERFORM eql_v2.remove_index('users', 'name', 'unique'); + ASSERT (SELECT NOT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'pending')); END; $$ LANGUAGE plpgsql; @@ -56,78 +53,78 @@ $$ LANGUAGE plpgsql; -- Add and remove multiple indexes from multiple tables -- -- ----------------------------------------------- -TRUNCATE TABLE cs_configuration_v1; +TRUNCATE TABLE eql_v2_configuration; DO $$ BEGIN -- Add indexes - PERFORM cs_add_index_v1('users', 'name', 'match'); + PERFORM eql_v2.add_index('users', 'name', 'match'); ASSERT (SELECT _index_exists('users', 'name', 'match')); - ASSERT (SELECT EXISTS (SELECT id FROM cs_configuration_v1 c + ASSERT (SELECT EXISTS (SELECT id FROM eql_v2_configuration c WHERE c.state = 'pending' AND c.data #> array['tables', 'users', 'name', 'indexes'] ? 'match')); -- Add index with cast - PERFORM cs_add_index_v1('blah', 'vtha', 'unique', 'int'); + PERFORM eql_v2.add_index('blah', 'vtha', 'unique', 'int'); ASSERT (SELECT _index_exists('blah', 'vtha', 'unique')); - ASSERT (SELECT EXISTS (SELECT id FROM cs_configuration_v1 c + ASSERT (SELECT EXISTS (SELECT id FROM eql_v2_configuration c WHERE c.state = 'pending' AND c.data #> array['tables', 'users', 'name', 'indexes'] ? 'match')); - ASSERT (SELECT EXISTS (SELECT id FROM cs_configuration_v1 c + ASSERT (SELECT EXISTS (SELECT id FROM eql_v2_configuration c WHERE c.state = 'pending' AND c.data #> array['tables', 'blah', 'vtha', 'indexes'] ? 'unique')); -- Match index removed - PERFORM cs_remove_index_v1('users', 'name', 'match'); + PERFORM eql_v2.remove_index('users', 'name', 'match'); ASSERT NOT (SELECT _index_exists('users', 'name', 'match')); -- Match index removed - PERFORM cs_remove_index_v1('blah', 'vtha', 'unique'); + PERFORM eql_v2.remove_index('blah', 'vtha', 'unique'); ASSERT NOT (SELECT _index_exists('users', 'vtha', 'unique')); -- All indexes removed, delete the emtpty pending config - ASSERT (SELECT NOT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'pending')); + ASSERT (SELECT NOT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'pending')); END; $$ LANGUAGE plpgsql; -SELECT FROM cs_configuration_v1 c WHERE c.state = 'pending'; +-- SELECT FROM eql_v2_configuration c WHERE c.state = 'pending'; -- ----------------------------------------------- -- Add & modify index -- Pending configuration created and contains the path `user/name.match.option` -- ----------------------------------------------- --- TRUNCATE TABLE cs_configuration_v1; +-- TRUNCATE TABLE eql_v2_configuration; DO $$ BEGIN - PERFORM cs_add_index_v1('users', 'name', 'match'); + PERFORM eql_v2.add_index('users', 'name', 'match'); ASSERT (SELECT _index_exists('users', 'name', 'match')); -- Pending configuration contains the path `user/name.match.option` - PERFORM cs_modify_index_v1('users', 'name', 'match', 'int', '{"option": "value"}'::jsonb); + PERFORM eql_v2.modify_index('users', 'name', 'match', 'int', '{"option": "value"}'::jsonb); ASSERT (SELECT _index_exists('users', 'name', 'match')); - ASSERT (SELECT EXISTS (SELECT id FROM cs_configuration_v1 c + ASSERT (SELECT EXISTS (SELECT id FROM eql_v2_configuration c WHERE c.state = 'pending' AND c.data #> array['tables', 'users', 'name', 'indexes', 'match'] ? 'option')); - ASSERT (SELECT EXISTS (SELECT id FROM cs_configuration_v1 c + ASSERT (SELECT EXISTS (SELECT id FROM eql_v2_configuration c WHERE c.state = 'pending' AND c.data #> array['tables', 'users', 'name'] ? 'cast_as')); -- All indexes removed, delete the emtpty pending config - PERFORM cs_remove_index_v1('users', 'name', 'match'); - ASSERT (SELECT NOT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'pending')); + PERFORM eql_v2.remove_index('users', 'name', 'match'); + ASSERT (SELECT NOT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'pending')); END; $$ LANGUAGE plpgsql; @@ -136,10 +133,10 @@ $$ LANGUAGE plpgsql; -- -- With existing active config -- -- Adding an index creates a new pending configuration -- -- ----------------------------------------------- -TRUNCATE TABLE cs_configuration_v1; +TRUNCATE TABLE eql_v2_configuration; -- create an active configuration -INSERT INTO cs_configuration_v1 (state, data) VALUES ( +INSERT INTO eql_v2_configuration (state, data) VALUES ( 'active', '{ "v": 1, @@ -165,7 +162,7 @@ DO $$ BEGIN ASSERT (SELECT _index_exists('users', 'blah', 'match', 'active')); - PERFORM cs_add_index_v1('users', 'name', 'match'); + PERFORM eql_v2.add_index('users', 'name', 'match'); -- index added to name ASSERT (SELECT _index_exists('users', 'name', 'match' )); @@ -182,41 +179,72 @@ $$ LANGUAGE plpgsql; -- -- Add and remove column -- -- -- -- ----------------------------------------------- -TRUNCATE TABLE cs_configuration_v1; +TRUNCATE TABLE eql_v2_configuration; +DO $$ + BEGIN + + PERFORM assert_exception( + 'Cannot add index to column that does not exist', + 'SELECT eql_v2.add_column(''user'', ''name'')'); + + PERFORM assert_no_result( + 'No configuration was created', + 'SELECT * FROM eql_v2_configuration'); + END; +$$ LANGUAGE plpgsql; + + + +-- -- ----------------------------------------------- +-- -- Add and remove column +-- -- +-- -- ----------------------------------------------- +TRUNCATE TABLE eql_v2_configuration; DO $$ BEGIN - -- Create pending configuration - PERFORM cs_add_column_v1('user', 'name'); - ASSERT (SELECT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'pending')); + -- reset the table + PERFORM create_table_with_encrypted(); + + PERFORM eql_v2.add_column('encrypted', 'e'); + + PERFORM assert_count( + 'Pending configuration was created', + 'SELECT * FROM eql_v2_configuration c WHERE c.state = ''pending''', + 1); + + + PERFORM eql_v2.remove_column('encrypted', 'e'); - PERFORM cs_remove_column_v1('user', 'name'); + PERFORM assert_no_result( + 'Pending configuration was removed', + 'SELECT * FROM eql_v2_configuration c WHERE c.state = ''pending'''); - -- Config now empty and removed - ASSERT (SELECT NOT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'pending')); END; $$ LANGUAGE plpgsql; + -- ----------------------------------------------- --- --- cs_configuration_v1 tyoe +-- eql_v2_configuration tyoe -- Validate configuration schema -- Try and insert many invalid configurations -- None should exist -- -- ----------------------------------------------- -TRUNCATE TABLE cs_configuration_v1; +TRUNCATE TABLE eql_v2_configuration; \set ON_ERROR_STOP off \set ON_ERROR_ROLLBACK on DO $$ BEGIN - RAISE NOTICE 'cs_configuration_v1 constraint tests: 4 errors expected here'; + RAISE NOTICE '------------------------------------------------------'; + RAISE NOTICE 'eql_v2_configuration constraint tests: 4 errors follow'; END; $$ LANGUAGE plpgsql; -- -- No schema version -INSERT INTO cs_configuration_v1 (data) VALUES ( +INSERT INTO eql_v2_configuration (data) VALUES ( '{ "tables": { "users": { @@ -231,7 +259,7 @@ INSERT INTO cs_configuration_v1 (data) VALUES ( -- -- Empty tables -INSERT INTO cs_configuration_v1 (data) VALUES ( +INSERT INTO eql_v2_configuration (data) VALUES ( '{ "v": 1, "tables": {} @@ -241,7 +269,7 @@ INSERT INTO cs_configuration_v1 (data) VALUES ( -- -- invalid cast -INSERT INTO cs_configuration_v1 (data) VALUES ( +INSERT INTO eql_v2_configuration (data) VALUES ( '{ "v": 1, "tables": { @@ -256,7 +284,7 @@ INSERT INTO cs_configuration_v1 (data) VALUES ( -- -- invalid index -INSERT INTO cs_configuration_v1 (data) VALUES ( +INSERT INTO eql_v2_configuration (data) VALUES ( '{ "v": 1, "tables": { @@ -276,7 +304,14 @@ INSERT INTO cs_configuration_v1 (data) VALUES ( -- Pending configuration should not be created; DO $$ BEGIN - ASSERT (SELECT NOT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'pending')); + ASSERT (SELECT NOT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'pending')); + END; +$$ LANGUAGE plpgsql; + +DO $$ + BEGIN + RAISE NOTICE 'eql_v2_configuration constraint tests: OK'; + RAISE NOTICE '------------------------------------------------------'; END; $$ LANGUAGE plpgsql; diff --git a/src/config/constraints.sql b/src/config/constraints.sql new file mode 100644 index 00000000..dbe1a4e7 --- /dev/null +++ b/src/config/constraints.sql @@ -0,0 +1,86 @@ +-- REQUIRE: src/config/types.sql + +-- +-- Extracts index keys/names from configuration json +-- +-- Used by the eql_v2.config_check_indexes as part of the configuration_data_v2 constraint +-- +CREATE FUNCTION eql_v2.config_get_indexes(val jsonb) + RETURNS SETOF text + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +BEGIN ATOMIC + SELECT jsonb_object_keys(jsonb_path_query(val,'$.tables.*.*.indexes')); +END; + +-- +-- _cs_check_config_get_indexes returns true if the table configuration only includes valid index types +-- +-- Used by the cs_configuration_data_v2_check constraint +-- +CREATE FUNCTION eql_v2.config_check_indexes(val jsonb) + RETURNS BOOLEAN + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + + IF (SELECT EXISTS (SELECT eql_v2.config_get_indexes(val))) THEN + IF (SELECT bool_and(index = ANY('{match, ore, unique, ste_vec}')) FROM eql_v2.config_get_indexes(val) AS index) THEN + RETURN true; + END IF; + RAISE 'Configuration has an invalid index (%). Index should be one of {match, ore, unique, ste_vec}', val; + END IF; + RETURN true; + END; +$$ LANGUAGE plpgsql; + + + +CREATE FUNCTION eql_v2.config_check_cast(val jsonb) + RETURNS BOOLEAN +AS $$ + BEGIN + IF EXISTS (SELECT jsonb_array_elements_text(jsonb_path_query_array(val, '$.tables.*.*.cast_as')) = ANY('{text, int, small_int, big_int, real, double, boolean, date, jsonb}')) THEN + RETURN true; + END IF; + RAISE 'Configuration has an invalid cast_as (%). Cast should be one of {text, int, small_int, big_int, real, double, boolean, date, jsonb}', val; + END; +$$ LANGUAGE plpgsql; + +-- +-- Should include a tables field +-- Tables should not be empty +CREATE FUNCTION eql_v2.config_check_tables(val jsonb) + RETURNS boolean +AS $$ + BEGIN + IF (val ? 'tables') AND (val->'tables' <> '{}'::jsonb) THEN + RETURN true; + END IF; + RAISE 'Configuration missing tables (tables) field: %', val; + END; +$$ LANGUAGE plpgsql; + +-- Should include a version field +CREATE FUNCTION eql_v2.config_check_version(val jsonb) + RETURNS boolean +AS $$ + BEGIN + IF (val ? 'v') THEN + RETURN true; + END IF; + RAISE 'Configuration missing version (v) field: %', val; + END; +$$ LANGUAGE plpgsql; + + +ALTER TABLE public.eql_v2_configuration DROP CONSTRAINT IF EXISTS eql_v2_configuration_data_check; + +ALTER TABLE public.eql_v2_configuration + ADD CONSTRAINT eql_v2_configuration_data_check CHECK ( + eql_v2.config_check_version(data) AND + eql_v2.config_check_tables(data) AND + eql_v2.config_check_cast(data) AND + eql_v2.config_check_indexes(data) +); + + diff --git a/sql/021-config-functions.sql b/src/config/functions.sql similarity index 58% rename from sql/021-config-functions.sql rename to src/config/functions.sql index dd7ddc34..08544108 100644 --- a/sql/021-config-functions.sql +++ b/src/config/functions.sql @@ -1,11 +1,11 @@ +-- REQUIRE: src/config/types.sql -- -- Configuration functions -- -- -DROP FUNCTION IF EXISTS _cs_config_default(config jsonb); -CREATE FUNCTION _cs_config_default(config jsonb) +CREATE FUNCTION eql_v2.config_default(config jsonb) RETURNS jsonb IMMUTABLE PARALLEL SAFE AS $$ @@ -18,11 +18,10 @@ AS $$ $$ LANGUAGE plpgsql; -DROP FUNCTION IF EXISTS _cs_config_add_table(table_name text, config jsonb); -CREATE FUNCTION _cs_config_add_table(table_name text, config jsonb) +CREATE FUNCTION eql_v2.config_add_table(table_name text, config jsonb) RETURNS jsonb - -- IMMUTABLE PARALLEL SAFE + IMMUTABLE PARALLEL SAFE AS $$ DECLARE tbl jsonb; @@ -36,9 +35,8 @@ $$ LANGUAGE plpgsql; -- Add the column if it doesn't exist -DROP FUNCTION IF EXISTS _cs_config_add_column(table_name text, column_name text, config jsonb); -CREATE FUNCTION _cs_config_add_column(table_name text, column_name text, config jsonb) +CREATE FUNCTION eql_v2.config_add_column(table_name text, column_name text, config jsonb) RETURNS jsonb IMMUTABLE PARALLEL SAFE AS $$ @@ -55,9 +53,8 @@ $$ LANGUAGE plpgsql; -- Set the cast -DROP FUNCTION IF EXISTS _cs_config_add_cast(table_name text, column_name text, cast_as text, config jsonb); -CREATE FUNCTION _cs_config_add_cast(table_name text, column_name text, cast_as text, config jsonb) +CREATE FUNCTION eql_v2.config_add_cast(table_name text, column_name text, cast_as text, config jsonb) RETURNS jsonb IMMUTABLE PARALLEL SAFE AS $$ @@ -69,9 +66,8 @@ $$ LANGUAGE plpgsql; -- Add the column if it doesn't exist -DROP FUNCTION IF EXISTS _cs_config_add_index(table_name text, column_name text, index_name text, opts jsonb, config jsonb); -CREATE FUNCTION _cs_config_add_index(table_name text, column_name text, index_name text, opts jsonb, config jsonb) +CREATE FUNCTION eql_v2.config_add_index(table_name text, column_name text, index_name text, opts jsonb, config jsonb) RETURNS jsonb IMMUTABLE PARALLEL SAFE AS $$ @@ -85,9 +81,8 @@ $$ LANGUAGE plpgsql; -- -- Default options for match index -- -DROP FUNCTION IF EXISTS _cs_config_match_default(); -CREATE FUNCTION _cs_config_match_default() +CREATE FUNCTION eql_v2.config_match_default() RETURNS jsonb LANGUAGE sql STRICT PARALLEL SAFE BEGIN ATOMIC @@ -100,12 +95,12 @@ BEGIN ATOMIC END; -- +-- Adds an index term to the configuration -- --- -DROP FUNCTION IF EXISTS cs_add_index_v1(table_name text, column_name text, index_name text, cast_as text, opts jsonb); -CREATE FUNCTION cs_add_index_v1(table_name text, column_name text, index_name text, cast_as text DEFAULT 'text', opts jsonb DEFAULT '{}') +CREATE FUNCTION eql_v2.add_index(table_name text, column_name text, index_name text, cast_as text DEFAULT 'text', opts jsonb DEFAULT '{}') RETURNS jsonb + AS $$ DECLARE o jsonb; @@ -113,7 +108,7 @@ AS $$ BEGIN -- set the active config - SELECT data INTO _config FROM cs_configuration_v1 WHERE state = 'active' OR state = 'pending' ORDER BY state DESC; + SELECT data INTO _config FROM public.eql_v2_configuration WHERE state = 'active' OR state = 'pending' ORDER BY state DESC; -- if index exists IF _config #> array['tables', table_name, column_name, 'indexes'] ? index_name THEN @@ -125,23 +120,23 @@ AS $$ END IF; -- set default config - SELECT _cs_config_default(_config) INTO _config; + SELECT eql_v2.config_default(_config) INTO _config; - SELECT _cs_config_add_table(table_name, _config) INTO _config; + SELECT eql_v2.config_add_table(table_name, _config) INTO _config; - SELECT _cs_config_add_column(table_name, column_name, _config) INTO _config; + SELECT eql_v2.config_add_column(table_name, column_name, _config) INTO _config; - SELECT _cs_config_add_cast(table_name, column_name, cast_as, _config) INTO _config; + SELECT eql_v2.config_add_cast(table_name, column_name, cast_as, _config) INTO _config; -- set default options for index if opts empty IF index_name = 'match' AND opts = '{}' THEN - SELECT _cs_config_match_default() INTO opts; + SELECT eql_v2.config_match_default() INTO opts; END IF; - SELECT _cs_config_add_index(table_name, column_name, index_name, opts, _config) INTO _config; + SELECT eql_v2.config_add_index(table_name, column_name, index_name, opts, _config) INTO _config; -- create a new pending record if we don't have one - INSERT INTO cs_configuration_v1 (state, data) VALUES ('pending', _config) + INSERT INTO public.eql_v2_configuration (state, data) VALUES ('pending', _config) ON CONFLICT (state) WHERE state = 'pending' DO UPDATE @@ -153,9 +148,8 @@ AS $$ $$ LANGUAGE plpgsql; -DROP FUNCTION IF EXISTS cs_remove_index_v1(table_name text, column_name text, index_name text); -CREATE FUNCTION cs_remove_index_v1(table_name text, column_name text, index_name text) +CREATE FUNCTION eql_v2.remove_index(table_name text, column_name text, index_name text) RETURNS jsonb AS $$ DECLARE @@ -163,7 +157,7 @@ AS $$ BEGIN -- set the active config - SELECT data INTO _config FROM cs_configuration_v1 WHERE state = 'active' OR state = 'pending' ORDER BY state DESC; + SELECT data INTO _config FROM public.eql_v2_configuration WHERE state = 'active' OR state = 'pending' ORDER BY state DESC; -- if no config IF _config IS NULL THEN @@ -182,7 +176,7 @@ AS $$ END IF; -- create a new pending record if we don't have one - INSERT INTO cs_configuration_v1 (state, data) VALUES ('pending', _config) + INSERT INTO public.eql_v2_configuration (state, data) VALUES ('pending', _config) ON CONFLICT (state) WHERE state = 'pending' DO NOTHING; @@ -203,9 +197,9 @@ AS $$ -- if config empty delete -- or update the config IF _config #> array['tables'] = '{}' THEN - DELETE FROM cs_configuration_v1 WHERE state = 'pending'; + DELETE FROM public.eql_v2_configuration WHERE state = 'pending'; ELSE - UPDATE cs_configuration_v1 SET data = _config WHERE state = 'pending'; + UPDATE public.eql_v2_configuration SET data = _config WHERE state = 'pending'; END IF; -- exeunt @@ -214,14 +208,13 @@ AS $$ $$ LANGUAGE plpgsql; -DROP FUNCTION IF EXISTS cs_modify_index_v1(table_name text, column_name text, index_name text, cast_as text, opts jsonb); -CREATE FUNCTION cs_modify_index_v1(table_name text, column_name text, index_name text, cast_as text DEFAULT 'text', opts jsonb DEFAULT '{}') +CREATE FUNCTION eql_v2.modify_index(table_name text, column_name text, index_name text, cast_as text DEFAULT 'text', opts jsonb DEFAULT '{}') RETURNS jsonb AS $$ BEGIN - PERFORM cs_remove_index_v1(table_name, column_name, index_name); - RETURN cs_add_index_v1(table_name, column_name, index_name, cast_as, opts); + PERFORM eql_v2.remove_index(table_name, column_name, index_name); + RETURN eql_v2.add_index(table_name, column_name, index_name, cast_as, opts); END; $$ LANGUAGE plpgsql; @@ -231,50 +224,46 @@ $$ LANGUAGE plpgsql; -- -- Marks the currently `pending` configuration as `encrypting`. -- --- Validates the database schema and raises an exception if the configured columns are not of `jsonb` or `cs_encrypted_v1` type. +-- Validates the database schema and raises an exception if the configured columns are not `cs_encrypted_v2` type. -- -- Accepts an optional `force` parameter. -- If `force` is `true`, the schema validation is skipped. -- -- Raises an exception if the configuration is already `encrypting` or if there is no `pending` configuration to encrypt. -- -DROP FUNCTION IF EXISTS cs_encrypt_v1(); -CREATE FUNCTION cs_encrypt_v1(force boolean DEFAULT false) +CREATE FUNCTION eql_v2.encrypt() RETURNS boolean AS $$ BEGIN - IF EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'encrypting') THEN + IF EXISTS (SELECT FROM public.eql_v2_configuration c WHERE c.state = 'encrypting') THEN RAISE EXCEPTION 'An encryption is already in progress'; END IF; - IF NOT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'pending') THEN + IF NOT EXISTS (SELECT FROM public.eql_v2_configuration c WHERE c.state = 'pending') THEN RAISE EXCEPTION 'No pending configuration exists to encrypt'; END IF; - IF NOT force THEN - IF NOT cs_ready_for_encryption_v1() THEN - RAISE EXCEPTION 'Some pending columns do not have an encrypted target'; - END IF; + IF NOT eql_v2.ready_for_encryption() THEN + RAISE EXCEPTION 'Some pending columns do not have an encrypted target'; END IF; - UPDATE cs_configuration_v1 SET state = 'encrypting' WHERE state = 'pending'; + UPDATE public.eql_v2_configuration SET state = 'encrypting' WHERE state = 'pending'; RETURN true; END; $$ LANGUAGE plpgsql; -DROP FUNCTION IF EXISTS cs_activate_v1(); -CREATE FUNCTION cs_activate_v1() +CREATE FUNCTION eql_v2.activate() RETURNS boolean AS $$ BEGIN - IF EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'encrypting') THEN - UPDATE cs_configuration_v1 SET state = 'inactive' WHERE state = 'active'; - UPDATE cs_configuration_v1 SET state = 'active' WHERE state = 'encrypting'; + IF EXISTS (SELECT FROM public.eql_v2_configuration c WHERE c.state = 'encrypting') THEN + UPDATE public.eql_v2_configuration SET state = 'inactive' WHERE state = 'active'; + UPDATE public.eql_v2_configuration SET state = 'active' WHERE state = 'encrypting'; RETURN true; ELSE RAISE EXCEPTION 'No encrypting configuration exists to activate'; @@ -283,14 +272,13 @@ AS $$ $$ LANGUAGE plpgsql; -DROP FUNCTION IF EXISTS cs_discard_v1(); -CREATE FUNCTION cs_discard_v1() +CREATE FUNCTION eql_v2.discard() RETURNS boolean AS $$ BEGIN - IF EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'pending') THEN - DELETE FROM cs_configuration_v1 WHERE state = 'pending'; + IF EXISTS (SELECT FROM public.eql_v2_configuration c WHERE c.state = 'pending') THEN + DELETE FROM public.eql_v2_configuration WHERE state = 'pending'; RETURN true; ELSE RAISE EXCEPTION 'No pending configuration exists to discard'; @@ -299,9 +287,8 @@ AS $$ $$ LANGUAGE plpgsql; -DROP FUNCTION IF EXISTS cs_add_column_v1(table_name text, column_name text, cast_as text); -CREATE FUNCTION cs_add_column_v1(table_name text, column_name text, cast_as text DEFAULT 'text') +CREATE FUNCTION eql_v2.add_column(table_name text, column_name text, cast_as text DEFAULT 'text') RETURNS jsonb AS $$ DECLARE @@ -309,38 +296,39 @@ AS $$ _config jsonb; BEGIN -- set the active config - SELECT data INTO _config FROM cs_configuration_v1 WHERE state = 'active' OR state = 'pending' ORDER BY state DESC; + SELECT data INTO _config FROM public.eql_v2_configuration WHERE state = 'active' OR state = 'pending' ORDER BY state DESC; -- set default config - SELECT _cs_config_default(_config) INTO _config; + SELECT eql_v2.config_default(_config) INTO _config; -- if index exists IF _config #> array['tables', table_name] ? column_name THEN RAISE EXCEPTION 'Config exists for column: % %', table_name, column_name; END IF; - SELECT _cs_config_add_table(table_name, _config) INTO _config; + SELECT eql_v2.config_add_table(table_name, _config) INTO _config; - SELECT _cs_config_add_column(table_name, column_name, _config) INTO _config; + SELECT eql_v2.config_add_column(table_name, column_name, _config) INTO _config; - SELECT _cs_config_add_cast(table_name, column_name, cast_as, _config) INTO _config; + SELECT eql_v2.config_add_cast(table_name, column_name, cast_as, _config) INTO _config; -- create a new pending record if we don't have one - INSERT INTO cs_configuration_v1 (state, data) VALUES ('pending', _config) + INSERT INTO public.eql_v2_configuration (state, data) VALUES ('pending', _config) ON CONFLICT (state) WHERE state = 'pending' DO UPDATE SET data = _config; + PERFORM eql_v2.add_encrypted_constraint(table_name, column_name); + -- exeunt RETURN _config; END; $$ LANGUAGE plpgsql; -DROP FUNCTION IF EXISTS cs_remove_column_v1(table_name text, column_name text); -CREATE FUNCTION cs_remove_column_v1(table_name text, column_name text) +CREATE FUNCTION eql_v2.remove_column(table_name text, column_name text) RETURNS jsonb AS $$ DECLARE @@ -348,7 +336,7 @@ AS $$ _config jsonb; BEGIN -- set the active config - SELECT data INTO _config FROM cs_configuration_v1 WHERE state = 'active' OR state = 'pending' ORDER BY state DESC; + SELECT data INTO _config FROM public.eql_v2_configuration WHERE state = 'active' OR state = 'pending' ORDER BY state DESC; -- if no config IF _config IS NULL THEN @@ -366,7 +354,7 @@ AS $$ END IF; -- create a new pending record if we don't have one - INSERT INTO cs_configuration_v1 (state, data) VALUES ('pending', _config) + INSERT INTO public.eql_v2_configuration (state, data) VALUES ('pending', _config) ON CONFLICT (state) WHERE state = 'pending' DO NOTHING; @@ -382,11 +370,13 @@ AS $$ -- if config empty delete -- or update the config IF _config #> array['tables'] = '{}' THEN - DELETE FROM cs_configuration_v1 WHERE state = 'pending'; + DELETE FROM public.eql_v2_configuration WHERE state = 'pending'; ELSE - UPDATE cs_configuration_v1 SET data = _config WHERE state = 'pending'; + UPDATE public.eql_v2_configuration SET data = _config WHERE state = 'pending'; END IF; + PERFORM eql_v2.remove_encrypted_constraint(table_name, column_name); + -- exeunt RETURN _config; @@ -394,23 +384,20 @@ AS $$ $$ LANGUAGE plpgsql; -DROP FUNCTION IF EXISTS cs_refresh_encrypt_config(); -CREATE FUNCTION cs_refresh_encrypt_config() +CREATE FUNCTION eql_v2.reload_config() RETURNS void LANGUAGE sql STRICT PARALLEL SAFE BEGIN ATOMIC RETURN NULL; END; -DROP FUNCTION IF EXISTS cs_config(); --- -- A convenience function to return the configuration in a tabular format, allowing for easier filtering, and querying. -- Query using `SELECT * FROM cs_config();` -- -CREATE FUNCTION cs_config() RETURNS TABLE ( - state cs_configuration_state_v1, +CREATE FUNCTION eql_v2.config() RETURNS TABLE ( + state eql_v2_configuration_state, relation text, col_name text, decrypts_as text, @@ -421,7 +408,7 @@ BEGIN RETURN QUERY WITH tables AS ( SELECT config.state, tables.key AS table, tables.value AS config - FROM cs_configuration_v1 config, jsonb_each(data->'tables') tables + FROM public.eql_v2_configuration config, jsonb_each(data->'tables') tables WHERE config.data->>'v' = '1' ) SELECT @@ -432,4 +419,4 @@ BEGIN column_config.value->'indexes' FROM tables, jsonb_each(tables.config) column_config; END; -$$ LANGUAGE plpgsql; \ No newline at end of file +$$ LANGUAGE plpgsql; diff --git a/src/config/indexes.sql b/src/config/indexes.sql new file mode 100644 index 00000000..570a7291 --- /dev/null +++ b/src/config/indexes.sql @@ -0,0 +1,11 @@ +-- REQUIRE: src/schema.sql +-- REQUIRE: src/config/tables.sql + + +-- +-- Define partial indexes to ensure that there is only one active, pending and encrypting config at a time +-- +CREATE UNIQUE INDEX ON public.eql_v2_configuration (state) WHERE state = 'active'; +CREATE UNIQUE INDEX ON public.eql_v2_configuration (state) WHERE state = 'pending'; +CREATE UNIQUE INDEX ON public.eql_v2_configuration (state) WHERE state = 'encrypting'; + diff --git a/src/config/tables.sql b/src/config/tables.sql new file mode 100644 index 00000000..8fded8c5 --- /dev/null +++ b/src/config/tables.sql @@ -0,0 +1,15 @@ +-- REQUIRE: src/config/types.sql + +-- +-- +-- CREATE the eql_v2_configuration TABLE +-- +CREATE TABLE IF NOT EXISTS public.eql_v2_configuration +( + id bigint GENERATED ALWAYS AS IDENTITY, + state eql_v2_configuration_state NOT NULL DEFAULT 'pending', + data jsonb, + created_at timestamptz not null default current_timestamp, + PRIMARY KEY(id) +); + diff --git a/src/config/types.sql b/src/config/types.sql new file mode 100644 index 00000000..a0d5cc40 --- /dev/null +++ b/src/config/types.sql @@ -0,0 +1,26 @@ +-- +-- cs_configuration_data_v2 is a jsonb column that stores the actual configuration +-- +-- For some reason CREATE DOMAIN and CREATE TYPE do not support IF NOT EXISTS +-- Types cannot be dropped if used by a table, and we never drop the configuration table +-- DOMAIN constraints are added separately and not tied to DOMAIN creation +-- +-- DO $$ +-- BEGIN +-- IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'configuration_data') THEN +-- CREATE DOMAIN eql_v2.configuration_data AS JSONB; +-- END IF; +-- END +-- $$; + +-- +-- cs_configuration_state_v2 is an ENUM that defines the valid configuration states +-- -- +DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'eql_v2_configuration_state') THEN + CREATE TYPE public.eql_v2_configuration_state AS ENUM ('active', 'inactive', 'encrypting', 'pending'); + END IF; + END +$$; + diff --git a/src/crypto.sql b/src/crypto.sql new file mode 100644 index 00000000..f4364d1d --- /dev/null +++ b/src/crypto.sql @@ -0,0 +1,4 @@ +-- REQUIRE: src/schema.sql + +CREATE EXTENSION IF NOT EXISTS pgcrypto; + diff --git a/src/encrypted/aggregates.sql b/src/encrypted/aggregates.sql new file mode 100644 index 00000000..a6684f67 --- /dev/null +++ b/src/encrypted/aggregates.sql @@ -0,0 +1,48 @@ +-- REQUIRE: src/encrypted/types.sql +-- REQUIRE: src/ore/types.sql +-- REQUIRE: src/ore/functions.sql + +-- Aggregate functions for ORE + +CREATE FUNCTION eql_v2.min(a eql_v2_encrypted, b eql_v2_encrypted) + RETURNS eql_v2_encrypted +STRICT +AS $$ + BEGIN + PERFORM eql_v2.log('eql_v2.min'); + IF eql_v2.ore_64_8_v2(a) < eql_v2.ore_64_8_v2(b) THEN + RETURN a; + ELSE + RETURN b; + END IF; + END; +$$ LANGUAGE plpgsql; + + +CREATE AGGREGATE eql_v2.min(eql_v2_encrypted) +( + sfunc = eql_v2.min, + stype = eql_v2_encrypted +); + + +CREATE FUNCTION eql_v2.max(a eql_v2_encrypted, b eql_v2_encrypted) +RETURNS eql_v2_encrypted +STRICT +AS $$ + BEGIN + PERFORM eql_v2.log('eql_v2.max'); + IF eql_v2.ore_64_8_v2(a) > eql_v2.ore_64_8_v2(b) THEN + RETURN a; + ELSE + RETURN b; + END IF; + END; +$$ LANGUAGE plpgsql; + + +CREATE AGGREGATE eql_v2.max(eql_v2_encrypted) +( + sfunc = eql_v2.max, + stype = eql_v2_encrypted +); diff --git a/tests/aggregate-ore.sql b/src/encrypted/aggregates_test.sql similarity index 91% rename from tests/aggregate-ore.sql rename to src/encrypted/aggregates_test.sql index 631deaed..2825fc12 100644 --- a/tests/aggregate-ore.sql +++ b/src/encrypted/aggregates_test.sql @@ -1,11 +1,10 @@ \set ON_ERROR_STOP on -- create table -DROP TABLE IF EXISTS agg_test; CREATE TABLE agg_test ( plain_int integer, - enc_int cs_encrypted_v1 + enc_int eql_v2_encrypted ); -- Add data. These are saved from the psql query output connected to Proxy. @@ -17,19 +16,19 @@ INSERT INTO agg_test (plain_int, enc_int) VALUES ), ( 3, - '{"c": "mBbJyWl%QyVQT_N?b~OpQj!$J7B7H2CK@gB#`36H312|)kY;SeM7R*dAl5{R*U)AI+$~k7(JPvj;hmQK^F_}g^7Zs^WuYa^B(7y{V{&N2hzy", "i": {"c": "encrypted_int4", "t": "encrypted"}, "k": "ct", "m": null, "o": ["ccccccccb06565ebd23d6a4c3eee512713175e673c6d995ff5d9b1d3492fe8eb289c3eb95029025f5b71fc6e06632b4a1302980e433361c7999724dbdd052739258d9444b0fbd43cc61368e60f4b0d5aeca2aa85c1c89933b53afffcc4eb0632dca75f632bb9bc792d1dbd6bced6253291f0db134552d384e9e378f4f5890c31ca9d115965a0e8fbf13ad8d1d33f88d360d5e2f9680fb158f98158443ffc769cd9aac94380f05e3226b785f58006e5b9da6b8d86a7441a88fd848099a2400ef59b494b0c30013568dc1be9bba560565fccb49309ba2ec3edcff6f9d7a67b519b3754b37b0025dff7592a6117949a04043c100353289628884fe06cb2099e7b4b49abea9797a73ee0b85283a5b6f69bcf45f87e6cd6d45ecfd1633903270781173ed9d31a682bba0e54ff355f456bf0c468e378e41cb54fcc074ad40fb4448f6fec892c1ecda15a5efffb8dde3a3b282865ac436d7e43d48d4327c439956733697d3f5b02ead4805a7f905bdae24c1b35252e34939676a07ddb5454c3580c7d76d792a97988e35142f43667112432623eda5126e9af2592dd"], "v": 1}'::cs_encrypted_v1 + '{"c": "mBbJyWl%QyVQT_N?b~OpQj!$J7B7H2CK@gB#`36H312|)kY;SeM7R*dAl5{R*U)AI+$~k7(JPvj;hmQK^F_}g^7Zs^WuYa^B(7y{V{&N2hzy", "i": {"c": "encrypted_int4", "t": "encrypted"}, "k": "ct", "m": null, "o": ["ccccccccb06565ebd23d6a4c3eee512713175e673c6d995ff5d9b1d3492fe8eb289c3eb95029025f5b71fc6e06632b4a1302980e433361c7999724dbdd052739258d9444b0fbd43cc61368e60f4b0d5aeca2aa85c1c89933b53afffcc4eb0632dca75f632bb9bc792d1dbd6bced6253291f0db134552d384e9e378f4f5890c31ca9d115965a0e8fbf13ad8d1d33f88d360d5e2f9680fb158f98158443ffc769cd9aac94380f05e3226b785f58006e5b9da6b8d86a7441a88fd848099a2400ef59b494b0c30013568dc1be9bba560565fccb49309ba2ec3edcff6f9d7a67b519b3754b37b0025dff7592a6117949a04043c100353289628884fe06cb2099e7b4b49abea9797a73ee0b85283a5b6f69bcf45f87e6cd6d45ecfd1633903270781173ed9d31a682bba0e54ff355f456bf0c468e378e41cb54fcc074ad40fb4448f6fec892c1ecda15a5efffb8dde3a3b282865ac436d7e43d48d4327c439956733697d3f5b02ead4805a7f905bdae24c1b35252e34939676a07ddb5454c3580c7d76d792a97988e35142f43667112432623eda5126e9af2592dd"], "v": 1}'::jsonb::eql_v2_encrypted ), ( 5, - '{"c": "mBbKSqWLK6yl>o%G%&x+2$jdg7F`-R(^>R1Q^wGod8-FZ5C$xFI4dN?Ap114=77xPZ9!cKxE}qmyXrhx#K`4ztbUrysQrOFqON6bV{&N2hzy", "i": {"c": "encrypted_int4", "t": "encrypted"}, "k": "ct", "m": null, "o": ["ccccccccb065659dd23d6a4c3eee512713175e673c6d995ff5d9b1d3492fe8eb289c3eb95029025f5b71fc6e06632b4a1302980e433361c7999724dbdd052739258d9444b0fbd43cc61368e60f4b0d5aeca2aa85c1c89933b53afffcc4eb0632dca75f632bb9bc792d1dbd6bced6253291f0db134552d384bec7bfb23290d7559fd8637b85ca7510cca465570029734ef0319c77177913ad84f54852bed2e2a67b6dafcab3eb70d3a2592414a43acc03703083cf1fa1984dfc0719337d5de4eefd0d137588641a0d38c771b77ab07ebab3fc9bfd7469c4222e1a8edee71188eeb24bfffcd82f711156381d8068223e3d75f5ba8a958182bc46a0ab58c29872cd17e559ed0b935a445249dbac5b51438cebaf9d28d5c8b67cd99f990d5295c1e37470ce5b33fe01eaf31d84c9a08b267c0e9e1aadfcce7f9e2253ababa71eaf1fec309dc988e454717a3c2e3bffb1c546a7195ecf274eb7d691abcf46a61e34d4c63c45d48831dc23aa11f981de692926cd1d1d77a340c9e54baf62da61d5f88960a93e120d3828f4053577b93b536cc9b05c889dcf171865"], "v": 1}'::cs_encrypted_v1 + '{"c": "mBbKSqWLK6yl>o%G%&x+2$jdg7F`-R(^>R1Q^wGod8-FZ5C$xFI4dN?Ap114=77xPZ9!cKxE}qmyXrhx#K`4ztbUrysQrOFqON6bV{&N2hzy", "i": {"c": "encrypted_int4", "t": "encrypted"}, "k": "ct", "m": null, "o": ["ccccccccb065659dd23d6a4c3eee512713175e673c6d995ff5d9b1d3492fe8eb289c3eb95029025f5b71fc6e06632b4a1302980e433361c7999724dbdd052739258d9444b0fbd43cc61368e60f4b0d5aeca2aa85c1c89933b53afffcc4eb0632dca75f632bb9bc792d1dbd6bced6253291f0db134552d384bec7bfb23290d7559fd8637b85ca7510cca465570029734ef0319c77177913ad84f54852bed2e2a67b6dafcab3eb70d3a2592414a43acc03703083cf1fa1984dfc0719337d5de4eefd0d137588641a0d38c771b77ab07ebab3fc9bfd7469c4222e1a8edee71188eeb24bfffcd82f711156381d8068223e3d75f5ba8a958182bc46a0ab58c29872cd17e559ed0b935a445249dbac5b51438cebaf9d28d5c8b67cd99f990d5295c1e37470ce5b33fe01eaf31d84c9a08b267c0e9e1aadfcce7f9e2253ababa71eaf1fec309dc988e454717a3c2e3bffb1c546a7195ecf274eb7d691abcf46a61e34d4c63c45d48831dc23aa11f981de692926cd1d1d77a340c9e54baf62da61d5f88960a93e120d3828f4053577b93b536cc9b05c889dcf171865"], "v": 1}'::jsonb::eql_v2_encrypted ), ( 1, - '{"c": "mBbJSy$p0fHEK%aOAOYi4PTJN7B@a-j{+xl7tffjGTN<-Znt3Zge#lGAX^WHzU`7ml<4vRHLKxoB%}NN2hzy", "i": {"c": "encrypted_int4", "t": "encrypted"}, "k": "ct", "m": null, "o": ["ccccccccb0656502d23d6a4c3eee512713175e673c6d995ff5d9b1d3492fe8eb289c3eb95029025f5b71fc6e06632b4a1302980e433361c7999724dbdd052739258d9444b0fbd43cc61368e60f4b0d5aeca2aa85c1c89933b53afffcc4eb0632dca75f632bb9bc792d1dbd6bced6253291f0db134552d384250ca116ef329616ddb341917699b9ea48901124a15a4547be1ff7c672c0c1bc6bb17e2a141f46138fc314f4bf8a55068bf031bc48f038c379e54cfbb1c64eb223c18c87cd68a91fb031905e11d9478f158b561399b527038efc594bfd9fb19c963a2778b75215e1d8933b08df04d1c62742fd48a4de310792031a70ca4b157bc218ab3fbadc6dc14b939422023331c03bcf4b673c5d261a19c3d13155cbaa1b84e9e90e389fa6973dde07fba08c13847006707488e288ce780d59700197452ebc68d22032ab03f7b445e45ed7abb1af34955199440f7db2c969c60b1eb49cdcd75d5e8f7de37848ddebb40df8e14d4b92910e15fedac3f61f22ef430805ba1bbf5fccc9fe792e4c0353beee48ca03ef23c7d3fab19e9aa218aefb44e6c26d70"], "v": 1}'::cs_encrypted_v1 + '{"c": "mBbJSy$p0fHEK%aOAOYi4PTJN7B@a-j{+xl7tffjGTN<-Znt3Zge#lGAX^WHzU`7ml<4vRHLKxoB%}NN2hzy", "i": {"c": "encrypted_int4", "t": "encrypted"}, "k": "ct", "m": null, "o": ["ccccccccb0656502d23d6a4c3eee512713175e673c6d995ff5d9b1d3492fe8eb289c3eb95029025f5b71fc6e06632b4a1302980e433361c7999724dbdd052739258d9444b0fbd43cc61368e60f4b0d5aeca2aa85c1c89933b53afffcc4eb0632dca75f632bb9bc792d1dbd6bced6253291f0db134552d384250ca116ef329616ddb341917699b9ea48901124a15a4547be1ff7c672c0c1bc6bb17e2a141f46138fc314f4bf8a55068bf031bc48f038c379e54cfbb1c64eb223c18c87cd68a91fb031905e11d9478f158b561399b527038efc594bfd9fb19c963a2778b75215e1d8933b08df04d1c62742fd48a4de310792031a70ca4b157bc218ab3fbadc6dc14b939422023331c03bcf4b673c5d261a19c3d13155cbaa1b84e9e90e389fa6973dde07fba08c13847006707488e288ce780d59700197452ebc68d22032ab03f7b445e45ed7abb1af34955199440f7db2c969c60b1eb49cdcd75d5e8f7de37848ddebb40df8e14d4b92910e15fedac3f61f22ef430805ba1bbf5fccc9fe792e4c0353beee48ca03ef23c7d3fab19e9aa218aefb44e6c26d70"], "v": 1}'::jsonb::eql_v2_encrypted ), ( 3, - '{"c": "mBbLa7Cm?&jvpfcv1d3hep>s)76qzUbwUky&M&C3mjDG_os-_y0MRaMGl@&p#AOuusN|3Lu=mBCcg_V{&N2hzy", "i": {"c": "encrypted_int4", "t": "encrypted"}, "k": "ct", "m": null, "o": ["ccccccccb06565ebd23d6a4c3eee512713175e673c6d995ff5d9b1d3492fe8eb289c3eb95029025f5b71fc6e06632b4a1302980e433361c7999724dbdd052739258d9444b0fbd43cc61368e60f4b0d5aeca2aa85c1c89933b53afffcc4eb0632dca75f632bb9bc792d1dbd6bced6253291f0db134552d384e9e378f4f5890c31ca9d115965a0e8fb2c3c60ccce84ffc03bddb22b27a1ce278eec118496fd23f083ebb21bb4b83b89eda8c0bdea50debc5ec4f2b2d91b63a80d39386194ad9d129bee2f5168341cb41ed26dc03466cac5e2dbe7336fdb74c0d37d63b396033ce60002c9950f5ac2970dacf4caace2eef5b81544df88a7ef2a8d69550d25d39c678c8e43a3dcc2857018a2c979b45c6b19dabd28ae7388d62916e6742763d6484d1b45154e6c8e6a66e02b03f64b67ddef24747dded32e226e3a93d5d1a92d11e760403cad04a0dd07c14da336a409739e8bbeb3b3d6b92117fa2d2c941da4996ea61b29ca3fffb4594ddbeab7105a1b4c5e422ec5ab8154db545103d8c2889be2e4591198912446d8b33b8708a4cc959a1e0957dcae6a50c3"], "v": 1}'::cs_encrypted_v1 + '{"c": "mBbLa7Cm?&jvpfcv1d3hep>s)76qzUbwUky&M&C3mjDG_os-_y0MRaMGl@&p#AOuusN|3Lu=mBCcg_V{&N2hzy", "i": {"c": "encrypted_int4", "t": "encrypted"}, "k": "ct", "m": null, "o": ["ccccccccb06565ebd23d6a4c3eee512713175e673c6d995ff5d9b1d3492fe8eb289c3eb95029025f5b71fc6e06632b4a1302980e433361c7999724dbdd052739258d9444b0fbd43cc61368e60f4b0d5aeca2aa85c1c89933b53afffcc4eb0632dca75f632bb9bc792d1dbd6bced6253291f0db134552d384e9e378f4f5890c31ca9d115965a0e8fb2c3c60ccce84ffc03bddb22b27a1ce278eec118496fd23f083ebb21bb4b83b89eda8c0bdea50debc5ec4f2b2d91b63a80d39386194ad9d129bee2f5168341cb41ed26dc03466cac5e2dbe7336fdb74c0d37d63b396033ce60002c9950f5ac2970dacf4caace2eef5b81544df88a7ef2a8d69550d25d39c678c8e43a3dcc2857018a2c979b45c6b19dabd28ae7388d62916e6742763d6484d1b45154e6c8e6a66e02b03f64b67ddef24747dded32e226e3a93d5d1a92d11e760403cad04a0dd07c14da336a409739e8bbeb3b3d6b92117fa2d2c941da4996ea61b29ca3fffb4594ddbeab7105a1b4c5e422ec5ab8154db545103d8c2889be2e4591198912446d8b33b8708a4cc959a1e0957dcae6a50c3"], "v": 1}'::jsonb::eql_v2_encrypted ) ; @@ -37,16 +36,16 @@ INSERT INTO agg_test (plain_int, enc_int) VALUES DO $$ BEGIN -- min null finds null - ASSERT ((SELECT cs_min_v1(enc_int) FROM agg_test where enc_int IS NULL) IS NULL); + ASSERT ((SELECT eql_v2.min(enc_int) FROM agg_test where enc_int IS NULL) IS NULL); -- min enc_int finds the minimum (1) - ASSERT ((SELECT enc_int FROM agg_test WHERE plain_int = 1) = (SELECT cs_min_v1(enc_int) FROM agg_test)); + ASSERT ((SELECT enc_int FROM agg_test WHERE plain_int = 1) = (SELECT eql_v2.min(enc_int) FROM agg_test)); -- max null finds null - ASSERT ((SELECT cs_max_v1(enc_int) FROM agg_test where enc_int IS NULL) IS NULL); + ASSERT ((SELECT eql_v2.max(enc_int) FROM agg_test where enc_int IS NULL) IS NULL); -- max enc_int finds the maximum (5) - ASSERT ((SELECT enc_int FROM agg_test WHERE plain_int = 5) = (SELECT cs_max_v1(enc_int) FROM agg_test)); + ASSERT ((SELECT enc_int FROM agg_test WHERE plain_int = 5) = (SELECT eql_v2.max(enc_int) FROM agg_test)); END; $$ LANGUAGE plpgsql; @@ -54,7 +53,7 @@ $$ LANGUAGE plpgsql; INSERT INTO agg_test (plain_int, enc_int) VALUES ( 3, - '{"c": "mBbLa7Cm?&jvpfcv1d3hep>s)76qzUbwUky&M&C3mjDG_os-_y0MRaMGl@&p#AOuusN|3Lu=mBCcg_V{&N2hzy", "i": {"c": "encrypted_int4", "t": "encrypted"}, "k": "ct", "m": null, "v": 1}'::cs_encrypted_v1 + '{"c": "mBbLa7Cm?&jvpfcv1d3hep>s)76qzUbwUky&M&C3mjDG_os-_y0MRaMGl@&p#AOuusN|3Lu=mBCcg_V{&N2hzy", "i": {"c": "encrypted_int4", "t": "encrypted"}, "k": "ct", "m": null, "v": 1}'::jsonb::eql_v2_encrypted ); -- run exceptional case @@ -63,7 +62,7 @@ DO $$ error_message text; BEGIN -- min enc_int raises exception - SELECT cs_min_v1(enc_int) FROM agg_test; + SELECT eql_v2.min(enc_int) FROM agg_test; EXCEPTION WHEN others THEN error_message := SQLERRM; @@ -82,7 +81,7 @@ DO $$ error_message text; BEGIN -- max enc_int raises exception - SELECT cs_max_v1(enc_int) FROM agg_test; + SELECT eql_v2.max(enc_int) FROM agg_test; EXCEPTION WHEN others THEN error_message := SQLERRM; diff --git a/src/encrypted/casts.sql b/src/encrypted/casts.sql new file mode 100644 index 00000000..158b49a6 --- /dev/null +++ b/src/encrypted/casts.sql @@ -0,0 +1,63 @@ + +-- REQUIRE: src/encrypted/types.sql + + +-- +-- Convert jsonb to eql_v2.encrypted +-- + +CREATE FUNCTION eql_v2.to_encrypted(data jsonb) +RETURNS public.eql_v2_encrypted AS $$ +BEGIN + RETURN ROW(data)::public.eql_v2_encrypted; +END; +$$ LANGUAGE plpgsql; + +-- +-- Cast jsonb to eql_v2.encrypted +-- + +CREATE CAST (jsonb AS public.eql_v2_encrypted) + WITH FUNCTION eql_v2.to_encrypted(jsonb) AS ASSIGNMENT; + + +-- +-- Convert text to eql_v2.encrypted +-- + +CREATE FUNCTION eql_v2.to_encrypted(data text) +RETURNS public.eql_v2_encrypted AS $$ +BEGIN + RETURN ROW(data::jsonb)::public.eql_v2_encrypted; +END; +$$ LANGUAGE plpgsql; + +-- +-- Cast text to eql_v2.encrypted +-- + +CREATE CAST (text AS public.eql_v2_encrypted) + WITH FUNCTION eql_v2.to_encrypted(text) AS ASSIGNMENT; + + + +-- +-- Convert eql_v2.encrypted to jsonb +-- + +CREATE FUNCTION eql_v2.to_jsonb(e public.eql_v2_encrypted) +RETURNS jsonb AS $$ +BEGIN + RETURN e.data; +END; +$$ LANGUAGE plpgsql; + +-- +-- Cast eql_v2.encrypted to jsonb +-- + +CREATE CAST (public.eql_v2_encrypted AS jsonb) + WITH FUNCTION eql_v2.to_jsonb(public.eql_v2_encrypted) AS ASSIGNMENT; + + + diff --git a/src/encrypted/constraints.sql b/src/encrypted/constraints.sql new file mode 100644 index 00000000..6b02d067 --- /dev/null +++ b/src/encrypted/constraints.sql @@ -0,0 +1,76 @@ +-- REQUIRE: src/schema.sql +-- REQUIRE: src/encrypted/types.sql +-- REQUIRE: src/encrypted/functions.sql + + +-- Should include an ident field +CREATE FUNCTION eql_v2._encrypted_check_i(val jsonb) + RETURNS boolean +AS $$ + BEGIN + IF val ? 'i' THEN + RETURN true; + END IF; + RAISE 'Encrypted column missing ident (i) field: %', val; + END; +$$ LANGUAGE plpgsql; + + +-- Ident field should include table and column +CREATE FUNCTION eql_v2._encrypted_check_i_ct(val jsonb) + RETURNS boolean +AS $$ + BEGIN + IF (val->'i' ?& array['t', 'c']) THEN + RETURN true; + END IF; + RAISE 'Encrypted column ident (i) missing table (t) or column (c) fields: %', val; + END; +$$ LANGUAGE plpgsql; + +-- -- Should include a version field +CREATE FUNCTION eql_v2._encrypted_check_v(val jsonb) + RETURNS boolean +AS $$ + BEGIN + IF (val ? 'v') THEN + RETURN true; + END IF; + RAISE 'Encrypted column missing version (v) field: %', val; + END; +$$ LANGUAGE plpgsql; + + +-- -- Should include a ciphertext field +CREATE FUNCTION eql_v2._encrypted_check_c(val jsonb) + RETURNS boolean +AS $$ + BEGIN + IF (val ? 'c') THEN + RETURN true; + END IF; + RAISE 'Encrypted column missing ciphertext (c) field: %', val; + END; +$$ LANGUAGE plpgsql; + + +CREATE FUNCTION eql_v2.check_encrypted(val jsonb) + RETURNS BOOLEAN +LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +BEGIN ATOMIC + RETURN ( + eql_v2._encrypted_check_v(val) AND + eql_v2._encrypted_check_c(val) AND + eql_v2._encrypted_check_i(val) AND + eql_v2._encrypted_check_i_ct(val) + ); +END; + + +CREATE FUNCTION eql_v2.check_encrypted(val eql_v2_encrypted) + RETURNS BOOLEAN +LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +BEGIN ATOMIC + RETURN eql_v2.check_encrypted(val.data); +END; + diff --git a/src/encrypted/constraints_test.sql b/src/encrypted/constraints_test.sql new file mode 100644 index 00000000..1ca47752 --- /dev/null +++ b/src/encrypted/constraints_test.sql @@ -0,0 +1,48 @@ +\set ON_ERROR_STOP on + +SELECT create_table_with_encrypted(); + +DO $$ + BEGIN + -- insert without constraint works + INSERT INTO encrypted(e) VALUES ('{}'::jsonb::eql_v2_encrypted); + + -- delete the data + PERFORM create_table_with_encrypted(); + + -- add constraint + PERFORM eql_v2.add_encrypted_constraint('encrypted', 'e'); + + PERFORM assert_exception( + 'Constraint catches invalid eql_v2_encrypted', + 'INSERT INTO encrypted (e) VALUES (''{}''::jsonb::eql_v2_encrypted)'); + + END; +$$ LANGUAGE plpgsql; + + +DO $$ + BEGIN + -- reset the table + PERFORM create_table_with_encrypted(); + + -- add constraint + PERFORM eql_v2.add_encrypted_constraint('encrypted', 'e'); + + PERFORM assert_exception( + 'Constraint catches invalid eql_v2_encrypted', + 'INSERT INTO encrypted (e) VALUES (''{}''::jsonb::eql_v2_encrypted)'); + + PERFORM eql_v2.remove_encrypted_constraint('encrypted', 'e'); + + PERFORM assert_result( + 'Insert invalid data without constraint', + 'INSERT INTO encrypted (e) VALUES (''{}''::jsonb::eql_v2_encrypted) RETURNING id'); + + END; +$$ LANGUAGE plpgsql; + + + + + diff --git a/src/encrypted/functions.sql b/src/encrypted/functions.sql new file mode 100644 index 00000000..82adc311 --- /dev/null +++ b/src/encrypted/functions.sql @@ -0,0 +1,75 @@ +-- REQUIRE: src/encrypted/types.sql +-- REQUIRE: src/match/types.sql +-- REQUIRE: src/ore/types.sql +-- REQUIRE: src/unique/types.sql + + + +CREATE FUNCTION eql_v2.ciphertext(val jsonb) + RETURNS text + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + IF val ? 'c' THEN + RETURN val->>'c'; + END IF; + RAISE 'Expected a ciphertext (c) value in json: %', val; + END; +$$ LANGUAGE plpgsql; + + + +CREATE FUNCTION eql_v2.ciphertext(val eql_v2_encrypted) + RETURNS text + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN eql_v2.ciphertext(val.data); + END; +$$ LANGUAGE plpgsql; + + + +CREATE FUNCTION eql_v2._first_grouped_value(jsonb, jsonb) +RETURNS jsonb AS $$ + SELECT COALESCE($1, $2); +$$ LANGUAGE sql IMMUTABLE; + + +CREATE AGGREGATE eql_v2.grouped_value(jsonb) ( + SFUNC = eql_v2._first_grouped_value, + STYPE = jsonb +); + + +-- +-- Adds eql_v2.check_encrypted constraint to the column_name in table_name +-- +-- Executes the ALTER TABLE statement +-- `ALTER TABLE {table_name} ADD CONSTRAINT eql_v2_encrypted_check_{column_name} CHECK (eql_v2.check_encrypted({column_name}))` +-- +-- +CREATE FUNCTION eql_v2.add_encrypted_constraint(table_name TEXT, column_name TEXT) + RETURNS void +AS $$ + BEGIN + EXECUTE format('ALTER TABLE %I ADD CONSTRAINT eql_v2_encrypted_check_%I CHECK (eql_v2.check_encrypted(%I))', table_name, column_name, column_name); + END; +$$ LANGUAGE plpgsql; + + +-- +-- Removes the eql_v2.check_encrypted constraint from the column_name in table_name +-- +-- Executes the ALTER TABLE statement +-- `ALTER TABLE {table_name} DROP CONSTRAINT eql_v2_encrypted_check_{column_name}` +-- +CREATE FUNCTION eql_v2.remove_encrypted_constraint(table_name TEXT, column_name TEXT) + RETURNS void +AS $$ + BEGIN + EXECUTE format('ALTER TABLE %I DROP CONSTRAINT IF EXISTS eql_v2_encrypted_check_%I', table_name, column_name); + END; +$$ LANGUAGE plpgsql; + + diff --git a/src/encrypted/types.sql b/src/encrypted/types.sql new file mode 100644 index 00000000..82695ed1 --- /dev/null +++ b/src/encrypted/types.sql @@ -0,0 +1,42 @@ +-- REQUIRE: src/schema.sql + +-- eql_v2_encrypted is a column type +-- defined as jsonb for maximum portability of encrypted data +-- defined in the public schema as it cannot be dropped if in use +-- DO $$ +-- BEGIN +-- IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'eql_v2_encrypted') THEN +-- CREATE DOMAIN public.eql_v2_encrypted AS jsonb; +-- END IF; +-- END +-- $$; + + +-- eql_v2.encrypted is an internal composite type +-- eql_v2_encrypted data is cast to eql_v2.encrypted for use in EQL functions + +-- +-- Create an eql_v2_encrypted type in the public schema +-- Public schema allows the EQL schema to be dropped and recreated without impacting the type +-- Customer data may be using this type for encrypted data +-- +-- DO NOT DROP UNLESS ABSOLUTELY POSITIVE NO DATA IS USING IT +-- +DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'eql_v2_encrypted') THEN + CREATE TYPE public.eql_v2_encrypted AS ( + data jsonb + ); + END IF; + END +$$; + + + + + + + + + diff --git a/sql/030-encryptindex.sql b/src/encryptindex/functions.sql similarity index 67% rename from sql/030-encryptindex.sql rename to src/encryptindex/functions.sql index 95ae358b..96c8d2e6 100644 --- a/sql/030-encryptindex.sql +++ b/src/encryptindex/functions.sql @@ -1,9 +1,8 @@ -- Return the diff of two configurations -- Returns the set of keys in a that have different values to b -- The json comparison is on object values held by the key -DROP FUNCTION IF EXISTS _cs_diff_config_v1(a JSONB, b JSONB); -CREATE FUNCTION _cs_diff_config_v1(a JSONB, b JSONB) +CREATE FUNCTION eql_v2.diff_config(a JSONB, b JSONB) RETURNS TABLE(table_name TEXT, column_name TEXT) IMMUTABLE STRICT PARALLEL SAFE AS $$ @@ -34,9 +33,8 @@ $$ LANGUAGE plpgsql; -- Returns the set of columns with pending configuration changes -- Compares the columns in pending configuration that do not match the active config -DROP FUNCTION IF EXISTS cs_select_pending_columns_v1(); -CREATE FUNCTION cs_select_pending_columns_v1() +CREATE FUNCTION eql_v2.select_pending_columns() RETURNS TABLE(table_name TEXT, column_name TEXT) AS $$ DECLARE @@ -44,14 +42,14 @@ AS $$ pending JSONB; config_id BIGINT; BEGIN - SELECT data INTO active FROM cs_configuration_v1 WHERE state = 'active'; + SELECT data INTO active FROM eql_v2_configuration WHERE state = 'active'; -- set default config IF active IS NULL THEN active := '{}'; END IF; - SELECT id, data INTO config_id, pending FROM cs_configuration_v1 WHERE state = 'pending'; + SELECT id, data INTO config_id, pending FROM eql_v2_configuration WHERE state = 'pending'; -- set default config IF config_id IS NULL THEN @@ -59,22 +57,21 @@ AS $$ END IF; RETURN QUERY - SELECT d.table_name, d.column_name FROM _cs_diff_config_v1(active, pending) as d; + SELECT d.table_name, d.column_name FROM eql_v2.diff_config(active, pending) as d; END; $$ LANGUAGE plpgsql; -- -- Returns the target columns with pending configuration -- --- A `pending` column may be either a plaintext variant or cs_encrypted_v1. --- A `target` column is always of type cs_encrypted_v1 +-- A `pending` column may be either a plaintext variant or eql_v2_encrypted. +-- A `target` column is always of type eql_v2_encrypted -- -- On initial encryption from plaintext the target column will be `{column_name}_encrypted ` -- OR NULL if the column does not exist -- -DROP FUNCTION IF EXISTS cs_select_target_columns_v1(); -CREATE FUNCTION cs_select_target_columns_v1() +CREATE FUNCTION eql_v2.select_target_columns() RETURNS TABLE(table_name TEXT, column_name TEXT, target_column TEXT) STABLE STRICT PARALLEL SAFE AS $$ @@ -83,46 +80,44 @@ AS $$ c.column_name, s.column_name as target_column FROM - cs_select_pending_columns_v1() c + eql_v2.select_pending_columns() c LEFT JOIN information_schema.columns s ON s.table_name = c.table_name AND (s.column_name = c.column_name OR s.column_name = c.column_name || '_encrypted') AND - (s.domain_name = 'cs_encrypted_v1' OR s.data_type = 'jsonb'); + s.udt_name = 'eql_v2_encrypted'; $$ LANGUAGE sql; -- -- Returns true if all pending columns have a target (encrypted) column -DROP FUNCTION IF EXISTS cs_ready_for_encryption_v1(); -CREATE FUNCTION cs_ready_for_encryption_v1() +CREATE FUNCTION eql_v2.ready_for_encryption() RETURNS BOOLEAN STABLE STRICT PARALLEL SAFE AS $$ SELECT EXISTS ( SELECT * - FROM cs_select_target_columns_v1() AS c + FROM eql_v2.select_target_columns() AS c WHERE c.target_column IS NOT NULL); $$ LANGUAGE sql; -- --- Creates cs_encrypted_v1 columns for any plaintext columns with pending configuration +-- Creates eql_v2_encrypted columns for any plaintext columns with pending configuration -- The new column name is `{column_name}_encrypted` -- -- Executes the ALTER TABLE statement --- `ALTER TABLE {target_table} ADD COLUMN {column_name}_encrypted cs_encrypted_v1;` +-- `ALTER TABLE {target_table} ADD COLUMN {column_name}_encrypted eql_v2_encrypted;` -- -DROP FUNCTION IF EXISTS cs_create_encrypted_columns_v1(); -CREATE FUNCTION cs_create_encrypted_columns_v1() +CREATE FUNCTION eql_v2.create_encrypted_columns() RETURNS TABLE(table_name TEXT, column_name TEXT) AS $$ BEGIN FOR table_name, column_name IN - SELECT c.table_name, (c.column_name || '_encrypted') FROM cs_select_target_columns_v1() AS c WHERE c.target_column IS NULL + SELECT c.table_name, (c.column_name || '_encrypted') FROM eql_v2.select_target_columns() AS c WHERE c.target_column IS NULL LOOP - EXECUTE format('ALTER TABLE %I ADD column %I cs_encrypted_v1', table_name, column_name); + EXECUTE format('ALTER TABLE %I ADD column %I eql_v2_encrypted;', table_name, column_name); RETURN NEXT; END LOOP; END; @@ -130,7 +125,7 @@ $$ LANGUAGE plpgsql; -- --- Renames plaintext and cs_encrypted_v1 columns created for the initial encryption. +-- Renames plaintext and eql_v2_encrypted columns created for the initial encryption. -- The source plaintext column is renamed to `{column_name}_plaintext` -- The target encrypted column is renamed from `{column_name}_encrypted` to `{column_name}` -- @@ -138,14 +133,13 @@ $$ LANGUAGE plpgsql; -- `ALTER TABLE {target_table} RENAME COLUMN {column_name} TO {column_name}_plaintext; -- `ALTER TABLE {target_table} RENAME COLUMN {column_name}_encrypted TO {column_name};` -- -DROP FUNCTION IF EXISTS cs_rename_encrypted_columns_v1(); -CREATE FUNCTION cs_rename_encrypted_columns_v1() +CREATE FUNCTION eql_v2.rename_encrypted_columns() RETURNS TABLE(table_name TEXT, column_name TEXT, target_column TEXT) AS $$ BEGIN FOR table_name, column_name, target_column IN - SELECT * FROM cs_select_target_columns_v1() as c WHERE c.target_column = c.column_name || '_encrypted' + SELECT * FROM eql_v2.select_target_columns() as c WHERE c.target_column = c.column_name || '_encrypted' LOOP EXECUTE format('ALTER TABLE %I RENAME %I TO %I;', table_name, column_name, column_name || '_plaintext'); EXECUTE format('ALTER TABLE %I RENAME %I TO %I;', table_name, target_column, column_name); @@ -155,16 +149,15 @@ AS $$ $$ LANGUAGE plpgsql; -DROP FUNCTION IF EXISTS cs_count_encrypted_with_active_config_v1(table_name TEXT, column_name TEXT); -CREATE FUNCTION cs_count_encrypted_with_active_config_v1(table_name TEXT, column_name TEXT) +CREATE FUNCTION eql_v2.count_encrypted_with_active_config(table_name TEXT, column_name TEXT) RETURNS BIGINT AS $$ DECLARE result BIGINT; BEGIN EXECUTE format( - 'SELECT COUNT(%I) FROM %s t WHERE %I->>%L = (SELECT id::TEXT FROM cs_configuration_v1 WHERE state = %L)', + 'SELECT COUNT(%I) FROM %s t WHERE %I->>%L = (SELECT id::TEXT FROM eql_v2_configuration WHERE state = %L)', column_name, table_name, column_name, 'v', 'active' ) INTO result; diff --git a/tests/encryptindex.sql b/src/encryptindex/functions_test.sql similarity index 52% rename from tests/encryptindex.sql rename to src/encryptindex/functions_test.sql index 26d3e420..2e35ebdd 100644 --- a/tests/encryptindex.sql +++ b/src/encryptindex/functions_test.sql @@ -5,7 +5,7 @@ -- Alter table from config -- -- ----------------------------------------------- -TRUNCATE TABLE cs_configuration_v1; +TRUNCATE TABLE eql_v2_configuration; -- Create a table with a plaintext column DROP TABLE IF EXISTS users; @@ -16,7 +16,7 @@ CREATE TABLE users PRIMARY KEY(id) ); -INSERT INTO cs_configuration_v1 (data) VALUES ( +INSERT INTO eql_v2_configuration (data) VALUES ( '{ "v": 1, "tables": { @@ -36,21 +36,21 @@ DO $$ BEGIN -- the column is pending encryptindexing - ASSERT (SELECT EXISTS (SELECT * FROM cs_select_pending_columns_v1() AS c WHERE c.column_name = 'name')); + ASSERT (SELECT EXISTS (SELECT * FROM eql_v2.select_pending_columns() AS c WHERE c.column_name = 'name')); -- the target column does not exist - ASSERT (SELECT EXISTS (SELECT * FROM cs_select_target_columns_v1() AS c WHERE c.target_column IS NULL)); + ASSERT (SELECT EXISTS (SELECT * FROM eql_v2.select_target_columns() AS c WHERE c.target_column IS NULL)); -- Add the vtha_encrypted column to the table - PERFORM cs_create_encrypted_columns_v1(); + PERFORM eql_v2.create_encrypted_columns(); ASSERT (SELECT EXISTS (SELECT * FROM information_schema.columns s WHERE s.column_name = 'name_encrypted')); - -- rename columns - PERFORM cs_rename_encrypted_columns_v1(); + -- -- rename columns + PERFORM eql_v2.rename_encrypted_columns(); ASSERT (SELECT EXISTS (SELECT * FROM information_schema.columns s WHERE s.column_name = 'name_plaintext')); - ASSERT (SELECT EXISTS (SELECT * FROM information_schema.columns s WHERE s.column_name = 'name' and s.domain_name = 'cs_encrypted_v1')); + ASSERT (SELECT EXISTS (SELECT * FROM information_schema.columns s WHERE s.column_name = 'name' and s.udt_name = 'eql_v2_encrypted')); ASSERT (SELECT NOT EXISTS (SELECT * FROM information_schema.columns s WHERE s.column_name = 'name_encrypted')); END; $$ LANGUAGE plpgsql; @@ -60,7 +60,7 @@ $$ LANGUAGE plpgsql; -- Create multiple columns -- -- ----------------------------------------------- -TRUNCATE TABLE cs_configuration_v1; +TRUNCATE TABLE eql_v2_configuration; -- Create a table with multiple plaintext columns DROP TABLE IF EXISTS users; @@ -72,7 +72,7 @@ CREATE TABLE users PRIMARY KEY(id) ); -INSERT INTO cs_configuration_v1 (data) VALUES ( +INSERT INTO eql_v2_configuration (data) VALUES ( '{ "v": 1, "tables": { @@ -99,13 +99,13 @@ DO $$ BEGIN -- the column is pending encryptindexing - ASSERT (SELECT EXISTS (SELECT * FROM cs_select_pending_columns_v1() AS c WHERE c.column_name = 'name')); + ASSERT (SELECT EXISTS (SELECT * FROM eql_v2.select_pending_columns() AS c WHERE c.column_name = 'name')); -- the target column does not exisgt - ASSERT (SELECT EXISTS (SELECT * FROM cs_select_target_columns_v1() AS c WHERE c.target_column IS NULL)); + ASSERT (SELECT EXISTS (SELECT * FROM eql_v2.select_target_columns() AS c WHERE c.target_column IS NULL)); -- create column - PERFORM cs_create_encrypted_columns_v1(); + PERFORM eql_v2.create_encrypted_columns(); ASSERT (SELECT EXISTS (SELECT * FROM information_schema.columns s WHERE s.column_name = 'name_encrypted')); ASSERT (SELECT EXISTS (SELECT * FROM information_schema.columns s WHERE s.column_name = 'email_encrypted')); @@ -113,60 +113,6 @@ DO $$ $$ LANGUAGE plpgsql; --- ----------------------------------------------- --- Start encryptindexing with no target table --- --- The schema should be validated first. --- Users table does not exist, so should fail. --- ----------------------------------------------- -DROP TABLE IF EXISTS users; -TRUNCATE TABLE cs_configuration_v1; - - -DO $$ - BEGIN - PERFORM cs_add_index_v1('users', 'name', 'match'); - - BEGIN - PERFORM cs_encrypt_v1(); - RAISE NOTICE 'Missing users table. Encrypt should have failed.'; - ASSERT false; -- skipped by exception - EXCEPTION - WHEN OTHERS THEN - ASSERT true; - END; - -- configuration state should not be changed - ASSERT (SELECT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'pending')); - ASSERT (SELECT NOT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'encrypting')); - - END; -$$ LANGUAGE plpgsql; - - --- ----------------------------------------------- --- FORCE start encryptindexing with no target table --- --- Schema validation is skipped --- ----------------------------------------------- -DROP TABLE IF EXISTS users; -TRUNCATE TABLE cs_configuration_v1; - - -DO $$ - BEGIN - PERFORM cs_add_index_v1('users', 'name', 'match'); - - PERFORM cs_encrypt_v1(true); - RAISE NOTICE 'Missing users table. Encrypt should have failed.'; - - -- configuration state should be changed - ASSERT (SELECT NOT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'pending')); - ASSERT (SELECT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'encrypting')); - - END; -$$ LANGUAGE plpgsql; - - -- ----------------------------------------------- -- With existing active config -- and an updated schema @@ -174,10 +120,10 @@ $$ LANGUAGE plpgsql; -- The active config is unchanged -- The pending config should now be encrypting -- ----------------------------------------------- -TRUNCATE TABLE cs_configuration_v1; +TRUNCATE TABLE eql_v2_configuration; -- create an active configuration -INSERT INTO cs_configuration_v1 (state, data) VALUES ( +INSERT INTO eql_v2_configuration (state, data) VALUES ( 'active', '{ "v": 1, @@ -200,7 +146,7 @@ CREATE TABLE users ( id bigint GENERATED ALWAYS AS IDENTITY, name TEXT, - name_encrypted cs_encrypted_v1, + name_encrypted eql_v2_encrypted, PRIMARY KEY(id) ); @@ -208,12 +154,12 @@ CREATE TABLE users -- An encrypting config should exist DO $$ BEGIN - PERFORM cs_add_index_v1('users', 'name', 'match'); - PERFORM cs_encrypt_v1(); + PERFORM eql_v2.add_index('users', 'name', 'match'); + PERFORM eql_v2.encrypt(); - ASSERT (SELECT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'active')); - ASSERT (SELECT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'encrypting')); - ASSERT (SELECT NOT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'pending')); + ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'active')); + ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'encrypting')); + ASSERT (SELECT NOT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'pending')); END; $$ LANGUAGE plpgsql; @@ -224,10 +170,10 @@ $$ LANGUAGE plpgsql; -- The active config is unchanged -- The pending config should now be encrypting -- ----------------------------------------------- -TRUNCATE TABLE cs_configuration_v1; +TRUNCATE TABLE eql_v2_configuration; -- create an active configuration -INSERT INTO cs_configuration_v1 (state, data) VALUES ( +INSERT INTO eql_v2_configuration (state, data) VALUES ( 'active', '{ "v": 1, @@ -250,7 +196,7 @@ CREATE TABLE users ( id bigint GENERATED ALWAYS AS IDENTITY, name TEXT, - name_encrypted jsonb, + name_encrypted eql_v2_encrypted, PRIMARY KEY(id) ); @@ -258,12 +204,12 @@ CREATE TABLE users -- An encrypting config should exist DO $$ BEGIN - PERFORM cs_add_index_v1('users', 'name', 'match'); - PERFORM cs_encrypt_v1(); + PERFORM eql_v2.add_index('users', 'name', 'match'); + PERFORM eql_v2.encrypt(); - ASSERT (SELECT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'active')); - ASSERT (SELECT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'encrypting')); - ASSERT (SELECT NOT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'pending')); + ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'active')); + ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'encrypting')); + ASSERT (SELECT NOT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'pending')); END; $$ LANGUAGE plpgsql; @@ -274,10 +220,10 @@ $$ LANGUAGE plpgsql; -- The active config is now inactive -- The encrypting config should now be active -- ----------------------------------------------- -TRUNCATE TABLE cs_configuration_v1; +TRUNCATE TABLE eql_v2_configuration; -- create an active configuration -INSERT INTO cs_configuration_v1 (state, data) VALUES ( +INSERT INTO eql_v2_configuration (state, data) VALUES ( 'active', '{ "v": 1, @@ -301,22 +247,22 @@ CREATE TABLE users ( id bigint GENERATED ALWAYS AS IDENTITY, name TEXT, - name_encrypted cs_encrypted_v1, + name_encrypted eql_v2_encrypted, PRIMARY KEY(id) ); -- An encrypting config should exist DO $$ BEGIN - PERFORM cs_add_index_v1('users', 'name', 'match'); + PERFORM eql_v2.add_index('users', 'name', 'match'); - PERFORM cs_encrypt_v1(); -- need to encrypt first - PERFORM cs_activate_v1(); + PERFORM eql_v2.encrypt(); -- need to encrypt first + PERFORM eql_v2.activate(); - ASSERT (SELECT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'active')); - ASSERT (SELECT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'inactive')); - ASSERT (SELECT NOT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'encrypting')); - ASSERT (SELECT NOT EXISTS (SELECT FROM cs_configuration_v1 c WHERE c.state = 'pending')); + ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'active')); + ASSERT (SELECT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'inactive')); + ASSERT (SELECT NOT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'encrypting')); + ASSERT (SELECT NOT EXISTS (SELECT FROM eql_v2_configuration c WHERE c.state = 'pending')); END; $$ LANGUAGE plpgsql; diff --git a/src/jsonb/functions.sql b/src/jsonb/functions.sql new file mode 100644 index 00000000..877ac3a3 --- /dev/null +++ b/src/jsonb/functions.sql @@ -0,0 +1,278 @@ +-- REQUIRE: src/schema.sql +-- REQUIRE: src/encrypted/types.sql + +-- The jsonpath operators @? and @@ suppress the following errors: +-- missing object field or array element, +-- unexpected JSON item type, +-- datetime and numeric errors. +-- The jsonpath-related functions described below can also be told to suppress these types of errors. +-- This behavior might be helpful when searching JSON document collections of varying structure. + + + +-- +-- +-- Returns the stevec encrypted element matching the selector +-- +-- If the selector is not found, the function returns NULL +-- If the selector is found, the function returns the matching element +-- +-- Array elements use the same selector +-- Multiple matching elements are wrapped into an eql_v2_encrypted with an array flag +-- +-- + +CREATE FUNCTION eql_v2.jsonb_path_query(val jsonb, selector text) + RETURNS SETOF eql_v2_encrypted + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + DECLARE + sv eql_v2_encrypted[]; + found jsonb[]; + e jsonb; + ary boolean; + BEGIN + + IF val IS NULL THEN + RETURN NEXT NULL; + END IF; + + sv := eql_v2.ste_vec(val); + + FOR idx IN 1..array_length(sv, 1) LOOP + e := sv[idx]; + + IF eql_v2.selector(e) = selector THEN + found := array_append(found, e); + IF eql_v2.is_ste_vec_array(e) THEN + ary := true; + END IF; + + END IF; + END LOOP; + + IF found IS NOT NULL THEN + + IF ary THEN + -- Wrap found array elements as eql_v2_encrypted + RETURN NEXT jsonb_build_object( + 'sv', found, + 'a', 1 + )::eql_v2_encrypted; + + ELSE + RETURN NEXT found[1]::eql_v2_encrypted; + END IF; + + END IF; + + RETURN; + END; +$$ LANGUAGE plpgsql; + + + +CREATE FUNCTION eql_v2.jsonb_path_query(val eql_v2_encrypted, selector text) + RETURNS SETOF eql_v2_encrypted + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN QUERY + SELECT * FROM eql_v2.jsonb_path_query(val.data, selector); + END; +$$ LANGUAGE plpgsql; + + + +CREATE FUNCTION eql_v2.jsonb_path_exists(val jsonb, selector text) + RETURNS boolean + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN EXISTS ( + SELECT eql_v2.jsonb_path_query(val, selector) + ); + END; +$$ LANGUAGE plpgsql; + + + +CREATE FUNCTION eql_v2.jsonb_path_exists(val eql_v2_encrypted, selector text) + RETURNS boolean + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN EXISTS ( + SELECT eql_v2.jsonb_path_query(val, selector) + ); + END; +$$ LANGUAGE plpgsql; + + +-- +-- + +CREATE FUNCTION eql_v2.jsonb_path_query_first(val jsonb, selector text) + RETURNS eql_v2_encrypted + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN ( + SELECT ( + SELECT e + FROM eql_v2.jsonb_path_query(val.data, selector) AS e + LIMIT 1 + ) + ); + END; +$$ LANGUAGE plpgsql; + + + +CREATE FUNCTION eql_v2.jsonb_path_query_first(val eql_v2_encrypted, selector text) + RETURNS eql_v2_encrypted + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN ( + SELECT e + FROM eql_v2.jsonb_path_query(val.data, selector) as e + LIMIT 1 + ); + END; +$$ LANGUAGE plpgsql; + + + +-- + + +-- ===================================================================== +-- +-- Returns the length of an encrypted jsonb array +--- +-- An encrypted is a jsonb array if it contains an "a" field/attribute with a truthy value +-- + +CREATE FUNCTION eql_v2.jsonb_array_length(val jsonb) + RETURNS integer + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + DECLARE + sv eql_v2_encrypted[]; + found eql_v2_encrypted[]; + BEGIN + + IF val IS NULL THEN + RETURN NULL; + END IF; + + IF eql_v2.is_ste_vec_array(val) THEN + sv := eql_v2.ste_vec(val); + RETURN array_length(sv, 1); + END IF; + + RAISE 'cannot get array length of a non-array'; + END; +$$ LANGUAGE plpgsql; + + + +CREATE FUNCTION eql_v2.jsonb_array_length(val eql_v2_encrypted) + RETURNS integer + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN ( + SELECT eql_v2.jsonb_array_length(val.data) + ); + END; +$$ LANGUAGE plpgsql; + + + + +-- ===================================================================== +-- +-- Returns the length of an encrypted jsonb array +--- +-- An encrypted is a jsonb array if it contains an "a" field/attribute with a truthy value +-- + +CREATE FUNCTION eql_v2.jsonb_array_elements(val jsonb) + RETURNS SETOF eql_v2_encrypted + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + DECLARE + sv eql_v2_encrypted[]; + found eql_v2_encrypted[]; + BEGIN + + IF NOT eql_v2.is_ste_vec_array(val) THEN + RAISE 'cannot extract elements from non-array'; + END IF; + + sv := eql_v2.ste_vec(val); + + FOR idx IN 1..array_length(sv, 1) LOOP + RETURN NEXT sv[idx]; + END LOOP; + + RETURN; + END; +$$ LANGUAGE plpgsql; + + + +CREATE FUNCTION eql_v2.jsonb_array_elements(val eql_v2_encrypted) + RETURNS SETOF eql_v2_encrypted + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN QUERY + SELECT * FROM eql_v2.jsonb_array_elements(val.data); + END; +$$ LANGUAGE plpgsql; + + + +-- ===================================================================== +-- +-- Returns the length of an encrypted jsonb array +--- +-- An encrypted is a jsonb array if it contains an "a" field/attribute with a truthy value +-- + +CREATE FUNCTION eql_v2.jsonb_array_elements_text(val jsonb) + RETURNS SETOF text + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + DECLARE + sv eql_v2_encrypted[]; + found eql_v2_encrypted[]; + BEGIN + IF NOT eql_v2.is_ste_vec_array(val) THEN + RAISE 'cannot extract elements from non-array'; + END IF; + + sv := eql_v2.ste_vec(val); + + FOR idx IN 1..array_length(sv, 1) LOOP + RETURN NEXT eql_v2.ciphertext(sv[idx]); + END LOOP; + + RETURN; + END; +$$ LANGUAGE plpgsql; + + + +CREATE FUNCTION eql_v2.jsonb_array_elements_text(val eql_v2_encrypted) + RETURNS SETOF text + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN QUERY + SELECT * FROM eql_v2.jsonb_array_elements_text(val.data); + END; +$$ LANGUAGE plpgsql; diff --git a/src/jsonb/functions_test.sql b/src/jsonb/functions_test.sql new file mode 100644 index 00000000..3068d498 --- /dev/null +++ b/src/jsonb/functions_test.sql @@ -0,0 +1,332 @@ +\set ON_ERROR_STOP on + + +SELECT create_table_with_encrypted(); +SELECT seed_encrypted_json(); + + +-- CREATE TABLE unencrypted +-- ( +-- id bigint GENERATED ALWAYS AS IDENTITY, +-- u jsonb, +-- PRIMARY KEY(id) +-- ); +-- INSERT INTO unencrypted (u) +-- VALUES +-- ('{"a": [1, 2, 3] }'), +-- ('{"a": [1, 2, 3, 4] }'), +-- ('{"a": [1, 2, 3, 4, 5] }'); + +-- SELECT * +-- FROM unencrypted +-- WHERE EXISTS ( +-- SELECT 1 +-- FROM jsonb_array_elements(u->'a') AS elem +-- WHERE elem::int < 2 +-- ); + +-- SELECT seed_encrypted(get_array_ste_vec()::eql_v2_encrypted); +-- SELECT * +-- FROM encrypted +-- WHERE EXISTS ( +-- SELECT 1 +-- FROM eql_v2.jsonb_array_elements(e->'f510853730e1c3dbd31b86963f029dd5') AS elem +-- WHERE elem > '{"ocf": "b0c0a7385cb2f7dfe32a2649a9d8294794b8fc05585a240c1315f1e45ee7d9012616db3f01b43fa94351618670a29c24fc75df1392d52764c757b34495888b1c"}'::jsonb +-- ); + +-- SELECT eql_v2.jsonb_path_query_first(e, '33743aed3ae636f6bf05cff11ac4b519') as e +-- FROM encrypted +-- WHERE eql_v2.jsonb_path_query(e, '33743aed3ae636f6bf05cff11ac4b519') IS NOT NULL; + + + +-- "ocf": "b0c0a7385cb2f7dfe32a2649a9d8294794b8fc05585a240c1315f1e45ee7d9012616db3f01b43fa94351618670a29c24fc75df1392d52764c757b34495888b1c", + +-- SELECT eql_v2.jsonb_array_elements(eql_v2.jsonb_path_query(e, 'f510853730e1c3dbd31b86963f029dd5')) as e FROM encrypted ; + + + + +-- -- SELECT eql_v2.jsonb_path_exists(e, ''f510853730e1c3dbd31b86963f029dd5'') FROM encrypted; +-- -- SELECT eql_v2.jsonb_path_query(e, 'f510853730e1c3dbd31b86963f029dd5') FROM encrypted; + +-- -- SELECT eql_v2.jsonb_path_query(e, 'f510853730e1c3dbd31b86963f029dd5') as e FROM encrypted; +-- -- SELECT eql_v2.jsonb_array_length(eql_v2.jsonb_path_query(e, 'f510853730e1c3dbd31b86963f029dd5')) as e FROM encrypted LIMIT 1; +-- -- SELECT eql_v2.jsonb_array_elements(eql_v2.jsonb_path_query(e, 'f510853730e1c3dbd31b86963f029dd5')) as e FROM encrypted ; +-- -- SELECT eql_v2.jsonb_array_elements_text(eql_v2.jsonb_path_query(e, 'f510853730e1c3dbd31b86963f029dd5')) as e FROM encrypted ; +-- -- SELECT eql_v2.jsonb_array_length(eql_v2.jsonb_path_query(e, 'f510853730e1c3dbd31b86963f029dd5')) as e FROM encrypted LIMIT 1; +-- -- SELECT eql_v2.jsonb_path_query(e, 'f510853730e1c3dbd31b86963f029dd5') as e FROM encrypted; + + + + +-- ======================================================================== +-- +-- Selector &.a[*] +-- -> 33743aed3ae636f6bf05cff11ac4b519 +-- +DO $$ +DECLARE + sv eql_v2_encrypted; + results eql_v2_encrypted[]; + BEGIN + + PERFORM seed_encrypted_json(); + PERFORM seed_encrypted(get_array_ste_vec()::eql_v2_encrypted); + + PERFORM assert_result( + 'jsonb_array_elements returns array elements from jsonb_path_query result', + 'SELECT eql_v2.jsonb_array_elements(eql_v2.jsonb_path_query(e, ''f510853730e1c3dbd31b86963f029dd5'')) as e FROM encrypted;'); + + PERFORM assert_count( + 'jsonb_array_elements returns the correct number of array elements from jsonb_path_query result', + 'SELECT eql_v2.jsonb_array_elements(eql_v2.jsonb_path_query(e, ''f510853730e1c3dbd31b86963f029dd5'')) as e FROM encrypted;', + 5); + + PERFORM assert_exception( + 'jsonb_array_elements exception if input is not an array', + 'SELECT eql_v2.jsonb_array_elements(eql_v2.jsonb_path_query(e, ''33743aed3ae636f6bf05cff11ac4b519'')) as e FROM encrypted LIMIT 1;'); + + END; +$$ LANGUAGE plpgsql; + + +-- -- ======================================================================== +-- -- +-- -- Selector &.a[*] +-- -- -> 33743aed3ae636f6bf05cff11ac4b519 +-- -- +DO $$ +DECLARE + sv eql_v2_encrypted; + results eql_v2_encrypted[]; + BEGIN + + PERFORM seed_encrypted_json(); + PERFORM seed_encrypted(get_array_ste_vec()::eql_v2_encrypted); + + PERFORM assert_result( + 'jsonb_array_elements_text returns array elements from jsonb_path_query result', + 'SELECT eql_v2.jsonb_array_elements_text(eql_v2.jsonb_path_query(e, ''f510853730e1c3dbd31b86963f029dd5'')) as e FROM encrypted;'); + + PERFORM assert_count( + 'jsonb_array_elements_text returns the correct number of array elements from jsonb_path_query result', + 'SELECT eql_v2.jsonb_array_elements_text(eql_v2.jsonb_path_query(e, ''f510853730e1c3dbd31b86963f029dd5'')) as e FROM encrypted;', + 5); + + PERFORM assert_exception( + 'jsonb_array_elements_text exception if input is not an array', + 'SELECT eql_v2.jsonb_array_elements_text(eql_v2.jsonb_path_query(e, ''33743aed3ae636f6bf05cff11ac4b519'')) as e FROM encrypted LIMIT 1;'); + + END; +$$ LANGUAGE plpgsql; + + +-- ======================================================================== +-- +-- Selector &.a[*] +-- -> 33743aed3ae636f6bf05cff11ac4b519 +-- +DO $$ +DECLARE + sv eql_v2_encrypted; + results eql_v2_encrypted[]; + BEGIN + + PERFORM seed_encrypted_json(); + PERFORM seed_encrypted(get_array_ste_vec()::eql_v2_encrypted); + + PERFORM assert_result( + 'jsonb_array_length returns array length of jsonb_path_query result', + 'SELECT eql_v2.jsonb_array_length(eql_v2.jsonb_path_query(e, ''f510853730e1c3dbd31b86963f029dd5'')) as e FROM encrypted LIMIT 1;', + '5'); + + PERFORM assert_exception( + 'jsonb_array_length exception if input is not an array', + 'SELECT eql_v2.jsonb_array_length(eql_v2.jsonb_path_query(e, ''33743aed3ae636f6bf05cff11ac4b519'')) as e FROM encrypted LIMIT 1;'); + + END; +$$ LANGUAGE plpgsql; + + + +-- -- ======================================================================== +-- +-- -- "{\"hello\": \"four\", \"n\": 20, \"a\": [1, 2, 3, 4, 5] }", +-- +-- Selector &.a[*] +-- -> 33743aed3ae636f6bf05cff11ac4b519 +-- +DO $$ +DECLARE + sv eql_v2_encrypted; + results eql_v2_encrypted[]; + BEGIN + + PERFORM seed_encrypted_json(); + + -- Insert a row with array selector + sv := get_array_ste_vec()::eql_v2_encrypted; + PERFORM seed_encrypted(sv); + + PERFORM assert_count( + 'jsonb_path_query with array selector returns count', + 'SELECT eql_v2.jsonb_path_query_first(e, ''33743aed3ae636f6bf05cff11ac4b519'') as e FROM encrypted;', + 4 + ); + + PERFORM assert_count( + 'jsonb_path_query with array selector returns count', + 'SELECT eql_v2.jsonb_path_query_first(e, ''33743aed3ae636f6bf05cff11ac4b519'') as e FROM encrypted WHERE eql_v2.jsonb_path_query_first(e, ''33743aed3ae636f6bf05cff11ac4b519'') IS NOT NULL;', + 1 + ); + + END; +$$ LANGUAGE plpgsql; + + +-- ------------------------------------------------------------------------ + + + +-- ------------------------------------------------------------------------ +-- +-- jsonb_path_query +-- + +-- Paths +-- $ -> bca213de9ccce676fa849ff9c4807963 +-- $.hello -> a7cea93975ed8c01f861ccb6bd082784 +-- $.n -> 2517068c0d1f9d4d41d2c666211f785e +-- +-- +DO $$ + BEGIN + PERFORM seed_encrypted_json(); + + PERFORM assert_result( + 'jsonb_path_query', + 'SELECT eql_v2.jsonb_path_query(e, ''2517068c0d1f9d4d41d2c666211f785e'') FROM encrypted LIMIT 1;'); + + PERFORM assert_count( + 'jsonb_path_query returns count', + 'SELECT eql_v2.jsonb_path_query(e, ''2517068c0d1f9d4d41d2c666211f785e'') FROM encrypted;', + 3); + + END; +$$ LANGUAGE plpgsql; + + +DO $$ + BEGIN + + PERFORM seed_encrypted_json(); + + PERFORM assert_result( + 'jsonb_path_exists returns true', + 'SELECT eql_v2.jsonb_path_exists(e, ''2517068c0d1f9d4d41d2c666211f785e'') FROM encrypted LIMIT 1;', + 'true'); + + PERFORM assert_result( + 'jsonb_path_exists returns false', + 'SELECT eql_v2.jsonb_path_exists(e, ''blahvtha'') FROM encrypted LIMIT 1;', + 'false'); + + PERFORM assert_count( + 'jsonb_path_exists returns count', + 'SELECT eql_v2.jsonb_path_exists(e, ''2517068c0d1f9d4d41d2c666211f785e'') FROM encrypted;', + 3); + END; +$$ LANGUAGE plpgsql; + + + +-- -- +-- -- Selector &.a[*] +-- -- -> 33743aed3ae636f6bf05cff11ac4b519 +-- -- +DO $$ +DECLARE + sv eql_v2_encrypted; + results eql_v2_encrypted[]; + BEGIN + + -- Insert a row with array selector + sv := get_array_ste_vec()::eql_v2_encrypted; + PERFORM seed_encrypted(sv); + + PERFORM assert_result( + 'jsonb_path_query with array selector', + 'SELECT eql_v2.jsonb_path_query(e, ''f510853730e1c3dbd31b86963f029dd5'') FROM encrypted;'); + + -- An array should be wrapped and returned as a single element + PERFORM assert_count( + 'jsonb_path_query with array selector returns one result', + 'SELECT eql_v2.jsonb_path_query(e, ''f510853730e1c3dbd31b86963f029dd5'') FROM encrypted;', + 1); + END; +$$ LANGUAGE plpgsql; + + + +-- -- +-- -- Selector &.a[*] +-- -- -> 33743aed3ae636f6bf05cff11ac4b519 +-- -- +DO $$ + DECLARE + sv eql_v2_encrypted; + results eql_v2_encrypted[]; + BEGIN + + PERFORM seed_encrypted_json(); + + -- Insert a row with array selector + sv := get_array_ste_vec()::eql_v2_encrypted; + PERFORM seed_encrypted(sv); + + PERFORM assert_result( + 'jsonb_path_exists with array selector', + 'SELECT eql_v2.jsonb_path_exists(e, ''f510853730e1c3dbd31b86963f029dd5'') FROM encrypted;'); + + PERFORM assert_count( + 'jsonb_path_exists with array selector returns correct number of records', + 'SELECT eql_v2.jsonb_path_exists(e, ''f510853730e1c3dbd31b86963f029dd5'') FROM encrypted;', + 4); + END; +$$ LANGUAGE plpgsql; + + + +-- -- +-- -- Selector &.a[*] +-- -- -> 33743aed3ae636f6bf05cff11ac4b519 +-- -- +DO $$ +DECLARE + sv eql_v2_encrypted; + results eql_v2_encrypted[]; + BEGIN + + PERFORM seed_encrypted_json(); + + -- Insert a row with array selector + sv := get_array_ste_vec()::eql_v2_encrypted; + PERFORM seed_encrypted(sv); + + PERFORM assert_count( + 'jsonb_path_query with array selector returns count', + 'SELECT eql_v2.jsonb_path_query_first(e, ''33743aed3ae636f6bf05cff11ac4b519'') as e FROM encrypted;', + 4 + ); + + PERFORM assert_count( + 'jsonb_path_query with array selector returns count', + 'SELECT eql_v2.jsonb_path_query_first(e, ''33743aed3ae636f6bf05cff11ac4b519'') as e FROM encrypted WHERE eql_v2.jsonb_path_query_first(e, ''33743aed3ae636f6bf05cff11ac4b519'') IS NOT NULL;', + 1 + ); + + END; +$$ LANGUAGE plpgsql; + + diff --git a/src/match/functions.sql b/src/match/functions.sql new file mode 100644 index 00000000..b7b65388 --- /dev/null +++ b/src/match/functions.sql @@ -0,0 +1,28 @@ +-- REQUIRE: src/schema.sql + + +-- extracts match index from an emcrypted column + +CREATE FUNCTION eql_v2.match(val jsonb) + RETURNS eql_v2.match_index + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + IF val ? 'm' THEN + RETURN ARRAY(SELECT jsonb_array_elements(val->'m'))::eql_v2.match_index; + END IF; + RAISE 'Expected a match index (m) value in json: %', val; + END; +$$ LANGUAGE plpgsql; + + +-- extracts unique index from an encrypted column + +CREATE FUNCTION eql_v2.match(val eql_v2_encrypted) + RETURNS eql_v2.match_index + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN (SELECT eql_v2.match(val.data)); + END; +$$ LANGUAGE plpgsql; diff --git a/src/match/functions_test.sql b/src/match/functions_test.sql new file mode 100644 index 00000000..de956ebd --- /dev/null +++ b/src/match/functions_test.sql @@ -0,0 +1,14 @@ +\set ON_ERROR_STOP on + +DO $$ + BEGIN + PERFORM assert_result( + 'Extract match index term from encrypted', + 'SELECT eql_v2.match(''{"m": []}''::jsonb)'); + + PERFORM assert_exception( + 'Missing match index term in encrypted raises exception', + 'SELECT eql_v2.match(''{}''::jsonb)'); + + END; +$$ LANGUAGE plpgsql; diff --git a/src/match/types.sql b/src/match/types.sql new file mode 100644 index 00000000..89843cf9 --- /dev/null +++ b/src/match/types.sql @@ -0,0 +1,4 @@ +-- REQUIRE: src/schema.sql + +CREATE DOMAIN eql_v2.match_index AS smallint[]; + diff --git a/src/operators/->.sql b/src/operators/->.sql new file mode 100644 index 00000000..6d7be6e5 --- /dev/null +++ b/src/operators/->.sql @@ -0,0 +1,91 @@ + +-- REQUIRE: src/schema.sql +-- REQUIRE: src/encrypted/types.sql +-- REQUIRE: src/encrypted/functions.sql + + +-- +-- The -> operator returns an encrypted matching the selector +-- Encyprted JSON is represented as an array of `eql_v2_encrypted`. +-- Each `eql_v2_encrypted` value has a selector, ciphertext, and an index term of +-- - blake3 +-- - ore_cllw_u64_8 +-- - ore_cllw_var_8 +-- +-- { +-- "sv": [ {"c": "", "s": "", "b": "" } ] +-- } +-- + + +CREATE FUNCTION eql_v2."->"(e eql_v2_encrypted, selector text) + RETURNS eql_v2_encrypted + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + DECLARE + sv eql_v2_encrypted[]; + found eql_v2_encrypted; + BEGIN + + IF e IS NULL THEN + RETURN NULL; + END IF; + + sv := eql_v2.ste_vec(e); + + FOR idx IN 1..array_length(sv, 1) LOOP + if eql_v2.selector(sv[idx]) = selector THEN + found := sv[idx]; + END IF; + END LOOP; + + RETURN found; + END; +$$ LANGUAGE plpgsql; + + +-- + + + + +CREATE FUNCTION eql_v2."->"(e eql_v2_encrypted, selector integer) + RETURNS eql_v2_encrypted + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + DECLARE + sv eql_v2_encrypted[]; + found eql_v2_encrypted; + BEGIN + IF NOT eql_v2.is_ste_vec_array(e) THEN + RETURN NULL; + END IF; + + sv := eql_v2.ste_vec(e); + + -- PostgreSQL arrays are 1-based + -- JSONB arrays are 0-based and so the selector is 0-based + FOR idx IN 1..array_length(sv, 1) LOOP + if (idx-1) = selector THEN + found := sv[idx]; + END IF; + END LOOP; + + RETURN found; + END; +$$ LANGUAGE plpgsql; + + +CREATE OPERATOR ->( + FUNCTION=eql_v2."->", + LEFTARG=eql_v2_encrypted, + RIGHTARG=text +); + + +CREATE OPERATOR ->( + FUNCTION=eql_v2."->", + LEFTARG=eql_v2_encrypted, + RIGHTARG=integer +); + diff --git a/src/operators/->>.sql b/src/operators/->>.sql new file mode 100644 index 00000000..d50977d7 --- /dev/null +++ b/src/operators/->>.sql @@ -0,0 +1,28 @@ +-- REQUIRE: src/schema.sql +-- REQUIRE: src/encrypted/types.sql +-- REQUIRE: src/encrypted/functions.sql + + + +CREATE FUNCTION eql_v2."->>"(e eql_v2_encrypted, selector text) + RETURNS text +IMMUTABLE STRICT PARALLEL SAFE +AS $$ + DECLARE + found eql_v2_encrypted; + BEGIN + + found = eql_v2."->"(e, selector); + + RETURN eql_v2.ciphertext(found); + END; +$$ LANGUAGE plpgsql; + + +CREATE OPERATOR ->> ( + FUNCTION=eql_v2."->>", + LEFTARG=eql_v2_encrypted, + RIGHTARG=text +); + + diff --git a/src/operators/->>_test.sql b/src/operators/->>_test.sql new file mode 100644 index 00000000..077d8b7d --- /dev/null +++ b/src/operators/->>_test.sql @@ -0,0 +1,45 @@ +\set ON_ERROR_STOP on + +SELECT create_table_with_encrypted(); +SELECT seed_encrypted_json(); + +-- +-- The ->> operator returns ciphertext matching the selector +DO $$ + BEGIN + PERFORM assert_result( + 'Selector ->> returns at least one eql_v2_encrypted', + 'SELECT e->>''bca213de9ccce676fa849ff9c4807963'' FROM encrypted;'); + + PERFORM assert_count( + 'Selector ->> returns all eql_v2_encrypted', + 'SELECT e->>''bca213de9ccce676fa849ff9c4807963'' FROM encrypted;', + 3); + END; +$$ LANGUAGE plpgsql; + + +-- +-- The ->> operator returns NULL if no matching selector +DO $$ + BEGIN + PERFORM assert_no_result( + 'Unknown selector -> returns null', + 'SELECT e->>''blahvtha'' FROM encrypted;'); + + END; +$$ LANGUAGE plpgsql; + + +-- +-- The ->> operator returns ciphertext matching the selector +DO $$ + BEGIN + + PERFORM assert_result( + 'Selector ->> returns all eql_v2_encrypted', + 'SELECT e->>''bca213de9ccce676fa849ff9c4807963'' FROM encrypted LIMIT 1;', + 'mBbLGB9xHAGzLvUj-`@Wmf=IhD87n7r3ir3n!Sk6AKir_YawR=0c>pk(OydB;ntIEXK~c>V&4>)rNkf_test.sql b/src/operators/->_test.sql new file mode 100644 index 00000000..439510f0 --- /dev/null +++ b/src/operators/->_test.sql @@ -0,0 +1,79 @@ +\set ON_ERROR_STOP on + + +SELECT create_table_with_encrypted(); +SELECT seed_encrypted_json(); + + +-- +-- The -> operator returns an encrypted matching the selector +DO $$ + BEGIN + PERFORM assert_result( + 'Selector -> returns at least one eql_v2_encrypted', + 'SELECT e->''bca213de9ccce676fa849ff9c4807963'' FROM encrypted;'); + + PERFORM assert_count( + 'Selector -> returns all eql_v2_encrypted', + 'SELECT e->''bca213de9ccce676fa849ff9c4807963'' FROM encrypted;', + 3); + END; +$$ LANGUAGE plpgsql; + + +-- +-- The -> operator returns NULL if no matching selector +DO $$ + BEGIN + PERFORM assert_no_result( + 'Unknown selector -> returns null', + 'SELECT e->''blahvtha'' FROM encrypted;'); + + END; +$$ LANGUAGE plpgsql; + + + +-- +-- encrypted returned from -> operator expression called via eql_v2.ciphertext +-- +DO $$ + DECLARE + result eql_v2_encrypted; + BEGIN + PERFORM assert_result( + 'Fetch ciphertext via selector', + 'SELECT eql_v2.ciphertext(e->''2517068c0d1f9d4d41d2c666211f785e'') FROM encrypted;'); + + PERFORM assert_count( + 'Fetch ciphertext via selector returns all eql_v2_encrypted', + 'SELECT eql_v2.ciphertext(e->''2517068c0d1f9d4d41d2c666211f785e'') FROM encrypted;', + 3); + END; +$$ LANGUAGE plpgsql; + + + +-- +-- encrypted returned from -> operator expression called via eql_v2.ciphertext +-- +DO $$ + DECLARE + result eql_v2_encrypted; + BEGIN + + PERFORM truncate_table_with_encrypted(); + PERFORM seed_encrypted(get_array_ste_vec()::eql_v2_encrypted); + + PERFORM assert_result( + '-> operator with integer returns array', + 'SELECT eql_v2.jsonb_path_query_first(e, ''f510853730e1c3dbd31b86963f029dd5'')->0 as e FROM encrypted'); + + PERFORM assert_count( + '-> operator with integer returns array', + 'SELECT eql_v2.jsonb_path_query_first(e, ''f510853730e1c3dbd31b86963f029dd5'')->0 as e FROM encrypted', + 1); + + END; +$$ LANGUAGE plpgsql; + diff --git a/src/operators/<.sql b/src/operators/<.sql new file mode 100644 index 00000000..f618dbe3 --- /dev/null +++ b/src/operators/<.sql @@ -0,0 +1,113 @@ +-- REQUIRE: src/schema.sql +-- REQUIRE: src/ore/types.sql +-- REQUIRE: src/ore/functions.sql +-- REQUIRE: src/ore/operators.sql + + +-- Operators for < less than comparisons of eql_v2_encrypted types +-- +-- Support for the following comparisons: +-- +-- eql_v2_encrypted = eql_v2_encrypted +-- eql_v2_encrypted = jsonb +-- jsonb = eql_v2_encrypted +-- +-- There are multiple index terms that provide equality comparisons +-- - ore_64_8_v2 +-- - ore_cllw_8_v2 +-- +-- We check these index terms in this order and use the first one that exists for both parameters +-- +-- + + +CREATE FUNCTION eql_v2.lt(a eql_v2_encrypted, b eql_v2_encrypted) +RETURNS boolean +AS $$ + BEGIN + + BEGIN + RETURN eql_v2.ore_cllw_u64_8(a) < eql_v2.ore_cllw_u64_8(b); + -- RETURN (eql_v2.unique(a) = eql_v2.unique(b)); + EXCEPTION WHEN OTHERS THEN + -- PERFORM eql_v2.log('eql_v2.lt no ore_cllw_u64_8 index'); + END; + + BEGIN + RETURN eql_v2.ore_cllw_var_8(a) < eql_v2.ore_cllw_var_8(b); + EXCEPTION WHEN OTHERS THEN + -- PERFORM eql_v2.log('eql_v2.lt no ore_cllw_var_8 index'); + END; + + BEGIN + RETURN eql_v2.ore_64_8_v2(a) < eql_v2.ore_64_8_v2(b); + EXCEPTION WHEN OTHERS THEN + -- PERFORM eql_v2.log('eql_v2.lt no ore_64_8_v2 index'); + END; + + RETURN false; + END; +$$ LANGUAGE plpgsql; + + + +CREATE FUNCTION eql_v2."<"(a eql_v2_encrypted, b eql_v2_encrypted) +RETURNS boolean +AS $$ + BEGIN + RETURN eql_v2.lt(a, b); + END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR <( + FUNCTION=eql_v2."<", + LEFTARG=eql_v2_encrypted, + RIGHTARG=eql_v2_encrypted, + COMMUTATOR = >, + NEGATOR = >=, + RESTRICT = scalarltsel, + JOIN = scalarltjoinsel +); + + + +CREATE FUNCTION eql_v2."<"(a eql_v2_encrypted, b jsonb) +RETURNS boolean +AS $$ + BEGIN + RETURN eql_v2.lt(a, b::eql_v2_encrypted); + END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR <( + FUNCTION=eql_v2."<", + LEFTARG=eql_v2_encrypted, + RIGHTARG=jsonb, + COMMUTATOR = >, + NEGATOR = >=, + RESTRICT = scalarltsel, + JOIN = scalarltjoinsel +); + + + +CREATE FUNCTION eql_v2."<"(a jsonb, b eql_v2_encrypted) +RETURNS boolean +AS $$ + BEGIN + RETURN eql_v2.lt(a::eql_v2_encrypted, b); + END; +$$ LANGUAGE plpgsql; + + +CREATE OPERATOR <( + FUNCTION=eql_v2."<", + LEFTARG=jsonb, + RIGHTARG=eql_v2_encrypted, + COMMUTATOR = >, + NEGATOR = >=, + RESTRICT = scalarltsel, + JOIN = scalarltjoinsel +); + + diff --git a/src/operators/<=.sql b/src/operators/<=.sql new file mode 100644 index 00000000..846ee0df --- /dev/null +++ b/src/operators/<=.sql @@ -0,0 +1,112 @@ +-- REQUIRE: src/encrypted/types.sql +-- REQUIRE: src/ore/types.sql +-- REQUIRE: src/ore/functions.sql +-- REQUIRE: src/ore/operators.sql + + +-- Operators for < less than comparisons of eql_v2_encrypted types +-- +-- Support for the following comparisons: +-- +-- eql_v2_encrypted = eql_v2_encrypted +-- eql_v2_encrypted = jsonb +-- jsonb = eql_v2_encrypted +-- +-- There are multiple index terms that provide equality comparisons +-- - ore_64_8_v2 +-- - ore_cllw_8_v2 +-- +-- We check these index terms in this order and use the first one that exists for both parameters +-- +-- + + +CREATE FUNCTION eql_v2.lte(a eql_v2_encrypted, b eql_v2_encrypted) + RETURNS boolean +AS $$ + BEGIN + + BEGIN + RETURN eql_v2.ore_cllw_u64_8(a) <= eql_v2.ore_cllw_u64_8(b); + EXCEPTION WHEN OTHERS THEN + -- PERFORM eql_v2.log('eql_v2.lte no ore_cllw_u64_8 index'); + END; + + BEGIN + RETURN eql_v2.ore_cllw_var_8(a) <= eql_v2.ore_cllw_var_8(b); + EXCEPTION WHEN OTHERS THEN + -- PERFORM eql_v2.log('eql_v2.lte no ore_cllw_var_8 index'); + END; + + BEGIN + RETURN eql_v2.ore_64_8_v2(a) <= eql_v2.ore_64_8_v2(b); + EXCEPTION WHEN OTHERS THEN + -- PERFORM eql_v2.log('eql_v2.lte no ore_64_8_v2 index'); + END; + + RETURN false; + END; +$$ LANGUAGE plpgsql; + + + +CREATE FUNCTION eql_v2."<="(a eql_v2_encrypted, b eql_v2_encrypted) +RETURNS boolean +AS $$ + BEGIN + RETURN eql_v2.lte(a, b); + END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR <=( + FUNCTION = eql_v2."<=", + LEFTARG = eql_v2_encrypted, + RIGHTARG = eql_v2_encrypted, + COMMUTATOR = >=, + NEGATOR = >, + RESTRICT = scalarltsel, + JOIN = scalarltjoinsel +); + + + +CREATE FUNCTION eql_v2."<="(a eql_v2_encrypted, b jsonb) +RETURNS boolean +AS $$ + BEGIN + RETURN eql_v2.lte(a, b::eql_v2_encrypted); + END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR <=( + FUNCTION = eql_v2."<=", + LEFTARG = eql_v2_encrypted, + RIGHTARG = jsonb, + COMMUTATOR = >=, + NEGATOR = >, + RESTRICT = scalarltsel, + JOIN = scalarltjoinsel +); + + + +CREATE FUNCTION eql_v2."<="(a jsonb, b eql_v2_encrypted) +RETURNS boolean +AS $$ + BEGIN + RETURN eql_v2.lte(a::eql_v2_encrypted, b); + END; +$$ LANGUAGE plpgsql; + + +CREATE OPERATOR <=( + FUNCTION = eql_v2."<=", + LEFTARG = jsonb, + RIGHTARG = eql_v2_encrypted, + COMMUTATOR = >=, + NEGATOR = >, + RESTRICT = scalarltsel, + JOIN = scalarltjoinsel +); + + diff --git a/src/operators/<=_test.sql b/src/operators/<=_test.sql new file mode 100644 index 00000000..f9d4625b --- /dev/null +++ b/src/operators/<=_test.sql @@ -0,0 +1,175 @@ +\set ON_ERROR_STOP on + +SELECT create_table_with_encrypted(); +SELECT seed_encrypted_json(); + +SELECT e FROM encrypted WHERE e->'a7cea93975ed8c01f861ccb6bd082784' <= '("{""c"": ""mBbM0#UZON2jQ3@LiWcvns2Yf6y3L;hykEh`}*fX#aF;n*=>+*o5Uarod39C7TF-SiCD-NgkG)l%Vw=l!tX>H*P bca213de9ccce676fa849ff9c4807963 +-- $.hello -> a7cea93975ed8c01f861ccb6bd082784 +-- $.n -> 2517068c0d1f9d4d41d2c666211f785e +-- +-- -- +DO $$ +DECLARE + sv eql_v2_encrypted; + term eql_v2_encrypted; + BEGIN + + -- This extracts the data associated with the field from the test eql_v2_encrypted + -- json n: 30 + sv := get_numeric_ste_vec_30()::eql_v2_encrypted; + -- extract the term at $.n returned as eql_v2_encrypted + term := sv->'2517068c0d1f9d4d41d2c666211f785e'; + + -- -- -- -- $.n + PERFORM assert_result( + format('eql_v2_encrypted <= eql_v2_encrypted with ore_cllw_u64_8 index term'), + format('SELECT e FROM encrypted WHERE e->''2517068c0d1f9d4d41d2c666211f785e'' <= %L::eql_v2_encrypted', term)); + + PERFORM assert_count( + format('eql_v2_encrypted <= eql_v2_encrypted with ore index term'), + format('SELECT e FROM encrypted WHERE e->''2517068c0d1f9d4d41d2c666211f785e'' <= %L::eql_v2_encrypted', term), + 3); + + -- -- Check the $.hello path + -- -- Returned encrypted does not have ore_cllw_u64_8 + PERFORM assert_no_result( + format('eql_v2_encrypted <= eql_v2_encrypted with ore_cllw_u64_8 index term'), + format('SELECT e FROM encrypted WHERE e->''a7cea93975ed8c01f861ccb6bd082784'' <= %L::eql_v2_encrypted', term)); + + END; +$$ LANGUAGE plpgsql; + +-- ------------------------------------------------------------------------ +-- ------------------------------------------------------------------------ +-- +-- eql_v2_encrypted < eql_v2_encrypted with ore_cllw_var_8 index +-- +-- Test data is in form '{"hello": "{one | two | three}", "n": {10 | 20 | 30} }' +-- +-- Paths +-- $ -> bca213de9ccce676fa849ff9c4807963 +-- $.hello -> a7cea93975ed8c01f861ccb6bd082784 +-- $.n -> 2517068c0d1f9d4d41d2c666211f785e +-- +-- -- +DO $$ +DECLARE + sv eql_v2_encrypted; + term eql_v2_encrypted; + BEGIN + + -- This extracts the data associated with the field from the test eql_v2_encrypted + -- json n: 30 + sv := get_numeric_ste_vec_30()::eql_v2_encrypted; + -- extract the term at $.n returned as eql_v2_encrypted + term := sv->'a7cea93975ed8c01f861ccb6bd082784'; + + -- -- -- -- $.n + PERFORM assert_result( + format('eql_v2_encrypted <= eql_v2_encrypted with ore_cllw_var_8 index term'), + format('SELECT e FROM encrypted WHERE e->''a7cea93975ed8c01f861ccb6bd082784'' <= %L::eql_v2_encrypted', term)); + + PERFORM assert_count( + format('eql_v2_encrypted <= eql_v2_encrypted with ore_cllw_var_8 index term'), + format('SELECT e FROM encrypted WHERE e->''a7cea93975ed8c01f861ccb6bd082784'' <= %L::eql_v2_encrypted', term), + 2); + + -- -- Check the $.n path + -- -- Returned encrypted does not have ore_cllw_u64_8 + PERFORM assert_no_result( + format('eql_v2_encrypted <= eql_v2_encrypted with ore_cllw_var_8 index term'), + format('SELECT e FROM encrypted WHERE e->''2517068c0d1f9d4d41d2c666211f785e'' <= %L::eql_v2_encrypted', term)); + + END; +$$ LANGUAGE plpgsql; + + + +-- +-- ORE - eql_v2_encrypted <= eql_v2_encrypted +-- +DO $$ +DECLARE + e eql_v2_encrypted; + ore_term jsonb; + BEGIN + + -- Record with a Numeric ORE term of 42 + e := create_encrypted_ore_json(42); + PERFORM seed_encrypted(e); + + PERFORM assert_result( + 'eql_v2_encrypted <= eql_v2_encrypted', + format('SELECT e FROM encrypted WHERE e <= %L::eql_v2_encrypted', e)); + + PERFORM assert_count( + format('eql_v2_encrypted <= eql_v2_encrypted'), + format('SELECT e FROM encrypted WHERE e <= %L;', e), + 4); + + e := create_encrypted_ore_json(20); + + PERFORM assert_result( + 'eql_v2_encrypted <= eql_v2_encrypted', + format('SELECT e FROM encrypted WHERE e <= %L::eql_v2_encrypted', e)); + + PERFORM assert_count( + format('eql_v2_encrypted <= eql_v2_encrypted'), + format('SELECT e FROM encrypted WHERE e <= %L;', e), + 2); + END; +$$ LANGUAGE plpgsql; + + +-- +-- ORE - eql_v2.gte(a eql_v2_encrypted, b eql_v2_encrypted) +-- +DO $$ +DECLARE + e eql_v2_encrypted; + ore_term jsonb; + BEGIN + -- Reset data + PERFORM seed_encrypted_json(); + + -- Record with a Numeric ORE term of 42 + e := create_encrypted_ore_json(42); + PERFORM seed_encrypted(e); + + PERFORM assert_result( + 'eql_v2.lte(a eql_v2_encrypted, b eql_v2_encrypted)', + format('SELECT e FROM encrypted WHERE eql_v2.lte(e, %L)', e)); + + -- include + PERFORM assert_count( + 'eql_v2.lte(a eql_v2_encrypted, b eql_v2_encrypted)', + format('SELECT e FROM encrypted WHERE eql_v2.lte(e, %L)', e), + 4); + + -- Record with a Numeric ORE term of 30 + e := create_encrypted_ore_json(30); + + PERFORM assert_result( + 'eql_v2.get(a eql_v2_encrypted, b eql_v2_encrypted)', + format('SELECT e FROM encrypted WHERE eql_v2.lte(e, %L)', e)); + + PERFORM assert_count( + 'eql_v2.get(a eql_v2_encrypted, b eql_v2_encrypted)', + format('SELECT e FROM encrypted WHERE eql_v2.lte(e, %L)', e), + 3); + END; +$$ LANGUAGE plpgsql; + + +SELECT drop_table_with_encrypted(); \ No newline at end of file diff --git a/src/operators/<>.sql b/src/operators/<>.sql new file mode 100644 index 00000000..3a13d594 --- /dev/null +++ b/src/operators/<>.sql @@ -0,0 +1,102 @@ +-- REQUIRE: src/encrypted/types.sql +-- REQUIRE: src/unique/types.sql +-- REQUIRE: src/unique/functions.sql +-- REQUIRE: src/ore/types.sql +-- REQUIRE: src/ore/functions.sql +-- REQUIRE: src/operators/=.sql + +-- Operators for equality comparisons of eql_v2_encrypted types +-- +-- Support for the following comparisons: +-- +-- eql_v2_encrypted <> eql_v2_encrypted +-- eql_v2_encrypted <> jsonb +-- jsonb <> eql_v2_encrypted +-- +-- There are multiple index terms that provide equality comparisons +-- - unique +-- - ore_64_8_v2 +-- - ore_cllw_8_v2 +-- +-- We check these index terms in this order and use the first one that exists for both parameters +-- +-- + +CREATE FUNCTION eql_v2.neq(a eql_v2_encrypted, b eql_v2_encrypted) + RETURNS boolean + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN NOT eql_v2.eq(a, b ); + END; +$$ LANGUAGE plpgsql; + + + +CREATE FUNCTION eql_v2."<>"(a eql_v2_encrypted, b eql_v2_encrypted) + RETURNS boolean + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN eql_v2.neq(a, b ); + END; +$$ LANGUAGE plpgsql; + + +CREATE OPERATOR <> ( + FUNCTION=eql_v2."<>", + LEFTARG=eql_v2_encrypted, + RIGHTARG=eql_v2_encrypted, + NEGATOR = =, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); + + +CREATE FUNCTION eql_v2."<>"(a eql_v2_encrypted, b jsonb) + RETURNS boolean + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN eql_v2.neq(a, b::eql_v2_encrypted); + END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR <> ( + FUNCTION=eql_v2."<>", + LEFTARG=eql_v2_encrypted, + RIGHTARG=jsonb, + NEGATOR = =, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); + + + +CREATE FUNCTION eql_v2."<>"(a jsonb, b eql_v2_encrypted) + RETURNS boolean + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN eql_v2.neq(a::eql_v2_encrypted, b); + END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR <> ( + FUNCTION=eql_v2."<>", + LEFTARG=jsonb, + RIGHTARG=eql_v2_encrypted, + NEGATOR = =, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); + + + + diff --git a/src/operators/<>_ore_cllw_u64_8_test.sql b/src/operators/<>_ore_cllw_u64_8_test.sql new file mode 100644 index 00000000..3e710eba --- /dev/null +++ b/src/operators/<>_ore_cllw_u64_8_test.sql @@ -0,0 +1,57 @@ +\set ON_ERROR_STOP on + +SELECT create_table_with_encrypted(); +SELECT seed_encrypted_json(); + + +-- ======================================================================== + + +-- ------------------------------------------------------------------------ +-- ------------------------------------------------------------------------ +-- +-- ore_cllw_u64_8 equality +-- +-- Test data is '{"hello": "world", "n": 42}' + +-- Paths +-- $ -> bca213de9ccce676fa849ff9c4807963 +-- $.hello -> a7cea93975ed8c01f861ccb6bd082784 +-- $.n -> 2517068c0d1f9d4d41d2c666211f785e +-- +-- + +DO $$ +DECLARE + sv eql_v2_encrypted; + term eql_v2_encrypted; + BEGIN + + -- This extracts the data associated with the field from the test eql_v2_encrypted + -- json n: 10 + sv := get_numeric_ste_vec_10()::eql_v2_encrypted; + -- extract the term at $.n returned as eql_v2_encrypted + term := sv->'2517068c0d1f9d4d41d2c666211f785e'; + + -- -- -- -- $.n + PERFORM assert_result( + format('eql_v2_encrypted <> eql_v2_encrypted with ore_cllw_u64_8 index term'), + format('SELECT e FROM encrypted WHERE (e->''2517068c0d1f9d4d41d2c666211f785e'') <> %L::eql_v2_encrypted', term)); + + PERFORM assert_count( + format('eql_v2_encrypted <> eql_v2_encrypted with ore index term'), + format('SELECT e FROM encrypted WHERE e->''2517068c0d1f9d4d41d2c666211f785e'' <> %L::eql_v2_encrypted', term), + 2); + + -- -- Check the $.hello path + -- -- Returned encrypted does not have ore_cllw_u64_8 + PERFORM assert_result( + format('eql_v2_encrypted <> eql_v2_encrypted with ore index term'), + format('SELECT e FROM encrypted WHERE e->''a7cea93975ed8c01f861ccb6bd082784'' <> %L::eql_v2_encrypted', term)); + + END; +$$ LANGUAGE plpgsql; + + + +SELECT drop_table_with_encrypted(); \ No newline at end of file diff --git a/src/operators/<>_ore_cllw_var_8_test.sql b/src/operators/<>_ore_cllw_var_8_test.sql new file mode 100644 index 00000000..02f8aef0 --- /dev/null +++ b/src/operators/<>_ore_cllw_var_8_test.sql @@ -0,0 +1,56 @@ +\set ON_ERROR_STOP on + +SELECT create_table_with_encrypted(); +SELECT seed_encrypted_json(); + + +-- ======================================================================== + + +-- ------------------------------------------------------------------------ +-- ------------------------------------------------------------------------ +-- +-- ore_cllw_u64_8 equality +-- +-- Test data is '{"hello": "world", "n": 42}' + +-- Paths +-- $ -> bca213de9ccce676fa849ff9c4807963 +-- $.hello -> a7cea93975ed8c01f861ccb6bd082784 +-- $.n -> 2517068c0d1f9d4d41d2c666211f785e +-- +-- +DO $$ +DECLARE + sv eql_v2_encrypted; + term eql_v2_encrypted; + BEGIN + + -- This extracts the data associated with the field from the test eql_v2_encrypted + -- json n: 10 + sv := get_numeric_ste_vec_10()::eql_v2_encrypted; + -- extract the term at $.n returned as eql_v2_encrypted + term := sv->'a7cea93975ed8c01f861ccb6bd082784'; + + -- -- -- -- $.n + PERFORM assert_result( + format('eql_v2_encrypted <> eql_v2_encrypted with ore_cllw_var_8 index term'), + format('SELECT e FROM encrypted WHERE (e->''a7cea93975ed8c01f861ccb6bd082784'') <> %L::eql_v2_encrypted', term)); + + PERFORM assert_count( + format('eql_v2_encrypted <> eql_v2_encrypted with ore_cllw_var_8 index term'), + format('SELECT e FROM encrypted WHERE e->''a7cea93975ed8c01f861ccb6bd082784'' <> %L::eql_v2_encrypted', term), + 2); + + -- -- Check the $.n path + -- -- Returned encrypted does not have ore_cllw_var_8 + PERFORM assert_result( + format('eql_v2_encrypted <> eql_v2_encrypted with ore_cllw_var_8 index term'), + format('SELECT e FROM encrypted WHERE e->''2517068c0d1f9d4d41d2c666211f785e'' <> %L::eql_v2_encrypted', term)); + + END; +$$ LANGUAGE plpgsql; + + + +SELECT drop_table_with_encrypted(); \ No newline at end of file diff --git a/src/operators/<>_ore_test.sql b/src/operators/<>_ore_test.sql new file mode 100644 index 00000000..df68fedf --- /dev/null +++ b/src/operators/<>_ore_test.sql @@ -0,0 +1,87 @@ +\set ON_ERROR_STOP on + +SELECT create_table_with_encrypted(); +SELECT seed_encrypted_json(); + + +-- +-- ORE - eql_v2_encrypted <> eql_v2_encrypted +-- +DO $$ +DECLARE + e eql_v2_encrypted; + ore_term jsonb; + BEGIN + + -- Record with a Numeric ORE term of 42 + e := create_encrypted_ore_json(42); + PERFORM seed_encrypted(e); + + PERFORM assert_result( + 'eql_v2_encrypted <> eql_v2_encrypted', + format('SELECT e FROM encrypted WHERE e <> %L::eql_v2_encrypted', e)); + + PERFORM assert_count( + format('eql_v2_encrypted <> eql_v2_encrypted'), + format('SELECT e FROM encrypted WHERE e <> %L;', e), + 3); + + e := create_encrypted_ore_json(20); + + PERFORM assert_result( + 'eql_v2_encrypted <> eql_v2_encrypted', + format('SELECT e FROM encrypted WHERE e <> %L::eql_v2_encrypted', e)); + + PERFORM assert_count( + format('eql_v2_encrypted <> eql_v2_encrypted'), + format('SELECT e FROM encrypted WHERE e <> %L;', e), + 3); + END; +$$ LANGUAGE plpgsql; + + +-- +-- ORE - eql_v2.gte(a eql_v2_encrypted, b eql_v2_encrypted) +-- +DO $$ +DECLARE + e eql_v2_encrypted; + ore_term jsonb; + BEGIN + -- Reset data + PERFORM seed_encrypted_json(); + + -- Record with a Numeric ORE term of 20 + e := create_encrypted_ore_json(20); + PERFORM seed_encrypted(e); + + PERFORM assert_result( + 'eql_v2.neq(a eql_v2_encrypted, b eql_v2_encrypted)', + format('SELECT e FROM encrypted WHERE eql_v2.neq(e, %L)', e)); + + -- include + PERFORM assert_count( + 'eql_v2.neq(a eql_v2_encrypted, b eql_v2_encrypted)', + format('SELECT e FROM encrypted WHERE eql_v2.neq(e, %L)', e), + 2); + + -- Record with a Numeric ORE term of 30 + e := create_encrypted_ore_json(30); + + PERFORM assert_result( + 'eql_v2.get(a eql_v2_encrypted, b eql_v2_encrypted)', + format('SELECT e FROM encrypted WHERE eql_v2.neq(e, %L)', e)); + + PERFORM assert_count( + 'eql_v2.get(a eql_v2_encrypted, b eql_v2_encrypted)', + format('SELECT e FROM encrypted WHERE eql_v2.neq(e, %L)', e), + 3); + END; +$$ LANGUAGE plpgsql; + + + +-- ======================================================================== + + +SELECT drop_table_with_encrypted(); \ No newline at end of file diff --git a/src/operators/<>_test.sql b/src/operators/<>_test.sql new file mode 100644 index 00000000..2959f0af --- /dev/null +++ b/src/operators/<>_test.sql @@ -0,0 +1,165 @@ +\set ON_ERROR_STOP on + +SELECT create_table_with_encrypted(); +SELECT seed_encrypted_json(); + + +-- +-- Unique equality - eql_v2_encrypted <> eql_v2_encrypted +-- +DO $$ +DECLARE + e eql_v2_encrypted; + BEGIN + + for i in 1..3 loop + e := create_encrypted_json(i, 'u'); + + PERFORM assert_count( + format('eql_v2_encrypted <> eql_v2_encrypted with unique index term %s of 3', i), + format('SELECT e FROM encrypted WHERE e <> %L;', e), + 2); + + end loop; + + -- record not in database + e := create_encrypted_json(91347, 'u'); + + PERFORM assert_no_result( + 'eql_v2_encrypted <> eql_v2_encrypted with no matching record', + format('SELECT e FROM encrypted WHERE e <> %L;', e)); + + END; +$$ LANGUAGE plpgsql; + + +-- ------------------------------------------------------------------------ +-- ------------------------------------------------------------------------ +-- +-- Unique equality - eql_v2.neq(eql_v2_encrypted, eql_v2_encrypted) +-- +DO $$ +DECLARE + e eql_v2_encrypted; + BEGIN + + for i in 1..3 loop + e := create_encrypted_json(i, 'u'); + + PERFORM assert_count( + format('eql_v2.neq(eql_v2_encrypted, eql_v2_encrypted) with unique index term %s of 3', i), + format('SELECT e FROM encrypted WHERE eql_v2.neq(e, %L);', e), + 2); + end loop; + + -- record not in database + e := create_encrypted_json(91347, 'u'); + + PERFORM assert_no_result( + 'eql_v2_encrypted <> eql_v2_encrypted with no matching record', + format('SELECT e FROM encrypted WHERE e <> %L;', e)); + + END; +$$ LANGUAGE plpgsql; + + + +-- ======================================================================== + +-- ------------------------------------------------------------------------ +-- ------------------------------------------------------------------------ +-- +-- Blake equality - eql_v2_encrypted <> eql_v2_encrypted +-- +DO $$ +DECLARE + e eql_v2_encrypted; + BEGIN + + for i in 1..3 loop + e := create_encrypted_json(i, 'b'); + + PERFORM assert_result( + format('eql_v2_encrypted <> eql_v2_encrypted with blake3 index term %s of 3', i), + format('SELECT e FROM encrypted WHERE e <> %L;', e)); + + end loop; + + -- remove the ore index term + e := create_encrypted_json(91347, 'b'); + + PERFORM assert_no_result( + 'eql_v2_encrypted <> eql_v2_encrypted with no matching record', + format('SELECT e FROM encrypted WHERE e <> %L;', e)); + + END; +$$ LANGUAGE plpgsql; + + +-- ------------------------------------------------------------------------ +-- ------------------------------------------------------------------------ +-- +-- Blake3 equality - eql_v2.neq(eql_v2_encrypted, eql_v2_encrypted) +-- +DO $$ +DECLARE + e eql_v2_encrypted; + BEGIN + + for i in 1..3 loop + e := create_encrypted_json(i, 'b'); + + PERFORM assert_result( + format('eql_v2.neq(eql_v2_encrypted, eql_v2_encrypted) with unique index term %s of 3', i), + format('SELECT e FROM encrypted WHERE eql_v2.neq(e, %L);', e)); + end loop; + + -- remove the ore index term + e := create_encrypted_json(91347, 'b'); + + PERFORM assert_no_result( + 'eql_v2.neq(eql_v2_encrypted, eql_v2_encrypted) with no matching record', + format('SELECT e FROM encrypted WHERE eql_v2.neq(e, %L);', e)); + + END; +$$ LANGUAGE plpgsql; + + +-- ------------------------------------------------------------------------ +-- ------------------------------------------------------------------------ +-- +-- Blake3 equality - eql_v2_encrypted = jsonb +-- +DO $$ +DECLARE + e jsonb; + BEGIN + for i in 1..3 loop + + -- remove the default + e := create_encrypted_json(i, 'b'); + + PERFORM assert_result( + format('eql_v2_encrypted = jsonb with unique index term %s of 3', i), + format('SELECT e FROM encrypted WHERE e <> %L::jsonb;', e)); + + PERFORM assert_result( + format('jsonb = eql_v2_encrypted with unique index term %s of 3', i), + format('SELECT e FROM encrypted WHERE %L::jsonb = e', e)); + end loop; + + e := create_encrypted_json(91347, 'b'); + + PERFORM assert_no_result( + 'eql_v2_encrypted = jsonb with no matching record', + format('SELECT e FROM encrypted WHERE e <> %L::jsonb', e)); + + PERFORM assert_no_result( + 'jsonb = eql_v2_encrypted with no matching record', + format('SELECT e FROM encrypted WHERE %L::jsonb = e', e)); + + END; +$$ LANGUAGE plpgsql; + + +SELECT drop_table_with_encrypted(); \ No newline at end of file diff --git a/src/operators/<@.sql b/src/operators/<@.sql new file mode 100644 index 00000000..c4350f0b --- /dev/null +++ b/src/operators/<@.sql @@ -0,0 +1,17 @@ +-- REQUIRE: src/schema.sql +-- REQUIRE: src/encrypted/types.sql +-- REQUIRE: src/ste_vec/functions.sql + + + +CREATE FUNCTION eql_v2."<@"(a eql_v2_encrypted, b eql_v2_encrypted) +RETURNS boolean AS $$ + -- Contains with reversed arguments + SELECT eql_v2.ste_vec_contains(b, a) +$$ LANGUAGE SQL; + +CREATE OPERATOR <@( + FUNCTION=eql_v2."<@", + LEFTARG=eql_v2_encrypted, + RIGHTARG=eql_v2_encrypted +); diff --git a/src/operators/<@_test.sql b/src/operators/<@_test.sql new file mode 100644 index 00000000..4eea0364 --- /dev/null +++ b/src/operators/<@_test.sql @@ -0,0 +1,44 @@ +\set ON_ERROR_STOP on + +SELECT create_table_with_encrypted(); +SELECT seed_encrypted_json(); + +-- ------------------------------------------------------------------------ +-- ------------------------------------------------------------------------ +-- +-- ore_cllw_u64_8 equality +-- +-- Test data is in form '{"hello": "{one | two | three}", "n": {10 | 20 | 30} }' +-- +-- Paths +-- $ -> bca213de9ccce676fa849ff9c4807963 +-- $.hello -> a7cea93975ed8c01f861ccb6bd082784 +-- $.n -> 2517068c0d1f9d4d41d2c666211f785e +-- +-- +DO $$ +DECLARE + sv eql_v2_encrypted; + term eql_v2_encrypted; + BEGIN + + -- This extracts the data associated with the field from the test eql_v2_encrypted + sv := get_numeric_ste_vec_10()::eql_v2_encrypted; + -- extract the term at $.n returned as eql_v2_encrypted + term := sv->'a7cea93975ed8c01f861ccb6bd082784'; + + -- -- -- -- $.n + PERFORM assert_result( + format('eql_v2_encrypted = eql_v2_encrypted with ore_cllw_u64_8 index term'), + format('SELECT e FROM encrypted WHERE %L::eql_v2_encrypted <@ e', term)); + + PERFORM assert_count( + format('eql_v2_encrypted = eql_v2_encrypted with ore index term'), + format('SELECT e FROM encrypted WHERE %L::eql_v2_encrypted <@ e', term), + 1); + + END; +$$ LANGUAGE plpgsql; + + +SELECT drop_table_with_encrypted(); \ No newline at end of file diff --git a/src/operators/<_test.sql b/src/operators/<_test.sql new file mode 100644 index 00000000..232d6867 --- /dev/null +++ b/src/operators/<_test.sql @@ -0,0 +1,157 @@ +\set ON_ERROR_STOP on + +SELECT create_table_with_encrypted(); +SELECT seed_encrypted_json(); + + +-- ------------------------------------------------------------------------ +-- ------------------------------------------------------------------------ +-- +-- eql_v2_encrypted < eql_v2_encrypted with ore_cllw_u64_8 index +-- +-- Test data is in form '{"hello": "{one | two | three}", "n": {10 | 20 | 30} }' +-- +-- Paths +-- $ -> bca213de9ccce676fa849ff9c4807963 +-- $.hello -> a7cea93975ed8c01f861ccb6bd082784 +-- $.n -> 2517068c0d1f9d4d41d2c666211f785e +-- +-- -- +DO $$ +DECLARE + sv eql_v2_encrypted; + term eql_v2_encrypted; + BEGIN + + -- This extracts the data associated with the field from the test eql_v2_encrypted + -- json n: 30 + sv := get_numeric_ste_vec_30()::eql_v2_encrypted; + -- extract the term at $.n returned as eql_v2_encrypted + term := sv->'2517068c0d1f9d4d41d2c666211f785e'; + + -- -- -- -- $.n + PERFORM assert_result( + format('eql_v2_encrypted < eql_v2_encrypted with ore_cllw_u64_8 index term'), + format('SELECT e FROM encrypted WHERE e->''2517068c0d1f9d4d41d2c666211f785e'' < %L::eql_v2_encrypted', term)); + + PERFORM assert_count( + format('eql_v2_encrypted < eql_v2_encrypted with ore index term'), + format('SELECT e FROM encrypted WHERE e->''2517068c0d1f9d4d41d2c666211f785e'' < %L::eql_v2_encrypted', term), + 2); + + -- -- Check the $.hello path + -- -- Returned encrypted does not have ore_cllw_u64_8 + PERFORM assert_no_result( + format('eql_v2_encrypted < eql_v2_encrypted with ore index term'), + format('SELECT e FROM encrypted WHERE e->''a7cea93975ed8c01f861ccb6bd082784'' < %L::eql_v2_encrypted', term)); + + END; +$$ LANGUAGE plpgsql; + + +-- ------------------------------------------------------------------------ +-- ------------------------------------------------------------------------ +-- +-- eql_v2_encrypted < eql_v2_encrypted with ore_cllw_var_8 index +-- +-- Test data is in form '{"hello": "{one | two | three}", "n": {10 | 20 | 30} }' +-- +-- Paths +-- $ -> bca213de9ccce676fa849ff9c4807963 +-- $.hello -> a7cea93975ed8c01f861ccb6bd082784 +-- $.n -> 2517068c0d1f9d4d41d2c666211f785e +-- +-- -- +DO $$ +DECLARE + sv eql_v2_encrypted; + term eql_v2_encrypted; + BEGIN + + -- This extracts the data associated with the field from the test eql_v2_encrypted + sv := get_numeric_ste_vec_30()::eql_v2_encrypted; + -- extract the term at $.n returned as eql_v2_encrypted + term := sv->'a7cea93975ed8c01f861ccb6bd082784'; + + -- -- -- -- $.n + PERFORM assert_result( + format('eql_v2_encrypted < eql_v2_encrypted with ore_cllw_var_8 index term'), + format('SELECT e FROM encrypted WHERE e->''a7cea93975ed8c01f861ccb6bd082784'' < %L::eql_v2_encrypted', term)); + + PERFORM assert_count( + format('eql_v2_encrypted < eql_v2_encrypted with ore_cllw_var_8 index term'), + format('SELECT e FROM encrypted WHERE e->''a7cea93975ed8c01f861ccb6bd082784'' < %L::eql_v2_encrypted', term), + 1); + + -- -- Check the $.n path + -- -- Returned encrypted does not have ore_cllw_var_8 + PERFORM assert_no_result( + format('eql_v2_encrypted < eql_v2_encrypted with ore_cllw_var_8 index term'), + format('SELECT e FROM encrypted WHERE e->''2517068c0d1f9d4d41d2c666211f785e'' < %L::eql_v2_encrypted', term)); + + END; +$$ LANGUAGE plpgsql; + + + +-- +-- ORE - eql_v2_encrypted < eql_v2_encrypted +-- +DO $$ +DECLARE + e eql_v2_encrypted; + ore_term eql_v2_encrypted; + BEGIN + SELECT ore.e FROM ore WHERE id = 42 INTO ore_term; + + PERFORM assert_count( + 'eql_v2_encrypted < eql_v2_encrypted', + format('SELECT id FROM ore WHERE e < %L ORDER BY e DESC', ore_term), + 41); + + -- Record with a Numeric ORE term of 1 + e := create_encrypted_ore_json(1); + + PERFORM assert_no_result( + format('eql_v2_encrypted < eql_v2_encrypted'), + format('SELECT e FROM encrypted WHERE e < %L;', e)); + + -- Record with a Numeric ORE term of 42 + e := create_encrypted_ore_json(42); + + PERFORM assert_result( + 'eql_v2_encrypted < eql_v2_encrypted', + format('SELECT e FROM encrypted WHERE e < %L', e)); + + PERFORM assert_count( + 'eql_v2_encrypted < eql_v2_encrypted', + format('SELECT e FROM encrypted WHERE e < %L', e), + 3); + END; +$$ LANGUAGE plpgsql; + + +-- -- +-- -- ORE - eql_v2.lt(a eql_v2_encrypted, b eql_v2_encrypted) +-- -- +DO $$ +DECLARE + e eql_v2_encrypted; + ore_term jsonb; + BEGIN + -- Record with a Numeric ORE term of 42 + e := create_encrypted_ore_json(42); + + PERFORM assert_result( + 'eql_v2.lt(a eql_v2_encrypted, b eql_v2_encrypted)', + format('SELECT e FROM encrypted WHERE eql_v2.lt(e, %L)', e)); + + PERFORM assert_count( + 'eql_v2.lt(a eql_v2_encrypted, b eql_v2_encrypted)', + format('SELECT e FROM encrypted WHERE eql_v2.lt(e, %L)', e), + 3); + END; +$$ LANGUAGE plpgsql; + + +SELECT drop_table_with_encrypted(); \ No newline at end of file diff --git a/src/operators/=.sql b/src/operators/=.sql new file mode 100644 index 00000000..3711f424 --- /dev/null +++ b/src/operators/=.sql @@ -0,0 +1,134 @@ +-- REQUIRE: src/encrypted/types.sql +-- REQUIRE: src/unique/types.sql +-- REQUIRE: src/unique/functions.sql +-- REQUIRE: src/ore/types.sql +-- REQUIRE: src/ore/functions.sql +-- REQUIRE: src/ore/operators.sql +-- REQUIRE: src/blake3/types.sql +-- REQUIRE: src/blake3/functions.sql +-- REQUIRE: src/ore_cllw_u64_8/types.sql +-- REQUIRE: src/ore_cllw_u64_8/functions.sql +-- REQUIRE: src/ore_cllw_u64_8/operators.sql + +-- Operators for equality comparisons of eql_v2_encrypted types +-- +-- Support for the following comparisons: +-- +-- eql_v2_encrypted = eql_v2_encrypted +-- eql_v2_encrypted = jsonb +-- jsonb = eql_v2_encrypted +-- +-- There are multiple index terms that provide equality comparisons +-- - unique +-- - ore_64_8_v2 +-- +-- +-- We check these index terms in this order and use the first one that exists for both parameters +-- +-- + + +CREATE FUNCTION eql_v2.eq(a eql_v2_encrypted, b eql_v2_encrypted) + RETURNS boolean + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + + BEGIN + RETURN eql_v2.unique(a) = eql_v2.unique(b); + EXCEPTION WHEN OTHERS THEN + -- PERFORM eql_v2.log('No unique index'); + END; + + BEGIN + RETURN eql_v2.blake3(a) = eql_v2.blake3(b); + EXCEPTION WHEN OTHERS THEN + -- PERFORM eql_v2.log('No blake3 index'); + END; + + BEGIN + RETURN eql_v2.ore_cllw_u64_8(a) = eql_v2.ore_cllw_u64_8(b); + EXCEPTION WHEN OTHERS THEN + -- PERFORM eql_v2.log('No ore_cllw_u64_8 index'); + END; + + BEGIN + RETURN eql_v2.ore_cllw_var_8(a) = eql_v2.ore_cllw_var_8(b); + EXCEPTION WHEN OTHERS THEN + -- PERFORM eql_v2.log('No ore_cllw_u64_8 index'); + END; + + BEGIN + RETURN eql_v2.ore_64_8_v2(a) = eql_v2.ore_64_8_v2(b); + EXCEPTION WHEN OTHERS THEN + -- PERFORM eql_v2.log('No ore_64_8_v2 index'); + END; + + RETURN false; + END; +$$ LANGUAGE plpgsql; + + + +CREATE FUNCTION eql_v2."="(a eql_v2_encrypted, b eql_v2_encrypted) + RETURNS boolean + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN eql_v2.eq(a, b); + END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR = ( + FUNCTION=eql_v2."=", + LEFTARG=eql_v2_encrypted, + RIGHTARG=eql_v2_encrypted, + NEGATOR = <>, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); + + +CREATE FUNCTION eql_v2."="(a eql_v2_encrypted, b jsonb) + RETURNS boolean + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN eql_v2.eq(a, b::eql_v2_encrypted); + END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR = ( + FUNCTION=eql_v2."=", + LEFTARG=eql_v2_encrypted, + RIGHTARG=jsonb, + NEGATOR = <>, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); + + +CREATE FUNCTION eql_v2."="(a jsonb, b eql_v2_encrypted) + RETURNS boolean + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN eql_v2.eq(a::eql_v2_encrypted, b); + END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR = ( + FUNCTION=eql_v2."=", + LEFTARG=jsonb, + RIGHTARG=eql_v2_encrypted, + NEGATOR = <>, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); + diff --git a/src/operators/=_ore_cllw_u64_8_test.sql b/src/operators/=_ore_cllw_u64_8_test.sql new file mode 100644 index 00000000..f0617950 --- /dev/null +++ b/src/operators/=_ore_cllw_u64_8_test.sql @@ -0,0 +1,56 @@ +\set ON_ERROR_STOP on + +SELECT create_table_with_encrypted(); +SELECT seed_encrypted_json(); + + +-- ======================================================================== + + +-- ------------------------------------------------------------------------ +-- ------------------------------------------------------------------------ +-- +-- ore_cllw_u64_8 equality +-- +-- Test data is in form '{"hello": "{one | two | three}", "n": {10 | 20 | 30} }' +-- +-- Paths +-- $ -> bca213de9ccce676fa849ff9c4807963 +-- $.hello -> a7cea93975ed8c01f861ccb6bd082784 +-- $.n -> 2517068c0d1f9d4d41d2c666211f785e +-- +-- +DO $$ +DECLARE + sv eql_v2_encrypted; + term eql_v2_encrypted; + BEGIN + + -- This extracts the data associated with the field from the test eql_v2_encrypted + -- json n: 10 + sv := get_numeric_ste_vec_10()::eql_v2_encrypted; + -- extract the term at $.n returned as eql_v2_encrypted + term := sv->'2517068c0d1f9d4d41d2c666211f785e'; + + -- -- -- -- $.n + PERFORM assert_result( + format('eql_v2_encrypted = eql_v2_encrypted with ore_cllw_u64_8 index term'), + format('SELECT e FROM encrypted WHERE (e->''2517068c0d1f9d4d41d2c666211f785e'') = %L::eql_v2_encrypted', term)); + + PERFORM assert_count( + format('eql_v2_encrypted = eql_v2_encrypted with ore index term'), + format('SELECT e FROM encrypted WHERE e->''2517068c0d1f9d4d41d2c666211f785e'' = %L::eql_v2_encrypted', term), + 1); + + -- -- Check the $.hello path + -- -- Returned encrypted does not have ore_cllw_u64_8 + PERFORM assert_no_result( + format('eql_v2_encrypted = eql_v2_encrypted with ore index term'), + format('SELECT e FROM encrypted WHERE e->''a7cea93975ed8c01f861ccb6bd082784'' = %L::eql_v2_encrypted', term)); + + END; +$$ LANGUAGE plpgsql; + + + +SELECT drop_table_with_encrypted(); \ No newline at end of file diff --git a/src/operators/=_ore_cllw_var_8_test.sql b/src/operators/=_ore_cllw_var_8_test.sql new file mode 100644 index 00000000..9cbe1952 --- /dev/null +++ b/src/operators/=_ore_cllw_var_8_test.sql @@ -0,0 +1,53 @@ +\set ON_ERROR_STOP on + +SELECT create_table_with_encrypted(); +SELECT seed_encrypted_json(); + + +-- ------------------------------------------------------------------------ +-- ------------------------------------------------------------------------ +-- +-- ore_cllw_u64_8 equality +-- +-- Test data is in form '{"hello": "{one | two | three}", "n": {10 | 20 | 30} }' +-- +-- Paths +-- $ -> bca213de9ccce676fa849ff9c4807963 +-- $.hello -> a7cea93975ed8c01f861ccb6bd082784 +-- $.n -> 2517068c0d1f9d4d41d2c666211f785e +-- +-- +DO $$ +DECLARE + sv eql_v2_encrypted; + term eql_v2_encrypted; + BEGIN + + -- This extracts the data associated with the field from the test eql_v2_encrypted + -- json n: 10 + sv := get_numeric_ste_vec_10()::eql_v2_encrypted; + -- extract the term at $.n returned as eql_v2_encrypted + term := sv->'a7cea93975ed8c01f861ccb6bd082784'; + + -- -- -- -- $.n + PERFORM assert_result( + format('eql_v2_encrypted = eql_v2_encrypted with ore_cllw_u64_8 index term'), + format('SELECT e FROM encrypted WHERE (e->''a7cea93975ed8c01f861ccb6bd082784'') = %L::eql_v2_encrypted', term)); + + PERFORM assert_count( + format('eql_v2_encrypted = eql_v2_encrypted with ore index term'), + format('SELECT e FROM encrypted WHERE e->''a7cea93975ed8c01f861ccb6bd082784'' = %L::eql_v2_encrypted', term), + 1); + + -- -- Check the $.n path + -- -- Returned encrypted does not have ore_cllw_u64_8 + PERFORM assert_no_result( + format('eql_v2_encrypted = eql_v2_encrypted with ore index term'), + format('SELECT e FROM encrypted WHERE e->''2517068c0d1f9d4d41d2c666211f785e'' = %L::eql_v2_encrypted', term)); + + END; +$$ LANGUAGE plpgsql; + + + +SELECT drop_table_with_encrypted(); \ No newline at end of file diff --git a/src/operators/=_ore_test.sql b/src/operators/=_ore_test.sql new file mode 100644 index 00000000..2f75a8d0 --- /dev/null +++ b/src/operators/=_ore_test.sql @@ -0,0 +1,87 @@ +\set ON_ERROR_STOP on + +SELECT create_table_with_encrypted(); +SELECT seed_encrypted_json(); + + +-- +-- ORE - eql_v2_encrypted = eql_v2_encrypted +-- +DO $$ +DECLARE + e eql_v2_encrypted; + ore_term jsonb; + BEGIN + + -- Record with a Numeric ORE term of 42 + e := create_encrypted_ore_json(42); + PERFORM seed_encrypted(e); + + PERFORM assert_result( + 'eql_v2_encrypted = eql_v2_encrypted', + format('SELECT e FROM encrypted WHERE e = %L::eql_v2_encrypted', e)); + + PERFORM assert_count( + format('eql_v2_encrypted = eql_v2_encrypted'), + format('SELECT e FROM encrypted WHERE e = %L;', e), + 1); + + e := create_encrypted_ore_json(20); + + PERFORM assert_result( + 'eql_v2_encrypted = eql_v2_encrypted', + format('SELECT e FROM encrypted WHERE e = %L::eql_v2_encrypted', e)); + + PERFORM assert_count( + format('eql_v2_encrypted = eql_v2_encrypted'), + format('SELECT e FROM encrypted WHERE e = %L;', e), + 1); + END; +$$ LANGUAGE plpgsql; + + +-- +-- ORE - eql_v2.gte(a eql_v2_encrypted, b eql_v2_encrypted) +-- +DO $$ +DECLARE + e eql_v2_encrypted; + ore_term jsonb; + BEGIN + -- Reset data + PERFORM seed_encrypted_json(); + + -- Record with a Numeric ORE term of 20 + e := create_encrypted_ore_json(20); + PERFORM seed_encrypted(e); + + PERFORM assert_result( + 'eql_v2.eq(a eql_v2_encrypted, b eql_v2_encrypted)', + format('SELECT e FROM encrypted WHERE eql_v2.eq(e, %L)', e)); + + -- include + PERFORM assert_count( + 'eql_v2.eq(a eql_v2_encrypted, b eql_v2_encrypted)', + format('SELECT e FROM encrypted WHERE eql_v2.eq(e, %L)', e), + 2); + + -- Record with a Numeric ORE term of 30 + e := create_encrypted_ore_json(30); + + PERFORM assert_result( + 'eql_v2.get(a eql_v2_encrypted, b eql_v2_encrypted)', + format('SELECT e FROM encrypted WHERE eql_v2.eq(e, %L)', e)); + + PERFORM assert_count( + 'eql_v2.get(a eql_v2_encrypted, b eql_v2_encrypted)', + format('SELECT e FROM encrypted WHERE eql_v2.eq(e, %L)', e), + 1); + END; +$$ LANGUAGE plpgsql; + + + +-- ======================================================================== + + +SELECT drop_table_with_encrypted(); \ No newline at end of file diff --git a/src/operators/=_test.sql b/src/operators/=_test.sql new file mode 100644 index 00000000..230e3c79 --- /dev/null +++ b/src/operators/=_test.sql @@ -0,0 +1,196 @@ +\set ON_ERROR_STOP on + +SELECT create_table_with_encrypted(); +SELECT seed_encrypted_json(); + + +-- +-- Unique equality - eql_v2_encrypted = eql_v2_encrypted +-- +DO $$ +DECLARE + e eql_v2_encrypted; + BEGIN + + for i in 1..3 loop + e := create_encrypted_json(i, 'u'); + + PERFORM assert_result( + format('eql_v2_encrypted = eql_v2_encrypted with unique index term %s of 3', i), + format('SELECT e FROM encrypted WHERE e = %L;', e)); + + end loop; + + -- remove the ore index term + e := create_encrypted_json(91347, 'u'); + + PERFORM assert_no_result( + 'eql_v2_encrypted = eql_v2_encrypted with no matching record', + format('SELECT e FROM encrypted WHERE e = %L;', e)); + + END; +$$ LANGUAGE plpgsql; + + +-- +-- Unique equality - eql_v2.eq(eql_v2_encrypted, eql_v2_encrypted) +-- +DO $$ +DECLARE + e eql_v2_encrypted; + BEGIN + + for i in 1..3 loop + e := create_encrypted_json(i)::jsonb-'o'; + + PERFORM assert_result( + format('eql_v2.eq(eql_v2_encrypted, eql_v2_encrypted) with unique index term %s of 3', i), + format('SELECT e FROM encrypted WHERE eql_v2.eq(e, %L);', e)); + end loop; + + -- remove the ore index term + e := create_encrypted_json(91347)::jsonb-'o'; + + PERFORM assert_no_result( + 'eql_v2.eq(eql_v2_encrypted, eql_v2_encrypted) with no matching record', + format('SELECT e FROM encrypted WHERE eql_v2.eq(e, %L);', e)); + + END; +$$ LANGUAGE plpgsql; + + +-- +-- Unique equality - eql_v2_encrypted = jsonb +-- +DO $$ +DECLARE + e jsonb; + BEGIN + for i in 1..3 loop + + -- remove the default + e := create_encrypted_json(i)::jsonb-'o'; + + PERFORM assert_result( + format('eql_v2_encrypted = jsonb with unique index term %s of 3', i), + format('SELECT e FROM encrypted WHERE e = %L::jsonb;', e)); + + PERFORM assert_result( + format('jsonb = eql_v2_encrypted with unique index term %s of 3', i), + format('SELECT e FROM encrypted WHERE %L::jsonb = e', e)); + end loop; + + e := create_encrypted_json(91347)::jsonb-'o'; + + PERFORM assert_no_result( + 'eql_v2_encrypted = jsonb with no matching record', + format('SELECT e FROM encrypted WHERE e = %L::jsonb', e)); + + PERFORM assert_no_result( + 'jsonb = eql_v2_encrypted with no matching record', + format('SELECT e FROM encrypted WHERE %L::jsonb = e', e)); + + END; +$$ LANGUAGE plpgsql; + + + +-- ======================================================================== + +-- ------------------------------------------------------------------------ +-- ------------------------------------------------------------------------ +-- +-- Blake equality - eql_v2_encrypted = eql_v2_encrypted +-- +DO $$ +DECLARE + e eql_v2_encrypted; + BEGIN + + for i in 1..3 loop + e := create_encrypted_json(i, 'b'); + + PERFORM assert_result( + format('eql_v2_encrypted = eql_v2_encrypted with unique index term %s of 3', i), + format('SELECT e FROM encrypted WHERE e = %L;', e)); + + end loop; + + -- remove the ore index term + e := create_encrypted_json(91347, 'b'); + + PERFORM assert_no_result( + 'eql_v2_encrypted = eql_v2_encrypted with no matching record', + format('SELECT e FROM encrypted WHERE e = %L;', e)); + + END; +$$ LANGUAGE plpgsql; + + +-- ------------------------------------------------------------------------ +-- ------------------------------------------------------------------------ +-- +-- Blake3 equality - eql_v2.eq(eql_v2_encrypted, eql_v2_encrypted) +-- +DO $$ +DECLARE + e eql_v2_encrypted; + BEGIN + + for i in 1..3 loop + e := create_encrypted_json(i, 'b'); + + PERFORM assert_result( + format('eql_v2.eq(eql_v2_encrypted, eql_v2_encrypted) with unique index term %s of 3', i), + format('SELECT e FROM encrypted WHERE eql_v2.eq(e, %L);', e)); + end loop; + + -- remove the ore index term + e := create_encrypted_json(91347, 'b'); + + PERFORM assert_no_result( + 'eql_v2.eq(eql_v2_encrypted, eql_v2_encrypted) with no matching record', + format('SELECT e FROM encrypted WHERE eql_v2.eq(e, %L);', e)); + + END; +$$ LANGUAGE plpgsql; + + +-- ------------------------------------------------------------------------ +-- ------------------------------------------------------------------------ +-- +-- Blake3 equality - eql_v2_encrypted = jsonb +-- +DO $$ +DECLARE + e jsonb; + BEGIN + for i in 1..3 loop + + -- remove the default + e := create_encrypted_json(i, 'b'); + + PERFORM assert_result( + format('eql_v2_encrypted = jsonb with unique index term %s of 3', i), + format('SELECT e FROM encrypted WHERE e = %L::jsonb;', e)); + + PERFORM assert_result( + format('jsonb = eql_v2_encrypted with unique index term %s of 3', i), + format('SELECT e FROM encrypted WHERE %L::jsonb = e', e)); + end loop; + + e := create_encrypted_json(91347, 'b'); + + PERFORM assert_no_result( + 'eql_v2_encrypted = jsonb with no matching record', + format('SELECT e FROM encrypted WHERE e = %L::jsonb', e)); + + PERFORM assert_no_result( + 'jsonb = eql_v2_encrypted with no matching record', + format('SELECT e FROM encrypted WHERE %L::jsonb = e', e)); + + END; +$$ LANGUAGE plpgsql; + + +SELECT drop_table_with_encrypted(); \ No newline at end of file diff --git a/src/operators/>.sql b/src/operators/>.sql new file mode 100644 index 00000000..70a71759 --- /dev/null +++ b/src/operators/>.sql @@ -0,0 +1,117 @@ +-- REQUIRE: src/encrypted/types.sql +-- REQUIRE: src/ore/types.sql +-- REQUIRE: src/ore/functions.sql +-- REQUIRE: src/ore/operators.sql + + +-- Operators for < less than comparisons of eql_v2_encrypted types +-- +-- Support for the following comparisons: +-- +-- eql_v2_encrypted = eql_v2_encrypted +-- eql_v2_encrypted = jsonb +-- jsonb = eql_v2_encrypted +-- +-- There are multiple index terms that provide equality comparisons +-- - ore_64_8_v2 +-- - ore_cllw_8_v2 +-- +-- We check these index terms in this order and use the first one that exists for both parameters +-- +-- + +-- WHERE a > b +-- WHERE eql_v2.gt(a, b) +-- + + + +CREATE FUNCTION eql_v2.gt(a eql_v2_encrypted, b eql_v2_encrypted) +RETURNS boolean +AS $$ + BEGIN + + BEGIN + RETURN eql_v2.ore_cllw_u64_8(a) > eql_v2.ore_cllw_u64_8(b); + EXCEPTION WHEN OTHERS THEN + -- PERFORM eql_v2.log('eql_v2.gt no ore_cllw_u64_8 index'); + END; + + BEGIN + RETURN eql_v2.ore_cllw_var_8(a) > eql_v2.ore_cllw_var_8(b); + EXCEPTION WHEN OTHERS THEN + -- PERFORM eql_v2.log('eql_v2.gt no ore_cllw_var_8 index'); + END; + + BEGIN + RETURN eql_v2.ore_64_8_v2(a) > eql_v2.ore_64_8_v2(b); + EXCEPTION WHEN OTHERS THEN + -- PERFORM eql_v2.log('eql_v2.gt no ore_64_8_v2 index'); + END; + + RETURN false; + END; +$$ LANGUAGE plpgsql; + + + +CREATE FUNCTION eql_v2.">"(a eql_v2_encrypted, b eql_v2_encrypted) +RETURNS boolean +AS $$ + BEGIN + RETURN eql_v2.gt(a, b); + END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR >( + FUNCTION=eql_v2.">", + LEFTARG=eql_v2_encrypted, + RIGHTARG=eql_v2_encrypted, + COMMUTATOR = <, + NEGATOR = <=, + RESTRICT = scalarltsel, + JOIN = scalarltjoinsel +); + + + +CREATE FUNCTION eql_v2.">"(a eql_v2_encrypted, b jsonb) +RETURNS boolean +AS $$ + BEGIN + RETURN eql_v2.gt(a, b::eql_v2_encrypted); + END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR >( + FUNCTION = eql_v2.">", + LEFTARG = eql_v2_encrypted, + RIGHTARG = jsonb, + COMMUTATOR = <, + NEGATOR = <=, + RESTRICT = scalarltsel, + JOIN = scalarltjoinsel +); + + + +CREATE FUNCTION eql_v2.">"(a jsonb, b eql_v2_encrypted) +RETURNS boolean +AS $$ + BEGIN + RETURN eql_v2.gt(a::eql_v2_encrypted, b); + END; +$$ LANGUAGE plpgsql; + + +CREATE OPERATOR >( + FUNCTION = eql_v2.">", + LEFTARG = jsonb, + RIGHTARG = eql_v2_encrypted, + COMMUTATOR = <, + NEGATOR = <=, + RESTRICT = scalarltsel, + JOIN = scalarltjoinsel +); + + diff --git a/src/operators/>=.sql b/src/operators/>=.sql new file mode 100644 index 00000000..37812226 --- /dev/null +++ b/src/operators/>=.sql @@ -0,0 +1,113 @@ +-- REQUIRE: src/encrypted/types.sql +-- REQUIRE: src/ore/types.sql +-- REQUIRE: src/ore/functions.sql +-- REQUIRE: src/ore/operators.sql + + +-- Operators for < less than comparisons of eql_v2_encrypted types +-- +-- Support for the following comparisons: +-- +-- eql_v2_encrypted = eql_v2_encrypted +-- eql_v2_encrypted = jsonb +-- jsonb = eql_v2_encrypted +-- +-- There are multiple index terms that provide equality comparisons +-- - ore_64_8_v2 +-- - ore_cllw_8_v2 +-- +-- We check these index terms in this order and use the first one that exists for both parameters +-- +-- + + +CREATE FUNCTION eql_v2.gte(a eql_v2_encrypted, b eql_v2_encrypted) + RETURNS boolean +AS $$ + BEGIN + + BEGIN + RETURN eql_v2.ore_cllw_u64_8(a) >= eql_v2.ore_cllw_u64_8(b); + EXCEPTION WHEN OTHERS THEN + -- PERFORM eql_v2.log('eql_v2.gte no ore_cllw_u64_8 index'); + END; + + BEGIN + RETURN eql_v2.ore_cllw_var_8(a) >= eql_v2.ore_cllw_var_8(b); + EXCEPTION WHEN OTHERS THEN + -- PERFORM eql_v2.log('eql_v2.gte no ore_cllw_var_8 index'); + END; + + BEGIN + RETURN eql_v2.ore_64_8_v2(a) >= eql_v2.ore_64_8_v2(b); + EXCEPTION WHEN OTHERS THEN + -- PERFORM eql_v2.log('eql_v2.gte no ore_64_8_v2 index'); + END; + + RETURN false; + END; +$$ LANGUAGE plpgsql; + + + +CREATE FUNCTION eql_v2.">="(a eql_v2_encrypted, b eql_v2_encrypted) + RETURNS boolean +AS $$ + BEGIN + RETURN eql_v2.gte(a, b); + END; +$$ LANGUAGE plpgsql; + + +CREATE OPERATOR >=( + FUNCTION = eql_v2.">=", + LEFTARG = eql_v2_encrypted, + RIGHTARG = eql_v2_encrypted, + COMMUTATOR = <=, + NEGATOR = <, + RESTRICT = scalarltsel, + JOIN = scalarltjoinsel +); + + + +CREATE FUNCTION eql_v2.">="(a eql_v2_encrypted, b jsonb) +RETURNS boolean +AS $$ + BEGIN + RETURN eql_v2.gte(a, b::eql_v2_encrypted); + END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR >=( + FUNCTION = eql_v2.">=", + LEFTARG = eql_v2_encrypted, + RIGHTARG=jsonb, + COMMUTATOR = <=, + NEGATOR = <, + RESTRICT = scalarltsel, + JOIN = scalarltjoinsel +); + + + +CREATE FUNCTION eql_v2.">="(a jsonb, b eql_v2_encrypted) +RETURNS boolean +AS $$ + BEGIN + RETURN eql_v2.gte(a::eql_v2_encrypted, b); + END; +$$ LANGUAGE plpgsql; + + +CREATE OPERATOR >=( + FUNCTION = eql_v2.">=", + LEFTARG = jsonb, + RIGHTARG =eql_v2_encrypted, + COMMUTATOR = <=, + NEGATOR = <, + RESTRICT = scalarltsel, + JOIN = scalarltjoinsel +); + + diff --git a/src/operators/>=_test.sql b/src/operators/>=_test.sql new file mode 100644 index 00000000..93cc9699 --- /dev/null +++ b/src/operators/>=_test.sql @@ -0,0 +1,173 @@ +\set ON_ERROR_STOP on + +SELECT create_table_with_encrypted(); +SELECT seed_encrypted_json(); + + +-- ------------------------------------------------------------------------ +-- ------------------------------------------------------------------------ +-- +-- eql_v2_encrypted < eql_v2_encrypted with ore_cllw_var_8 index +-- +-- Test data is in form '{"hello": "{one | two | three}", "n": {10 | 20 | 30} }' +-- +-- Paths +-- $ -> bca213de9ccce676fa849ff9c4807963 +-- $.hello -> a7cea93975ed8c01f861ccb6bd082784 +-- $.n -> 2517068c0d1f9d4d41d2c666211f785e +-- +-- -- +DO $$ +DECLARE + sv eql_v2_encrypted; + term eql_v2_encrypted; + BEGIN + + -- This extracts the data associated with the field from the test eql_v2_encrypted + -- json n: 30 + sv := get_numeric_ste_vec_10()::eql_v2_encrypted; + -- extract the term at $.n returned as eql_v2_encrypted + term := sv->'2517068c0d1f9d4d41d2c666211f785e'; + + -- -- -- -- $.n + PERFORM assert_result( + format('eql_v2_encrypted >= eql_v2_encrypted with ore_cllw_u64_8 index term'), + format('SELECT e FROM encrypted WHERE e->''2517068c0d1f9d4d41d2c666211f785e'' >= %L::eql_v2_encrypted', term)); + + PERFORM assert_count( + format('eql_v2_encrypted >= eql_v2_encrypted with ore index term'), + format('SELECT e FROM encrypted WHERE e->''2517068c0d1f9d4d41d2c666211f785e'' >= %L::eql_v2_encrypted', term), + 3); + + -- -- Check the $.hello path + -- -- Returned encrypted does not have ore_cllw_u64_8 + PERFORM assert_no_result( + format('eql_v2_encrypted >= eql_v2_encrypted with ore index term'), + format('SELECT e FROM encrypted WHERE e->''a7cea93975ed8c01f861ccb6bd082784'' >= %L::eql_v2_encrypted', term)); + + END; +$$ LANGUAGE plpgsql; + + +-- ------------------------------------------------------------------------ +-- ------------------------------------------------------------------------ +-- +-- eql_v2_encrypted < eql_v2_encrypted with ore_cllw_var_8 index +-- +-- Test data is in form '{"hello": "{one | two | three}", "n": {10 | 20 | 30} }' +-- +-- Paths +-- $ -> bca213de9ccce676fa849ff9c4807963 +-- $.hello -> a7cea93975ed8c01f861ccb6bd082784 +-- $.n -> 2517068c0d1f9d4d41d2c666211f785e +-- +-- -- +DO $$ +DECLARE + sv eql_v2_encrypted; + term eql_v2_encrypted; + BEGIN + + -- This extracts the data associated with the field from the test eql_v2_encrypted + -- json n: 30 + sv := get_numeric_ste_vec_10()::eql_v2_encrypted; + -- extract the term at $.n returned as eql_v2_encrypted + term := sv->'a7cea93975ed8c01f861ccb6bd082784'; + + -- -- -- -- $.n + PERFORM assert_result( + format('eql_v2_encrypted >= eql_v2_encrypted with ore_cllw_u64_8 index term'), + format('SELECT e FROM encrypted WHERE e->''a7cea93975ed8c01f861ccb6bd082784'' >= %L::eql_v2_encrypted', term)); + + PERFORM assert_count( + format('eql_v2_encrypted >= eql_v2_encrypted with ore index term'), + format('SELECT e FROM encrypted WHERE e->''a7cea93975ed8c01f861ccb6bd082784'' >= %L::eql_v2_encrypted', term), + 3); + + -- -- Check the $.hello path + -- -- Returned encrypted does not have ore_cllw_u64_8 + PERFORM assert_no_result( + format('eql_v2_encrypted >= eql_v2_encrypted with ore index term'), + format('SELECT e FROM encrypted WHERE e->''2517068c0d1f9d4d41d2c666211f785e'' >= %L::eql_v2_encrypted', term)); + + END; +$$ LANGUAGE plpgsql; + + +-- +-- ORE - eql_v2_encrypted >= eql_v2_encrypted +-- +DO $$ +DECLARE + e eql_v2_encrypted; + ore_term jsonb; + BEGIN + + -- Record with a Numeric ORE term of 12 + e := create_encrypted_ore_json(12); + PERFORM seed_encrypted(e); + + PERFORM assert_result( + 'eql_v2_encrypted >= eql_v2_encrypted', + format('SELECT e FROM encrypted WHERE e >= %L', e)); + + PERFORM assert_count( + format('eql_v2_encrypted >= eql_v2_encrypted'), + format('SELECT e FROM encrypted WHERE e >= %L;', e), + 3); + + e := create_encrypted_ore_json(20); + + PERFORM assert_result( + 'eql_v2_encrypted >= eql_v2_encrypted', + format('SELECT e FROM encrypted WHERE e >= %L::eql_v2_encrypted', e)); + + PERFORM assert_count( + format('eql_v2_encrypted >= eql_v2_encrypted'), + format('SELECT e FROM encrypted WHERE e >= %L;', e), + 2); + END; +$$ LANGUAGE plpgsql; + + +-- +-- ORE - eql_v2.gte(a eql_v2_encrypted, b eql_v2_encrypted) +-- +DO $$ +DECLARE + e eql_v2_encrypted; + ore_term jsonb; + BEGIN + -- Reset data + PERFORM seed_encrypted_json(); + + -- Record with a Numeric ORE term of 42 + e := create_encrypted_ore_json(42); + PERFORM seed_encrypted(e); + + PERFORM assert_result( + 'eql_v2.gte(a eql_v2_encrypted, b eql_v2_encrypted)', + format('SELECT e FROM encrypted WHERE eql_v2.gte(e, %L)', e)); + + -- include + PERFORM assert_count( + 'eql_v2.gte(a eql_v2_encrypted, b eql_v2_encrypted)', + format('SELECT e FROM encrypted WHERE eql_v2.gte(e, %L)', e), + 1); + + -- Record with a Numeric ORE term of 30 + e := create_encrypted_ore_json(30); + + PERFORM assert_result( + 'eql_v2.get(a eql_v2_encrypted, b eql_v2_encrypted)', + format('SELECT e FROM encrypted WHERE eql_v2.gte(e, %L)', e)); + + PERFORM assert_count( + 'eql_v2.get(a eql_v2_encrypted, b eql_v2_encrypted)', + format('SELECT e FROM encrypted WHERE eql_v2.gte(e, %L)', e), + 2); + END; +$$ LANGUAGE plpgsql; + + +SELECT drop_table_with_encrypted(); \ No newline at end of file diff --git a/src/operators/>_test.sql b/src/operators/>_test.sql new file mode 100644 index 00000000..67aaa45f --- /dev/null +++ b/src/operators/>_test.sql @@ -0,0 +1,157 @@ +\set ON_ERROR_STOP on + +SELECT create_table_with_encrypted(); +SELECT seed_encrypted_json(); + + +-- ------------------------------------------------------------------------ +-- ------------------------------------------------------------------------ +-- +-- eql_v2_encrypted < eql_v2_encrypted with ore_cllw_u64_8 index +-- +-- Test data is in form '{"hello": "{one | two | three}", "n": {10 | 20 | 30} }' +-- +-- Paths +-- $ -> bca213de9ccce676fa849ff9c4807963 +-- $.hello -> a7cea93975ed8c01f861ccb6bd082784 +-- $.n -> 2517068c0d1f9d4d41d2c666211f785e +-- +-- -- +DO $$ +DECLARE + sv eql_v2_encrypted; + term eql_v2_encrypted; + BEGIN + + -- This extracts the data associated with the field from the test eql_v2_encrypted + -- json n: 30 + sv := get_numeric_ste_vec_20()::eql_v2_encrypted; + -- extract the term at $.n returned as eql_v2_encrypted + term := sv->'2517068c0d1f9d4d41d2c666211f785e'; + + -- -- -- -- $.n + PERFORM assert_result( + format('eql_v2_encrypted > eql_v2_encrypted with ore_cllw_u64_8 index term'), + format('SELECT e FROM encrypted WHERE e->''2517068c0d1f9d4d41d2c666211f785e'' > %L::eql_v2_encrypted', term)); + + PERFORM assert_count( + format('eql_v2_encrypted > eql_v2_encrypted with ore index term'), + format('SELECT e FROM encrypted WHERE e->''2517068c0d1f9d4d41d2c666211f785e'' > %L::eql_v2_encrypted', term), + 1); + + -- -- Check the $.hello path + -- -- Returned encrypted does not have ore_cllw_u64_8 + PERFORM assert_no_result( + format('eql_v2_encrypted > eql_v2_encrypted with ore index term'), + format('SELECT e FROM encrypted WHERE e->''a7cea93975ed8c01f861ccb6bd082784'' > %L::eql_v2_encrypted', term)); + + END; +$$ LANGUAGE plpgsql; + + +-- ------------------------------------------------------------------------ +-- ------------------------------------------------------------------------ +-- +-- eql_v2_encrypted < eql_v2_encrypted with ore_cllw_var_8 index +-- +-- Test data is in form '{"hello": "{one | two | three}", "n": {10 | 20 | 30} }' +-- +-- Paths +-- $ -> bca213de9ccce676fa849ff9c4807963 +-- $.hello -> a7cea93975ed8c01f861ccb6bd082784 +-- $.n -> 2517068c0d1f9d4d41d2c666211f785e +-- +-- -- +DO $$ +DECLARE + sv eql_v2_encrypted; + term eql_v2_encrypted; + BEGIN + + -- This extracts the data associated with the field from the test eql_v2_encrypted + -- json n: 30 + sv := get_numeric_ste_vec_30()::eql_v2_encrypted; + -- extract the term at $.n returned as eql_v2_encrypted + term := sv->'a7cea93975ed8c01f861ccb6bd082784'; + + -- -- -- -- $.n + PERFORM assert_result( + format('eql_v2_encrypted > eql_v2_encrypted with ore_cllw_u64_8 index term'), + format('SELECT e FROM encrypted WHERE e->''a7cea93975ed8c01f861ccb6bd082784'' > %L::eql_v2_encrypted', term)); + + PERFORM assert_count( + format('eql_v2_encrypted > eql_v2_encrypted with ore index term'), + format('SELECT e FROM encrypted WHERE e->''a7cea93975ed8c01f861ccb6bd082784'' > %L::eql_v2_encrypted', term), + 1); + + -- -- Check the $.hello path + -- -- Returned encrypted does not have ore_cllw_var_8 + PERFORM assert_no_result( + format('eql_v2_encrypted > eql_v2_encrypted with ore index term'), + format('SELECT e FROM encrypted WHERE e->''2517068c0d1f9d4d41d2c666211f785e'' > %L::eql_v2_encrypted', term)); + + END; +$$ LANGUAGE plpgsql; + + +-- +-- ORE - eql_v2_encrypted > eql_v2_encrypted +-- +DO $$ +DECLARE + e eql_v2_encrypted; + ore_term eql_v2_encrypted; + BEGIN + SELECT ore.e FROM ore WHERE id = 42 INTO ore_term; + + PERFORM assert_count( + 'eql_v2_encrypted > eql_v2_encrypted', + format('SELECT id FROM ore WHERE e > %L ORDER BY e DESC', ore_term), + 57); + + -- Record with a Numeric ORE term of 1 + e := create_encrypted_ore_json(99); + + PERFORM assert_no_result( + format('eql_v2_encrypted > eql_v2_encrypted'), + format('SELECT e FROM encrypted WHERE e > %L;', e)); + + -- Record with a Numeric ORE term of 42 + e := create_encrypted_ore_json(17); + + PERFORM assert_result( + 'eql_v2_encrypted > eql_v2_encrypted', + format('SELECT e FROM encrypted WHERE e > %L', e)); + + PERFORM assert_count( + 'eql_v2_encrypted > eql_v2_encrypted', + format('SELECT e FROM encrypted WHERE e > %L', e), + 2); + END; +$$ LANGUAGE plpgsql; + + +-- -- +-- -- ORE - eql_v2.lt(a eql_v2_encrypted, b eql_v2_encrypted) +-- -- +DO $$ +DECLARE + e eql_v2_encrypted; + ore_term jsonb; + BEGIN + -- Record with a Numeric ORE term of 42 + e := create_encrypted_ore_json(21); + + PERFORM assert_result( + 'eql_v2.lt(a eql_v2_encrypted, b eql_v2_encrypted)', + format('SELECT e FROM encrypted WHERE eql_v2.gt(e, %L)', e)); + + PERFORM assert_count( + 'eql_v2.lt(a eql_v2_encrypted, b eql_v2_encrypted)', + format('SELECT e FROM encrypted WHERE eql_v2.gt(e, %L)', e), + 1); + END; +$$ LANGUAGE plpgsql; + + +SELECT drop_table_with_encrypted(); \ No newline at end of file diff --git a/src/operators/@>.sql b/src/operators/@>.sql new file mode 100644 index 00000000..0d5e8b6b --- /dev/null +++ b/src/operators/@>.sql @@ -0,0 +1,16 @@ +-- REQUIRE: src/schema.sql +-- REQUIRE: src/encrypted/types.sql +-- REQUIRE: src/ste_vec/functions.sql + + + +CREATE FUNCTION eql_v2."@>"(a eql_v2_encrypted, b eql_v2_encrypted) +RETURNS boolean AS $$ + SELECT eql_v2.ste_vec_contains(a, b) +$$ LANGUAGE SQL; + +CREATE OPERATOR @>( + FUNCTION=eql_v2."@>", + LEFTARG=eql_v2_encrypted, + RIGHTARG=eql_v2_encrypted +); diff --git a/src/operators/@>_test.sql b/src/operators/@>_test.sql new file mode 100644 index 00000000..9baadf81 --- /dev/null +++ b/src/operators/@>_test.sql @@ -0,0 +1,94 @@ +\set ON_ERROR_STOP on + +SELECT create_table_with_encrypted(); +SELECT seed_encrypted_json(); + + +-- ------------------------------------------------------------------------ +-- ------------------------------------------------------------------------ +-- +-- eql_v2_encrypted contains itself +-- +-- +DO $$ + DECLARE + a eql_v2_encrypted; + b eql_v2_encrypted; + BEGIN + + a := get_numeric_ste_vec_10()::eql_v2_encrypted; + b := get_numeric_ste_vec_10()::eql_v2_encrypted; + + ASSERT a @> b; + ASSERT b @> a; + END; +$$ LANGUAGE plpgsql; + + +-- ------------------------------------------------------------------------ +-- ------------------------------------------------------------------------ +-- +-- eql_v2_encrypted contains a term +-- +-- +DO $$ + DECLARE + a eql_v2_encrypted; + b eql_v2_encrypted; + term eql_v2_encrypted; + BEGIN + + a := get_numeric_ste_vec_10()::eql_v2_encrypted; + b := get_numeric_ste_vec_10()::eql_v2_encrypted; + + -- $.n + term := b->'2517068c0d1f9d4d41d2c666211f785e'; + + ASSERT a @> term; + + ASSERT NOT term @> a; + END; +$$ LANGUAGE plpgsql; + + + +-- ------------------------------------------------------------------------ +-- ------------------------------------------------------------------------ +-- +-- ore_cllw_u64_8 equality +-- +-- Test data is in form '{"hello": "{one | two | three}", "n": {10 | 20 | 30} }' +-- +-- Paths +-- $ -> bca213de9ccce676fa849ff9c4807963 +-- $.hello -> a7cea93975ed8c01f861ccb6bd082784 +-- $.n -> 2517068c0d1f9d4d41d2c666211f785e +-- +-- +DO $$ +DECLARE + sv eql_v2_encrypted; + term eql_v2_encrypted; + BEGIN + + -- This extracts the data associated with the field from the test eql_v2_encrypted + sv := get_numeric_ste_vec_10()::eql_v2_encrypted; + -- extract the term at $.n returned as eql_v2_encrypted + term := sv->'a7cea93975ed8c01f861ccb6bd082784'; + + -- -- -- -- $.n + PERFORM assert_result( + format('eql_v2_encrypted = eql_v2_encrypted with ore_cllw_u64_8 index term'), + format('SELECT e FROM encrypted WHERE e @> %L::eql_v2_encrypted', term)); + + PERFORM assert_count( + format('eql_v2_encrypted = eql_v2_encrypted with ore index term'), + format('SELECT e FROM encrypted WHERE e @> %L::eql_v2_encrypted', term), + 1); + + END; +$$ LANGUAGE plpgsql; + + + +SELECT drop_table_with_encrypted(); \ No newline at end of file diff --git a/src/operators/operator_class.sql b/src/operators/operator_class.sql new file mode 100644 index 00000000..be9ea3cc --- /dev/null +++ b/src/operators/operator_class.sql @@ -0,0 +1,38 @@ +-- REQUIRE: src/schema.sql +-- REQUIRE: src/encrypted/types.sql +-- REQUIRE: src/encrypted/functions.sql +-- REQUIRE: src/ore/types.sql +-- REQUIRE: src/ore/functions.sql +-- REQUIRE: src/operators/<.sql +-- REQUIRE: src/operators/<=.sql +-- REQUIRE: src/operators/=.sql +-- REQUIRE: src/operators/>=.sql +-- REQUIRE: src/operators/>.sql + + +CREATE FUNCTION eql_v2.compare(a eql_v2_encrypted, b eql_v2_encrypted) + RETURNS integer + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + DECLARE + a_ore eql_v2.ore_64_8_v2; + b_ore eql_v2.ore_64_8_v2; + BEGIN + + a_ore := eql_v2.ore_64_8_v2(a); + b_ore := eql_v2.ore_64_8_v2(b); + + RETURN eql_v2.compare_ore_array(a_ore.terms, b_ore.terms); + END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR FAMILY eql_v2.encrypted_operator USING btree; + +CREATE OPERATOR CLASS eql_v2.encrypted_operator DEFAULT FOR TYPE eql_v2_encrypted USING btree FAMILY eql_v2.encrypted_operator AS + OPERATOR 1 <, + OPERATOR 2 <=, + OPERATOR 3 =, + OPERATOR 4 >=, + OPERATOR 5 >, + FUNCTION 1 eql_v2.compare(a eql_v2_encrypted, b eql_v2_encrypted); + diff --git a/src/operators/operator_class_test.sql b/src/operators/operator_class_test.sql new file mode 100644 index 00000000..31132f67 --- /dev/null +++ b/src/operators/operator_class_test.sql @@ -0,0 +1,62 @@ +\set ON_ERROR_STOP on + +SELECT create_table_with_encrypted(); +SELECT seed_encrypted_json(); + +-- +-- ORE ORDER BY +-- +DO $$ +DECLARE + ore_term eql_v2_encrypted; + BEGIN + + PERFORM assert_id( + 'ORDER BY eql_v2_encrypted DESC', + 'SELECT id FROM ore ORDER BY e DESC LIMIT 1', + 99); + + PERFORM assert_id( + 'ORDER BY eql_v2_encrypted DESC', + 'SELECT id FROM ore ORDER BY e ASC LIMIT 1', + 1); + + + SELECT e FROM ore WHERE id = 42 INTO ore_term; + + PERFORM assert_id( + 'eql_v2_encrypted < eql_v2_encrypted', + format('SELECT id FROM ore WHERE e < %L ORDER BY e DESC LIMIT 1', ore_term), + 41); + + END; +$$ LANGUAGE plpgsql; + + + +-- +-- ORE GROUP BY +-- +DO $$ + BEGIN + + -- Copy ORE data into encrypted + INSERT INTO encrypted(e) SELECT e FROM ore WHERE ore.id=42; + INSERT INTO encrypted(e) SELECT e FROM ore WHERE ore.id=42; + INSERT INTO encrypted(e) SELECT e FROM ore WHERE ore.id=42; + INSERT INTO encrypted(e) SELECT e FROM ore WHERE ore.id=42; + INSERT INTO encrypted(e) SELECT e FROM ore WHERE ore.id=99; + INSERT INTO encrypted(e) SELECT e FROM ore WHERE ore.id=99; + + -- Should be the rows with value of 42 + PERFORM assert_id( + 'GROUP BY eql_v2_encrypted', + 'SELECT count(id) FROM encrypted GROUP BY e ORDER BY count(id) DESC', + 4); + + + END; +$$ LANGUAGE plpgsql; + + +SELECT drop_table_with_encrypted(); \ No newline at end of file diff --git a/src/operators/order_by.sql b/src/operators/order_by.sql new file mode 100644 index 00000000..80003da4 --- /dev/null +++ b/src/operators/order_by.sql @@ -0,0 +1,71 @@ +-- REQUIRE: src/encrypted/types.sql +-- REQUIRE: src/ore/types.sql +-- REQUIRE: src/ore/functions.sql +-- REQUIRE: src/ore/operators.sql +-- REQUIRE: src/ore_cllw_u64_8/types.sql +-- REQUIRE: src/ore_cllw_u64_8/functions.sql +-- REQUIRE: src/ore_cllw_u64_8/operators.sql + +-- order_by function for ordering when operators are not available. +-- +-- There are multiple index terms that provide equality comparisons +-- - ore_cllw_u64_8 +-- - ore_cllw_var_8 +-- - ore_64_8_v2 +-- +-- We check these index terms in this order and use the first one that exists for both parameters +-- +-- + + +CREATE FUNCTION eql_v2.order_by(a eql_v2_encrypted) + RETURNS eql_v2.ore_64_8_v2 + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + BEGIN + RETURN eql_v2.ore_64_8_v2(a); + EXCEPTION WHEN OTHERS THEN + -- PERFORM eql_v2.log('No ore_64_8_v2 index'); + END; + + RETURN false; + END; +$$ LANGUAGE plpgsql; + +-- TODO: make this work +-- fails with jsonb format issue, which I think is due to the type casting +-- +CREATE FUNCTION eql_v2.order_by_any(a anyelement) + RETURNS anyelement + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + DECLARE + e eql_v2_encrypted; + result ALIAS FOR $0; + BEGIN + + e := a::eql_v2_encrypted; + + BEGIN + result := eql_v2.ore_cllw_u64_8(e); + EXCEPTION WHEN OTHERS THEN + -- PERFORM eql_v2.log('No ore_cllw_u64_8 index'); + END; + + BEGIN + result := eql_v2.ore_cllw_var_8(e); + EXCEPTION WHEN OTHERS THEN + -- PERFORM eql_v2.log('No ore_cllw_u64_8 index'); + END; + + BEGIN + result := eql_v2.ore_64_8_v2(e); + EXCEPTION WHEN OTHERS THEN + -- PERFORM eql_v2.log('No ore_64_8_v2 index'); + END; + + RETURN result; + END; +$$ LANGUAGE plpgsql; + diff --git a/src/operators/order_by_test.sql b/src/operators/order_by_test.sql new file mode 100644 index 00000000..2802d157 --- /dev/null +++ b/src/operators/order_by_test.sql @@ -0,0 +1,28 @@ +\set ON_ERROR_STOP on + +-- +-- ORE - ORDER BY ore_64_8_v2(eql_v2_encrypted) +-- +DO $$ +DECLARE + e eql_v2_encrypted; + ore_term eql_v2_encrypted; + BEGIN + SELECT ore.e FROM ore WHERE id = 42 INTO ore_term; + + PERFORM assert_count( + 'ORDER BY eql_v2.order_by(e) DESC', + format('SELECT id FROM ore WHERE e < %L ORDER BY eql_v2.order_by(e) DESC', ore_term), + 41); + + PERFORM assert_result( + 'ORDER BY eql_v2.order_by(e) DESC returns correct record', + format('SELECT id FROM ore WHERE e < %L ORDER BY eql_v2.order_by(e) DESC LIMIT 1', ore_term), + '41'); + + PERFORM assert_result( + 'ORDER BY eql_v2.order_by(e) ASC', + format('SELECT id FROM ore WHERE e < %L ORDER BY eql_v2.order_by(e) ASC LIMIT 1', ore_term), + '1'); + END; +$$ LANGUAGE plpgsql; \ No newline at end of file diff --git a/src/operators/~~.sql b/src/operators/~~.sql new file mode 100644 index 00000000..08390e59 --- /dev/null +++ b/src/operators/~~.sql @@ -0,0 +1,129 @@ +-- REQUIRE: src/schema.sql +-- REQUIRE: src/encrypted/types.sql +-- REQUIRE: src/match/types.sql +-- REQUIRE: src/match/functions.sql + +-- Operators for match comparisons of eql_v2_encrypted types +-- +-- Support for the following comparisons: +-- +-- eql_v2_encrypted ~~ eql_v2_encrypted +-- eql_v2_encrypted ~~ jsonb +-- eql_v2_encrypted ~~ eql_v2.match_index +-- + + + +CREATE FUNCTION eql_v2.like(a eql_v2_encrypted, b eql_v2_encrypted) +RETURNS boolean AS $$ + SELECT eql_v2.match(a) @> eql_v2.match(b); +$$ LANGUAGE SQL; + + +-- +-- Case sensitivity depends on the index term configuration +-- Function preserves the SQL semantics +-- +CREATE FUNCTION eql_v2.ilike(a eql_v2_encrypted, b eql_v2_encrypted) +RETURNS boolean AS $$ + SELECT eql_v2.match(a) @> eql_v2.match(b); +$$ LANGUAGE SQL; + + + + + +CREATE FUNCTION eql_v2."~~"(a eql_v2_encrypted, b eql_v2_encrypted) + RETURNS boolean +AS $$ + BEGIN + RETURN eql_v2.like(a, b); + END; +$$ LANGUAGE plpgsql; + +CREATE OPERATOR ~~( + FUNCTION=eql_v2."~~", + LEFTARG=eql_v2_encrypted, + RIGHTARG=eql_v2_encrypted, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); + + +CREATE OPERATOR ~~*( + FUNCTION=eql_v2."~~", + LEFTARG=eql_v2_encrypted, + RIGHTARG=eql_v2_encrypted, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); + + + +CREATE FUNCTION eql_v2."~~"(a eql_v2_encrypted, b jsonb) + RETURNS boolean +AS $$ + BEGIN + RETURN eql_v2.like(a, b::eql_v2_encrypted); + END; +$$ LANGUAGE plpgsql; + + +CREATE OPERATOR ~~( + FUNCTION=eql_v2."~~", + LEFTARG=eql_v2_encrypted, + RIGHTARG=jsonb, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); + +CREATE OPERATOR ~~*( + FUNCTION=eql_v2."~~", + LEFTARG=eql_v2_encrypted, + RIGHTARG=jsonb, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); + + + + +CREATE FUNCTION eql_v2."~~"(a jsonb, b eql_v2_encrypted) + RETURNS boolean +AS $$ + BEGIN + RETURN eql_v2.like(a::eql_v2_encrypted, b); + END; +$$ LANGUAGE plpgsql; + + +CREATE OPERATOR ~~( + FUNCTION=eql_v2."~~", + LEFTARG=jsonb, + RIGHTARG=eql_v2_encrypted, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); + +CREATE OPERATOR ~~*( + FUNCTION=eql_v2."~~", + LEFTARG=jsonb, + RIGHTARG=eql_v2_encrypted, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); + + +-- ----------------------------------------------------------------------------- diff --git a/src/operators/~~_test.sql b/src/operators/~~_test.sql new file mode 100644 index 00000000..e3e88662 --- /dev/null +++ b/src/operators/~~_test.sql @@ -0,0 +1,108 @@ +\set ON_ERROR_STOP on + +SELECT create_table_with_encrypted(); +SELECT seed_encrypted_json(); + +-- +-- Match - eql_v2_encrypted ~~ eql_v2_encrypted +-- +DO $$ +DECLARE + e eql_v2_encrypted; + BEGIN + + for i in 1..3 loop + e := create_encrypted_json(i); + + PERFORM assert_result( + format('eql_v2_encrypted ~~ eql_v2_encrypted %s of 3', i), + format('SELECT e FROM encrypted WHERE e ~~ %L;', e)); + + PERFORM assert_result( + format('eql_v2_encrypted LIKE eql_v2_encrypted %s of 3', i), + format('SELECT e FROM encrypted WHERE e LIKE %L;', e)); + + end loop; + + -- Partial match + e := create_encrypted_json('m')::jsonb || '{"m": [10, 11]}'; + + PERFORM assert_result( + 'eql_v2_encrypted ~~ eql_v2_encrypted with partial match', + format('SELECT e FROM encrypted WHERE e ~~ %L;', e)); + + PERFORM assert_result( + 'eql_v2_encrypted LIKE eql_v2_encrypted with partial match', + format('SELECT e FROM encrypted WHERE e LIKE %L;', e)); + + END; +$$ LANGUAGE plpgsql; + + +-- +-- Match - eql_v2_encrypted ~~* eql_v2_encrypted +-- +DO $$ +DECLARE + e eql_v2_encrypted; + BEGIN + + for i in 1..3 loop + e := create_encrypted_json(i, 'm'); + + PERFORM assert_result( + format('eql_v2_encrypted ~~* eql_v2_encrypted %s of 3', i), + format('SELECT e FROM encrypted WHERE e ~~* %L;', e)); + + PERFORM assert_result( + format('eql_v2_encrypted LIKE eql_v2_encrypted %s of 3', i), + format('SELECT e FROM encrypted WHERE e ILIKE %L;', e)); + + end loop; + + -- Partial match + e := create_encrypted_json('m')::jsonb || '{"m": [10, 11]}'; + + PERFORM assert_result( + 'eql_v2_encrypted ~~* eql_v2_encrypted with partial match', + format('SELECT e FROM encrypted WHERE e ~~* %L;', e)); + + PERFORM assert_result( + 'eql_v2_encrypted LIKE eql_v2_encrypted with partial match', + format('SELECT e FROM encrypted WHERE e ILIKE %L;', e)); + + END; +$$ LANGUAGE plpgsql; + + +-- +-- Match - eql_v2.match(eql_v2_encrypted, eql_v2_encrypted) +-- +DO $$ +DECLARE + e eql_v2_encrypted; + BEGIN + + for i in 1..3 loop + e := create_encrypted_json(i, 'm'); + + PERFORM assert_result( + format('eql_v2.like(eql_v2_encrypted, eql_v2_encrypted)', i), + format('SELECT e FROM encrypted WHERE eql_v2.like(e, %L);', e)); + + end loop; + + -- Partial match + e := create_encrypted_json('m')::jsonb || '{"m": [10, 11]}'; + + PERFORM assert_result( + 'eql_v2.like(eql_v2_encrypted, eql_v2_encrypted)', + format('SELECT e FROM encrypted WHERE eql_v2.like(e, %L);', e)); + + END; +$$ LANGUAGE plpgsql; + + + + +SELECT drop_table_with_encrypted(); \ No newline at end of file diff --git a/src/ore/casts.sql b/src/ore/casts.sql new file mode 100644 index 00000000..7a1a9c6f --- /dev/null +++ b/src/ore/casts.sql @@ -0,0 +1,16 @@ +-- REQUIRE: src/schema.sql +-- REQUIRE: src/ore/types.sql + +-- casts text to ore_64_8_v2_term (bytea) + +CREATE FUNCTION eql_v2.text_to_ore_64_8_v2_term(t text) + RETURNS eql_v2.ore_64_8_v2_term + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE +BEGIN ATOMIC + RETURN t::bytea; +END; + +-- cast to cleanup ore_64_8_v2 extraction + +CREATE CAST (text AS eql_v2.ore_64_8_v2_term) + WITH FUNCTION eql_v2.text_to_ore_64_8_v2_term(text) AS IMPLICIT; diff --git a/src/ore/functions.sql b/src/ore/functions.sql new file mode 100644 index 00000000..99f48ceb --- /dev/null +++ b/src/ore/functions.sql @@ -0,0 +1,210 @@ +-- REQUIRE: src/schema.sql +-- REQUIRE: src/encrypted/types.sql +-- REQUIRE: src/encrypted/functions.sql +-- REQUIRE: src/ore/types.sql + + + +-- Casts a jsonb array of hex-encoded strings to the `ore_64_8_v2` composite type. +-- In other words, this function takes the ORE index format sent through in the +-- EQL payload from Proxy and decodes it as the composite type that we use for +-- ORE operations on the Postgres side. +-- CREATE FUNCTION eql_v2.jsonb_array_to_ore_64_8_v2(val jsonb) +-- RETURNS eql_v2.ore_64_8_v2 AS $$ +-- DECLARE +-- terms_arr eql_v2.ore_64_8_v2_term[]; +-- BEGIN +-- IF jsonb_typeof(val) = 'null' THEN +-- RETURN NULL; +-- END IF; + +-- SELECT array_agg(ROW(decode(value::text, 'hex'))::eql_v2.ore_64_8_v2_term) +-- INTO terms_arr +-- FROM jsonb_array_elements_text(val) AS value; + +-- PERFORM eql_v2.log('terms', terms_arr::text); + +-- RETURN ROW(terms_arr)::eql_v2.ore_64_8_v2; +-- END; +-- $$ LANGUAGE plpgsql; + + +CREATE FUNCTION eql_v2.jsonb_array_to_ore_64_8_v2(val jsonb) +RETURNS eql_v2.ore_64_8_v2 AS $$ +DECLARE + terms eql_v2.ore_64_8_v2_term[]; +BEGIN + IF jsonb_typeof(val) = 'null' THEN + RETURN NULL; + END IF; + + SELECT array_agg(ROW(b)::eql_v2.ore_64_8_v2_term) + INTO terms + FROM unnest(eql_v2.jsonb_array_to_bytea_array(val)) AS b; + + RETURN ROW(terms)::eql_v2.ore_64_8_v2; +END; +$$ LANGUAGE plpgsql; + + +-- extracts ore index from jsonb + +CREATE FUNCTION eql_v2.ore_64_8_v2(val jsonb) + RETURNS eql_v2.ore_64_8_v2 + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + IF val ? 'o' THEN + RETURN eql_v2.jsonb_array_to_ore_64_8_v2(val->'o'); + END IF; + RAISE 'Expected an ore index (o) value in json: %', val; + END; +$$ LANGUAGE plpgsql; + + +-- extracts ore index from an encrypted column + +CREATE FUNCTION eql_v2.ore_64_8_v2(val eql_v2_encrypted) + RETURNS eql_v2.ore_64_8_v2 + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN eql_v2.ore_64_8_v2(val.data); + END; +$$ LANGUAGE plpgsql; + + +-- This function uses lexicographic comparison + +CREATE FUNCTION eql_v2.compare_ore_64_8_v2(a eql_v2.ore_64_8_v2, b eql_v2.ore_64_8_v2) +RETURNS integer AS $$ + BEGIN + -- Recursively compare blocks bailing as soon as we can make a decision + RETURN eql_v2.compare_ore_array(a.terms, b.terms); + END +$$ LANGUAGE plpgsql; + + + +CREATE FUNCTION eql_v2.compare_ore_64_8_v2_term(a eql_v2.ore_64_8_v2_term, b eql_v2.ore_64_8_v2_term) + RETURNS integer +AS $$ + DECLARE + eq boolean := true; + unequal_block smallint := 0; + hash_key bytea; + target_block bytea; + + left_block_size CONSTANT smallint := 16; + right_block_size CONSTANT smallint := 32; + right_offset CONSTANT smallint := 136; -- 8 * 17 + + indicator smallint := 0; + BEGIN + IF a IS NULL AND b IS NULL THEN + RETURN 0; + END IF; + + IF a IS NULL THEN + RETURN -1; + END IF; + + IF b IS NULL THEN + RETURN 1; + END IF; + + IF bit_length(a.bytes) != bit_length(b.bytes) THEN + RAISE EXCEPTION 'Ciphertexts are different lengths'; + END IF; + + FOR block IN 0..7 LOOP + -- Compare each PRP (byte from the first 8 bytes) and PRF block (8 byte + -- chunks of the rest of the value). + -- NOTE: + -- * Substr is ordinally indexed (hence 1 and not 0, and 9 and not 8). + -- * We are not worrying about timing attacks here; don't fret about + -- the OR or !=. + IF + substr(a.bytes, 1 + block, 1) != substr(b.bytes, 1 + block, 1) + OR substr(a.bytes, 9 + left_block_size * block, left_block_size) != substr(b.bytes, 9 + left_block_size * BLOCK, left_block_size) + THEN + -- set the first unequal block we find + IF eq THEN + unequal_block := block; + END IF; + eq = false; + END IF; + END LOOP; + + IF eq THEN + RETURN 0::integer; + END IF; + + -- Hash key is the IV from the right CT of b + hash_key := substr(b.bytes, right_offset + 1, 16); + + -- first right block is at right offset + nonce_size (ordinally indexed) + target_block := substr(b.bytes, right_offset + 17 + (unequal_block * right_block_size), right_block_size); + + indicator := ( + get_bit( + encrypt( + substr(a.bytes, 9 + (left_block_size * unequal_block), left_block_size), + hash_key, + 'aes-ecb' + ), + 0 + ) + get_bit(target_block, get_byte(a.bytes, unequal_block))) % 2; + + IF indicator = 1 THEN + RETURN 1::integer; + ELSE + RETURN -1::integer; + END IF; + END; +$$ LANGUAGE plpgsql; + + +-- Compare the "head" of each array and recurse if necessary +-- This function assumes an empty string is "less than" everything else +-- so if a is empty we return -1, if be is empty and a isn't, we return 1. +-- If both are empty we return 0. This cases probably isn't necessary as equality +-- doesn't always make sense but it's here for completeness. +-- If both are non-empty, we compare the first element. If they are equal +-- we need to consider the next block so we recurse, otherwise we return the comparison result. + +CREATE FUNCTION eql_v2.compare_ore_array(a eql_v2.ore_64_8_v2_term[], b eql_v2.ore_64_8_v2_term[]) +RETURNS integer AS $$ + DECLARE + cmp_result integer; + BEGIN + + -- NULLs are NULL + IF a IS NULL OR b IS NULL THEN + RETURN NULL; + END IF; + + -- empty a and b + IF cardinality(a) = 0 AND cardinality(b) = 0 THEN + RETURN 0; + END IF; + + -- empty a and some b + IF (cardinality(a) = 0) AND cardinality(b) > 0 THEN + RETURN -1; + END IF; + + -- some a and empty b + IF cardinality(a) > 0 AND (cardinality(b) = 0) THEN + RETURN 1; + END IF; + + cmp_result := eql_v2.compare_ore_64_8_v2_term(a[1], b[1]); + IF cmp_result = 0 THEN + -- Removes the first element in the array, and calls this fn again to compare the next element/s in the array. + RETURN eql_v2.compare_ore_array(a[2:array_length(a,1)], b[2:array_length(b,1)]); + END IF; + + RETURN cmp_result; + END +$$ LANGUAGE plpgsql; diff --git a/src/ore/functions_test.sql b/src/ore/functions_test.sql new file mode 100644 index 00000000..ef1b37f8 --- /dev/null +++ b/src/ore/functions_test.sql @@ -0,0 +1,41 @@ +\set ON_ERROR_STOP on + +DO $$ + BEGIN + PERFORM assert_result( + 'Extract ore index term from encrypted', + 'SELECT eql_v2.ore_64_8_v2(''{"o": []}''::jsonb)'); + + PERFORM assert_exception( + 'Missing ore index term in encrypted raises exception', + 'SELECT eql_v2.ore_64_8_v2(''{}''::jsonb)'); + + END; +$$ LANGUAGE plpgsql; + +-- +-- ORE - ORDER BY ore_64_8_v2(eql_v2_encrypted) +-- +DO $$ +DECLARE + e eql_v2_encrypted; + ore_term eql_v2_encrypted; + BEGIN + SELECT ore.e FROM ore WHERE id = 42 INTO ore_term; + + PERFORM assert_count( + 'ORDER BY eql_v2.ore_64_8_v2(e) DESC', + format('SELECT id FROM ore WHERE e < %L ORDER BY eql_v2.ore_64_8_v2(e) DESC', ore_term), + 41); + + PERFORM assert_result( + 'ORDER BY eql_v2.ore_64_8_v2(e) DESC returns correct record', + format('SELECT id FROM ore WHERE e < %L ORDER BY eql_v2.ore_64_8_v2(e) DESC LIMIT 1', ore_term), + '41'); + + PERFORM assert_result( + 'ORDER BY eql_v2.ore_64_8_v2(e) ASC', + format('SELECT id FROM ore WHERE e < %L ORDER BY eql_v2.ore_64_8_v2(e) ASC LIMIT 1', ore_term), + '1'); + END; +$$ LANGUAGE plpgsql; \ No newline at end of file diff --git a/src/ore/operator_class.sql b/src/ore/operator_class.sql new file mode 100644 index 00000000..ad610aa3 --- /dev/null +++ b/src/ore/operator_class.sql @@ -0,0 +1,13 @@ +-- REQUIRE: src/schema.sql +-- REQUIRE: src/ore/types.sql + + +CREATE OPERATOR FAMILY eql_v2.ore_64_8_v2_btree_ops USING btree; + +CREATE OPERATOR CLASS eql_v2.ore_64_8_v2_btree_ops DEFAULT FOR TYPE eql_v2.ore_64_8_v2 USING btree FAMILY eql_v2.ore_64_8_v2_btree_ops AS + OPERATOR 1 <, + OPERATOR 2 <=, + OPERATOR 3 =, + OPERATOR 4 >=, + OPERATOR 5 >, + FUNCTION 1 eql_v2.compare_ore_64_8_v2(a eql_v2.ore_64_8_v2, b eql_v2.ore_64_8_v2); diff --git a/src/ore/operators.sql b/src/ore/operators.sql new file mode 100644 index 00000000..2812bcf1 --- /dev/null +++ b/src/ore/operators.sql @@ -0,0 +1,119 @@ +-- REQUIRE: src/schema.sql +-- REQUIRE: src/crypto.sql +-- REQUIRE: src/ore/types.sql +-- REQUIRE: src/ore/functions.sql + + + +CREATE FUNCTION eql_v2.ore_64_8_v2_eq(a eql_v2.ore_64_8_v2, b eql_v2.ore_64_8_v2) +RETURNS boolean AS $$ + SELECT eql_v2.compare_ore_64_8_v2(a, b) = 0 +$$ LANGUAGE SQL; + + + +CREATE FUNCTION eql_v2.ore_64_8_v2_neq(a eql_v2.ore_64_8_v2, b eql_v2.ore_64_8_v2) +RETURNS boolean AS $$ + SELECT eql_v2.compare_ore_64_8_v2(a, b) <> 0 +$$ LANGUAGE SQL; + + + +CREATE FUNCTION eql_v2.ore_64_8_v2_lt(a eql_v2.ore_64_8_v2, b eql_v2.ore_64_8_v2) +RETURNS boolean AS $$ + SELECT eql_v2.compare_ore_64_8_v2(a, b) = -1 +$$ LANGUAGE SQL; + + + +CREATE FUNCTION eql_v2.ore_64_8_v2_lte(a eql_v2.ore_64_8_v2, b eql_v2.ore_64_8_v2) +RETURNS boolean AS $$ + SELECT eql_v2.compare_ore_64_8_v2(a, b) != 1 +$$ LANGUAGE SQL; + + + +CREATE FUNCTION eql_v2.ore_64_8_v2_gt(a eql_v2.ore_64_8_v2, b eql_v2.ore_64_8_v2) +RETURNS boolean AS $$ + SELECT eql_v2.compare_ore_64_8_v2(a, b) = 1 +$$ LANGUAGE SQL; + + + +CREATE FUNCTION eql_v2.ore_64_8_v2_gte(a eql_v2.ore_64_8_v2, b eql_v2.ore_64_8_v2) +RETURNS boolean AS $$ + SELECT eql_v2.compare_ore_64_8_v2(a, b) != -1 +$$ LANGUAGE SQL; + + + +CREATE OPERATOR = ( + FUNCTION=eql_v2.ore_64_8_v2_eq, + LEFTARG=eql_v2.ore_64_8_v2, + RIGHTARG=eql_v2.ore_64_8_v2, + NEGATOR = <>, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); + + + +CREATE OPERATOR <> ( + FUNCTION=eql_v2.ore_64_8_v2_neq, + LEFTARG=eql_v2.ore_64_8_v2, + RIGHTARG=eql_v2.ore_64_8_v2, + NEGATOR = =, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); + + +CREATE OPERATOR > ( + FUNCTION=eql_v2.ore_64_8_v2_gt, + LEFTARG=eql_v2.ore_64_8_v2, + RIGHTARG=eql_v2.ore_64_8_v2, + COMMUTATOR = <, + NEGATOR = <=, + RESTRICT = scalargtsel, + JOIN = scalargtjoinsel +); + + + +CREATE OPERATOR < ( + FUNCTION=eql_v2.ore_64_8_v2_lt, + LEFTARG=eql_v2.ore_64_8_v2, + RIGHTARG=eql_v2.ore_64_8_v2, + COMMUTATOR = >, + NEGATOR = >=, + RESTRICT = scalarltsel, + JOIN = scalarltjoinsel +); + + + +CREATE OPERATOR <= ( + FUNCTION=eql_v2.ore_64_8_v2_lte, + LEFTARG=eql_v2.ore_64_8_v2, + RIGHTARG=eql_v2.ore_64_8_v2, + COMMUTATOR = >=, + NEGATOR = >, + RESTRICT = scalarlesel, + JOIN = scalarlejoinsel +); + + + +CREATE OPERATOR >= ( + FUNCTION=eql_v2.ore_64_8_v2_gte, + LEFTARG=eql_v2.ore_64_8_v2, + RIGHTARG=eql_v2.ore_64_8_v2, + COMMUTATOR = <=, + NEGATOR = <, + RESTRICT = scalarlesel, + JOIN = scalarlejoinsel +); diff --git a/src/ore/types.sql b/src/ore/types.sql new file mode 100644 index 00000000..2d86044a --- /dev/null +++ b/src/ore/types.sql @@ -0,0 +1,11 @@ +-- REQUIRE: src/schema.sql + + +CREATE TYPE eql_v2.ore_64_8_v2_term AS ( + bytes bytea +); + + +CREATE TYPE eql_v2.ore_64_8_v2 AS ( + terms eql_v2.ore_64_8_v2_term[] +); diff --git a/src/ore_cllw_u64_8/functions.sql b/src/ore_cllw_u64_8/functions.sql new file mode 100644 index 00000000..09063587 --- /dev/null +++ b/src/ore_cllw_u64_8/functions.sql @@ -0,0 +1,108 @@ +-- REQUIRE: src/schema.sql +-- REQUIRE: src/common.sql +-- REQUIRE: src/ore_cllw_u64_8/types.sql + + + +-- extracts ste_vec index from a jsonb value + +-- extracts ore_cllw_u64_8 index from a jsonb value + +CREATE FUNCTION eql_v2.ore_cllw_u64_8(val jsonb) + RETURNS eql_v2.ore_cllw_u64_8 + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + + IF NOT (val ? 'ocf') THEN + RAISE 'Expected a ore_cllw_u64_8 index (ocf) value in json: %', val; + END IF; + + IF val->>'ocf' IS NULL THEN + RETURN NULL; + END IF; + + RETURN ROW(decode(val->>'ocf', 'hex')); + END; +$$ LANGUAGE plpgsql; + + +-- extracts ore_cllw_u64_8 index from an eql_v2_encrypted value + +CREATE FUNCTION eql_v2.ore_cllw_u64_8(val eql_v2_encrypted) + RETURNS eql_v2.ore_cllw_u64_8 + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN (SELECT eql_v2.ore_cllw_u64_8(val.data)); + END; +$$ LANGUAGE plpgsql; + + + +-- +-- Compare ore cllw bytes +-- Used by both fixed and variable ore cllw variants +-- + +CREATE FUNCTION eql_v2.compare_ore_cllw(a bytea, b bytea) +RETURNS int AS $$ +DECLARE + len_a INT; + x BYTEA; + y BYTEA; + i INT; + differing RECORD; +BEGIN + len_a := LENGTH(a); + + -- Iterate over each byte and compare them + FOR i IN 1..len_a LOOP + x := SUBSTRING(a FROM i FOR 1); + y := SUBSTRING(b FROM i FOR 1); + + -- Check if there's a difference + IF x != y THEN + differing := (x, y); + EXIT; + END IF; + END LOOP; + + -- If a difference is found, compare the bytes as in Rust logic + IF differing IS NOT NULL THEN + IF (get_byte(y, 0) + 1) % 256 = get_byte(x, 0) THEN + RETURN 1; + ELSE + RETURN -1; + END IF; + ELSE + RETURN 0; + END IF; +END; +$$ LANGUAGE plpgsql; + + + + +CREATE FUNCTION eql_v2.compare_ore_cllw_u64_8(a eql_v2.ore_cllw_u64_8, b eql_v2.ore_cllw_u64_8) +RETURNS int AS $$ +DECLARE + len_a INT; + len_b INT; +BEGIN + IF a IS NULL OR b IS NULL THEN + RETURN NULL; + END IF; + + -- Check if the lengths of the two bytea arguments are the same + len_a := LENGTH(a.bytes); + len_b := LENGTH(b.bytes); + + IF len_a != len_b THEN + RAISE EXCEPTION 'ore_cllw_u64_8 index terms are not the same length'; + END IF; + + RETURN eql_v2.compare_ore_cllw(a.bytes, b.bytes); +END; +$$ LANGUAGE plpgsql; + diff --git a/src/ore_cllw_u64_8/operator_class.sql b/src/ore_cllw_u64_8/operator_class.sql new file mode 100644 index 00000000..015fa31c --- /dev/null +++ b/src/ore_cllw_u64_8/operator_class.sql @@ -0,0 +1,17 @@ +-- REQUIRE: src/schema.sql +-- REQUIRE: src/ore_cllw_u64_8/types.sql +-- REQUIRE: src/ore_cllw_u64_8/functions.sql +-- REQUIRE: src/ore_cllw_u64_8/operators.sql + + + +CREATE OPERATOR FAMILY eql_v2.ore_cllw_u64_8_btree_ops USING btree; + + +CREATE OPERATOR CLASS eql_v2.ore_cllw_u64_8_btree_ops DEFAULT FOR TYPE eql_v2.ore_cllw_u64_8 USING btree FAMILY eql_v2.ore_cllw_u64_8_btree_ops AS + OPERATOR 1 <, + OPERATOR 2 <=, + OPERATOR 3 =, + OPERATOR 4 >=, + OPERATOR 5 >, + FUNCTION 1 eql_v2.compare_ore_cllw_u64_8(a eql_v2.ore_cllw_u64_8, b eql_v2.ore_cllw_u64_8); diff --git a/src/ore_cllw_u64_8/operators.sql b/src/ore_cllw_u64_8/operators.sql new file mode 100644 index 00000000..d11d18f5 --- /dev/null +++ b/src/ore_cllw_u64_8/operators.sql @@ -0,0 +1,129 @@ +-- REQUIRE: src/schema.sql +-- REQUIRE: src/common.sql +-- REQUIRE: src/ore_cllw_u64_8/types.sql +-- REQUIRE: src/ore_cllw_u64_8/functions.sql + + +CREATE FUNCTION eql_v2.ore_cllw_u64_8_eq(a eql_v2.ore_cllw_u64_8, b eql_v2.ore_cllw_u64_8) +RETURNS boolean AS $$ + SELECT eql_v2.bytea_eq(a.bytes, b.bytes) +$$ LANGUAGE SQL; + + + +CREATE FUNCTION eql_v2.ore_cllw_u64_8_neq(a eql_v2.ore_cllw_u64_8, b eql_v2.ore_cllw_u64_8) +RETURNS boolean AS $$ + SELECT NOT eql_v2.bytea_eq(a.bytes, b.bytes) +$$ LANGUAGE SQL; + + + +CREATE FUNCTION eql_v2.ore_cllw_u64_8_lt(a eql_v2.ore_cllw_u64_8, b eql_v2.ore_cllw_u64_8) +RETURNS boolean +AS $$ + BEGIN + RETURN eql_v2.compare_ore_cllw_u64_8(a, b) = -1; + END +$$ LANGUAGE plpgsql; + + + +CREATE FUNCTION eql_v2.ore_cllw_u64_8_lte(a eql_v2.ore_cllw_u64_8, b eql_v2.ore_cllw_u64_8) +RETURNS boolean AS $$ + SELECT eql_v2.compare_ore_cllw_u64_8(a, b) != 1 +$$ LANGUAGE SQL; + + + +CREATE FUNCTION eql_v2.ore_cllw_u64_8_gt(a eql_v2.ore_cllw_u64_8, b eql_v2.ore_cllw_u64_8) +RETURNS boolean AS $$ + SELECT eql_v2.compare_ore_cllw_u64_8(a, b) = 1 +$$ LANGUAGE SQL; + + + +CREATE FUNCTION eql_v2.ore_cllw_u64_8_gte(a eql_v2.ore_cllw_u64_8, b eql_v2.ore_cllw_u64_8) +RETURNS boolean AS $$ + SELECT eql_v2.compare_ore_cllw_u64_8(a, b) != -1 +$$ LANGUAGE SQL; + + + +CREATE OPERATOR = ( + FUNCTION=eql_v2.ore_cllw_u64_8_eq, + LEFTARG=eql_v2.ore_cllw_u64_8, + RIGHTARG=eql_v2.ore_cllw_u64_8, + NEGATOR = <>, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); + + + +CREATE OPERATOR <> ( + FUNCTION=eql_v2.ore_cllw_u64_8_neq, + LEFTARG=eql_v2.ore_cllw_u64_8, + RIGHTARG=eql_v2.ore_cllw_u64_8, + NEGATOR = =, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); + + + +CREATE OPERATOR > ( + FUNCTION=eql_v2.ore_cllw_u64_8_gt, + LEFTARG=eql_v2.ore_cllw_u64_8, + RIGHTARG=eql_v2.ore_cllw_u64_8, + NEGATOR = <=, + RESTRICT = scalarltsel, + JOIN = scalarltjoinsel, + HASHES, + MERGES +); + + + +CREATE OPERATOR < ( + FUNCTION=eql_v2.ore_cllw_u64_8_lt, + LEFTARG=eql_v2.ore_cllw_u64_8, + RIGHTARG=eql_v2.ore_cllw_u64_8, + NEGATOR = >=, + RESTRICT = scalargtsel, + JOIN = scalargtjoinsel, + HASHES, + MERGES +); + + + +CREATE OPERATOR >= ( + FUNCTION=eql_v2.ore_cllw_u64_8_gte, + LEFTARG=eql_v2.ore_cllw_u64_8, + RIGHTARG=eql_v2.ore_cllw_u64_8, + NEGATOR = <, + RESTRICT = scalarltsel, + JOIN = scalarltjoinsel, + HASHES, + MERGES +); + + + +CREATE OPERATOR <= ( + FUNCTION=eql_v2.ore_cllw_u64_8_lte, + LEFTARG=eql_v2.ore_cllw_u64_8, + RIGHTARG=eql_v2.ore_cllw_u64_8, + NEGATOR = >, + RESTRICT = scalargtsel, + JOIN = scalargtjoinsel, + HASHES, + MERGES +); + + + diff --git a/src/ore_cllw_u64_8/operators_test.sql b/src/ore_cllw_u64_8/operators_test.sql new file mode 100644 index 00000000..a00cde67 --- /dev/null +++ b/src/ore_cllw_u64_8/operators_test.sql @@ -0,0 +1,234 @@ +\set ON_ERROR_STOP on + +SELECT create_table_with_encrypted(); +SELECT seed_encrypted_json(); + + +-- ------------------------------------------------------------------------ +-- ------------------------------------------------------------------------ +-- +-- ore_cllw_u64_8 < ore_cllw_u64_8 +-- +-- Test data is '{"hello": "world", "n": 42}' + +-- Paths +-- $ -> bca213de9ccce676fa849ff9c4807963 +-- $.hello -> a7cea93975ed8c01f861ccb6bd082784 +-- $.n -> 2517068c0d1f9d4d41d2c666211f785e +-- +-- +DO $$ +DECLARE + sv eql_v2_encrypted; + term eql_v2_encrypted; + BEGIN + + -- json n: 20 + sv := get_numeric_ste_vec_20()::eql_v2_encrypted; + -- extract the term at $.n + term := sv->'2517068c0d1f9d4d41d2c666211f785e'; + + PERFORM eql_v2.log('term', term::text); + + -- -- -- -- $.n + PERFORM assert_result( + format('eql_v2_encrypted < eql_v2_encrypted with ore_cllw_u64_8 index term'), + format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_u64_8(e->''2517068c0d1f9d4d41d2c666211f785e'') < eql_v2.ore_cllw_u64_8(%L::eql_v2_encrypted)', term)); + + PERFORM assert_count( + format('eql_v2_encrypted < eql_v2_encrypted with ore index term'), + format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_u64_8(e->''2517068c0d1f9d4d41d2c666211f785e'') < eql_v2.ore_cllw_u64_8(%L::eql_v2_encrypted)', term), + 1); + + -- Check the $.hello path + -- Returned encrypted does not have ore_cllw_u64_8 + PERFORM assert_exception( + format('eql_v2_encrypted < eql_v2_encrypted with ore index term'), + format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_u64_8(e->''a7cea93975ed8c01f861ccb6bd082784'') < eql_v2.ore_cllw_u64_8(%L::eql_v2_encrypted)', term)); + + END; +$$ LANGUAGE plpgsql; + + + + +-- ------------------------------------------------------------------------ +-- ------------------------------------------------------------------------ +-- +-- ore_cllw_u64_8 <= ore_cllw_u64_8 +-- +-- Test data is '{"hello": "world", "n": 42}' + +-- Paths +-- $ -> bca213de9ccce676fa849ff9c4807963 +-- $.hello -> a7cea93975ed8c01f861ccb6bd082784 +-- $.n -> 2517068c0d1f9d4d41d2c666211f785e +-- +-- +DO $$ +DECLARE + sv eql_v2_encrypted; + term eql_v2_encrypted; + BEGIN + + -- json n: 20 + sv := get_numeric_ste_vec_20()::eql_v2_encrypted; + + -- extract the term at $.n + term := sv->'2517068c0d1f9d4d41d2c666211f785e'; + + -- -- -- -- $.n + PERFORM assert_result( + format('eql_v2_encrypted <= eql_v2_encrypted with ore_cllw_u64_8 index term'), + format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_u64_8(e->''2517068c0d1f9d4d41d2c666211f785e'') <= eql_v2.ore_cllw_u64_8(%L::eql_v2_encrypted)', term)); + + PERFORM assert_count( + format('eql_v2_encrypted <= eql_v2_encrypted with ore index term'), + format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_u64_8(e->''2517068c0d1f9d4d41d2c666211f785e'') <= eql_v2.ore_cllw_u64_8(%L::eql_v2_encrypted)', term), + 2); + + -- Check the $.hello path + -- Returned encrypted does not have ore_cllw_u64_8 + PERFORM assert_exception( + format('eql_v2_encrypted <= eql_v2_encrypted with ore index term'), + format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_u64_8(e->''a7cea93975ed8c01f861ccb6bd082784'') <= eql_v2.ore_cllw_u64_8(%L::eql_v2_encrypted)', term)); + + END; +$$ LANGUAGE plpgsql; + + + +-- ------------------------------------------------------------------------ +-- ------------------------------------------------------------------------ +-- +-- ore_cllw_u64_8 >= ore_cllw_u64_8 +-- +-- Test data is '{"hello": "world", "n": 42}' + +-- Paths +-- $ -> bca213de9ccce676fa849ff9c4807963 +-- $.hello -> a7cea93975ed8c01f861ccb6bd082784 +-- $.n -> 2517068c0d1f9d4d41d2c666211f785e +-- +-- +DO $$ +DECLARE + sv eql_v2_encrypted; + term eql_v2_encrypted; + BEGIN + + -- json n: 30 + sv := get_numeric_ste_vec_30()::eql_v2_encrypted; + + -- extract the term at $.n + term := sv->'2517068c0d1f9d4d41d2c666211f785e'; + + -- -- -- -- $.n + PERFORM assert_result( + format('eql_v2_encrypted >= eql_v2_encrypted with ore_cllw_u64_8 index term'), + format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_u64_8(e->''2517068c0d1f9d4d41d2c666211f785e'') >= eql_v2.ore_cllw_u64_8(%L::eql_v2_encrypted)', term)); + + PERFORM assert_count( + format('eql_v2_encrypted >= eql_v2_encrypted with ore index term'), + format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_u64_8(e->''2517068c0d1f9d4d41d2c666211f785e'') >= eql_v2.ore_cllw_u64_8(%L::eql_v2_encrypted)', term), + 1); + + -- Check the $ path + -- Returned encrypted does not have ore_cllw_u64_8 + PERFORM assert_exception( + format('eql_v2_encrypted >= eql_v2_encrypted with ore index term'), + format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_u64_8(e->''bca213de9ccce676fa849ff9c4807963'') >= eql_v2.ore_cllw_u64_8(%L::eql_v2_encrypted)', term)); + + END; +$$ LANGUAGE plpgsql; + + + +-- ------------------------------------------------------------------------ +-- ------------------------------------------------------------------------ +-- +-- ore_cllw_u64_8 > ore_cllw_u64_8 +-- +-- Test data is '{"hello": "world", "n": 42}' + +-- Paths +-- $ -> bca213de9ccce676fa849ff9c4807963 +-- $.hello -> a7cea93975ed8c01f861ccb6bd082784 +-- $.n -> 2517068c0d1f9d4d41d2c666211f785e +-- +-- +DO $$ +DECLARE + sv eql_v2_encrypted; + term eql_v2_encrypted; + BEGIN + + -- json n: 20 + sv := get_numeric_ste_vec_20()::eql_v2_encrypted; + + -- extract the term at $.n + term := sv->'2517068c0d1f9d4d41d2c666211f785e'; + + -- -- -- -- $.n + PERFORM assert_result( + format('eql_v2_encrypted > eql_v2_encrypted with ore_cllw_u64_8 index term'), + format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_u64_8(e->''2517068c0d1f9d4d41d2c666211f785e'') > eql_v2.ore_cllw_u64_8(%L::eql_v2_encrypted)', term)); + + PERFORM assert_count( + format('eql_v2_encrypted > eql_v2_encrypted with ore index term'), + format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_u64_8(e->''2517068c0d1f9d4d41d2c666211f785e'') > eql_v2.ore_cllw_u64_8(%L::eql_v2_encrypted)', term), + 1); + + -- Check the $ path + -- Returned encrypted does not have ore_cllw_u64_8 + PERFORM assert_exception( + format('eql_v2_encrypted > eql_v2_encrypted with ore index term'), + format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_u64_8(e->''bca213de9ccce676fa849ff9c4807963'') > eql_v2.ore_cllw_u64_8(%L::eql_v2_encrypted)', term)); + + END; +$$ LANGUAGE plpgsql; + + + + +-- -- ------------------------------------------------------------------------ +-- -- ------------------------------------------------------------------------ +-- -- +-- -- ore_cllw_u64_8 = ore_cllw_u64_8 +-- -- +-- -- Test data is '{"hello": "world", "n": 42}' + +-- -- Paths +-- -- $ -> bca213de9ccce676fa849ff9c4807963 +-- -- $.hello -> a7cea93975ed8c01f861ccb6bd082784 +-- -- $.n -> 2517068c0d1f9d4d41d2c666211f785e +-- -- +-- -- +DO $$ +DECLARE + sv eql_v2_encrypted; + term eql_v2_encrypted; + BEGIN + + -- json n: 20 + sv := get_numeric_ste_vec_10()::eql_v2_encrypted; + -- extract the term at $.n + term := sv->'2517068c0d1f9d4d41d2c666211f785e'; + + PERFORM assert_result( + format('eql_v2_encrypted = eql_v2_encrypted with ore_cllw_u64_8 index term'), + format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_u64_8(e->''2517068c0d1f9d4d41d2c666211f785e'') = eql_v2.ore_cllw_u64_8(%L::eql_v2_encrypted)', term)); + + PERFORM assert_count( + format('eql_v2_encrypted = eql_v2_encrypted with ore_cllw_u64_8 index term'), + format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_u64_8(e->''2517068c0d1f9d4d41d2c666211f785e'') = eql_v2.ore_cllw_u64_8(%L::eql_v2_encrypted)', term), + 1); + + -- Check the $.n path + -- Returned encrypted does not have ore_cllw_u64_8 and raises exception + PERFORM assert_exception( + format('eql_v2_encrypted = eql_v2_encrypted with ore_cllw_u64_8 index term'), + format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_u64_8(e->''a7cea93975ed8c01f861ccb6bd082784'') = eql_v2.ore_cllw_u64_8(%L::eql_v2_encrypted)', term)); + + END; +$$ LANGUAGE plpgsql; diff --git a/src/ore_cllw_u64_8/types.sql b/src/ore_cllw_u64_8/types.sql new file mode 100644 index 00000000..bb4e6b41 --- /dev/null +++ b/src/ore_cllw_u64_8/types.sql @@ -0,0 +1,7 @@ +-- REQUIRE: src/schema.sql + +-- Represents a ciphertext encrypted with the CLLW ORE scheme for a fixed output size +-- Each output block is 8-bits +CREATE TYPE eql_v2.ore_cllw_u64_8 AS ( + bytes bytea +); diff --git a/src/ore_cllw_var_8/functions.sql b/src/ore_cllw_var_8/functions.sql new file mode 100644 index 00000000..57063087 --- /dev/null +++ b/src/ore_cllw_var_8/functions.sql @@ -0,0 +1,97 @@ +-- REQUIRE: src/schema.sql +-- REQUIRE: src/common.sql +-- REQUIRE: src/ore_cllw_var_8/types.sql +-- REQUIRE: src/ore_cllw_u64_8/functions.sql + + + +-- extracts ore_cllw_var_8 index from a jsonb value + +CREATE FUNCTION eql_v2.ore_cllw_var_8(val jsonb) + RETURNS eql_v2.ore_cllw_var_8 + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + + IF NOT (val ? 'ocv') THEN + RAISE 'Expected a ore_cllw_var_8 index (ocv) value in json: %', val; + END IF; + + IF val->>'ocv' IS NULL THEN + RETURN NULL; + END IF; + + RETURN ROW(decode(val->>'ocv', 'hex')); + END; +$$ LANGUAGE plpgsql; + + +-- extracts ore_cllw_var_8 index from an eql_v2_encrypted value + +CREATE FUNCTION eql_v2.ore_cllw_var_8(val eql_v2_encrypted) + RETURNS eql_v2.ore_cllw_var_8 + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN (SELECT eql_v2.ore_cllw_var_8(val.data)); + END; +$$ LANGUAGE plpgsql; + + + +CREATE FUNCTION eql_v2.compare_ore_cllw_var_8(a eql_v2.ore_cllw_var_8, b eql_v2.ore_cllw_var_8) +RETURNS int AS $$ +DECLARE + len_a INT; + len_b INT; + -- length of the common part of the two bytea values + common_len INT; + cmp_result INT; +BEGIN + IF a IS NULL OR b IS NULL THEN + RETURN NULL; + END IF; + + -- Get the lengths of both bytea inputs + len_a := LENGTH(a.bytes); + len_b := LENGTH(b.bytes); + + -- Handle empty cases + IF len_a = 0 AND len_b = 0 THEN + RETURN 0; + ELSIF len_a = 0 THEN + RETURN -1; + ELSIF len_b = 0 THEN + RETURN 1; + END IF; + + -- Find the length of the shorter bytea + IF len_a < len_b THEN + common_len := len_a; + ELSE + common_len := len_b; + END IF; + + -- Use the compare_bytea function to compare byte by byte + cmp_result := eql_v2.compare_ore_cllw( + SUBSTRING(a.bytes FROM 1 FOR common_len), + SUBSTRING(b.bytes FROM 1 FOR common_len) + ); + + -- If the comparison returns 'less' or 'greater', return that result + IF cmp_result = -1 THEN + RETURN -1; + ELSIF cmp_result = 1 THEN + RETURN 1; + END IF; + + -- If the bytea comparison is 'equal', compare lengths + IF len_a < len_b THEN + RETURN -1; + ELSIF len_a > len_b THEN + RETURN 1; + ELSE + RETURN 0; + END IF; +END; +$$ LANGUAGE plpgsql; diff --git a/src/ore_cllw_var_8/functions_test.sql b/src/ore_cllw_var_8/functions_test.sql new file mode 100644 index 00000000..e69de29b diff --git a/src/ore_cllw_var_8/operator_class.sql b/src/ore_cllw_var_8/operator_class.sql new file mode 100644 index 00000000..c6db0f11 --- /dev/null +++ b/src/ore_cllw_var_8/operator_class.sql @@ -0,0 +1,17 @@ +-- REQUIRE: src/schema.sql +-- REQUIRE: src/ore_cllw_var_8/types.sql +-- REQUIRE: src/ore_cllw_var_8/functions.sql +-- REQUIRE: src/ore_cllw_var_8/operators.sql + + + +CREATE OPERATOR FAMILY eql_v2.ore_cllw_var_8_variable_btree_ops USING btree; + + +CREATE OPERATOR CLASS eql_v2.ore_cllw_var_8_variable_btree_ops DEFAULT FOR TYPE eql_v2.ore_cllw_var_8 USING btree FAMILY eql_v2.ore_cllw_var_8_variable_btree_ops AS + OPERATOR 1 <, + OPERATOR 2 <=, + OPERATOR 3 =, + OPERATOR 4 >=, + OPERATOR 5 >, + FUNCTION 1 eql_v2.compare_ore_cllw_var_8(a eql_v2.ore_cllw_var_8, b eql_v2.ore_cllw_var_8); diff --git a/src/ore_cllw_var_8/operators.sql b/src/ore_cllw_var_8/operators.sql new file mode 100644 index 00000000..697fdc28 --- /dev/null +++ b/src/ore_cllw_var_8/operators.sql @@ -0,0 +1,115 @@ +-- REQUIRE: src/ore_cllw_var_8/types.sql +-- REQUIRE: src/ore_cllw_var_8/functions.sql + + + +-- Lexical comparison operators + + +CREATE OR REPLACE FUNCTION eql_v2.ore_cllw_var_8_eq(a eql_v2.ore_cllw_var_8, b eql_v2.ore_cllw_var_8) RETURNS boolean AS $$ + SELECT eql_v2.bytea_eq(a.bytes, b.bytes) +$$ LANGUAGE SQL; + + +CREATE OR REPLACE FUNCTION eql_v2.ore_cllw_var_8_neq(a eql_v2.ore_cllw_var_8, b eql_v2.ore_cllw_var_8) RETURNS boolean AS $$ + SELECT NOT eql_v2.bytea_eq(a.bytes, b.bytes) +$$ LANGUAGE SQL; + + +CREATE OR REPLACE FUNCTION eql_v2.ore_cllw_var_8_lt(a eql_v2.ore_cllw_var_8, b eql_v2.ore_cllw_var_8) +RETURNS boolean +-- AS $$ +-- SELECT eql_v2.compare_ore_cllw_var_8(a, b) = -1 +-- $$ LANGUAGE SQL; +AS $$ + BEGIN + RETURN eql_v2.compare_ore_cllw_var_8(a, b) = -1; + END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION eql_v2.ore_cllw_var_8_lte(a eql_v2.ore_cllw_var_8, b eql_v2.ore_cllw_var_8) RETURNS boolean AS $$ + SELECT eql_v2.compare_ore_cllw_var_8(a, b) != 1 +$$ LANGUAGE SQL; + + +CREATE OR REPLACE FUNCTION eql_v2.ore_cllw_var_8_gt(a eql_v2.ore_cllw_var_8, b eql_v2.ore_cllw_var_8) RETURNS boolean AS $$ + SELECT eql_v2.compare_ore_cllw_var_8(a, b) = 1 +$$ LANGUAGE SQL; + + +CREATE OR REPLACE FUNCTION eql_v2.ore_cllw_var_8_gte(a eql_v2.ore_cllw_var_8, b eql_v2.ore_cllw_var_8) RETURNS boolean AS $$ + SELECT eql_v2.compare_ore_cllw_var_8(a, b) != -1 +$$ LANGUAGE SQL; + + +CREATE OPERATOR = ( + FUNCTION=eql_v2.ore_cllw_var_8_eq, + LEFTARG=eql_v2.ore_cllw_var_8, + RIGHTARG=eql_v2.ore_cllw_var_8, + NEGATOR = <>, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); + + +CREATE OPERATOR <> ( + FUNCTION=eql_v2.ore_cllw_var_8_neq, + LEFTARG=eql_v2.ore_cllw_var_8, + RIGHTARG=eql_v2.ore_cllw_var_8, + NEGATOR = =, + RESTRICT = eqsel, + JOIN = eqjoinsel, + HASHES, + MERGES +); + + +CREATE OPERATOR > ( + FUNCTION=eql_v2.ore_cllw_var_8_gt, + LEFTARG=eql_v2.ore_cllw_var_8, + RIGHTARG=eql_v2.ore_cllw_var_8, + NEGATOR = <=, + RESTRICT = scalarltsel, + JOIN = scalarltjoinsel, + HASHES, + MERGES +); + + +CREATE OPERATOR < ( + FUNCTION=eql_v2.ore_cllw_var_8_lt, + LEFTARG=eql_v2.ore_cllw_var_8, + RIGHTARG=eql_v2.ore_cllw_var_8, + NEGATOR = >=, + RESTRICT = scalargtsel, + JOIN = scalargtjoinsel, + HASHES, + MERGES +); + + +CREATE OPERATOR >= ( + FUNCTION=eql_v2.ore_cllw_var_8_gte, + LEFTARG=eql_v2.ore_cllw_var_8, + RIGHTARG=eql_v2.ore_cllw_var_8, + NEGATOR = <, + RESTRICT = scalarltsel, + JOIN = scalarltjoinsel, + HASHES, + MERGES +); + + +CREATE OPERATOR <= ( + FUNCTION=eql_v2.ore_cllw_var_8_lte, + LEFTARG=eql_v2.ore_cllw_var_8, + RIGHTARG=eql_v2.ore_cllw_var_8, + NEGATOR = >, + RESTRICT = scalargtsel, + JOIN = scalargtjoinsel, + HASHES, + MERGES +); diff --git a/src/ore_cllw_var_8/operators_test.sql b/src/ore_cllw_var_8/operators_test.sql new file mode 100644 index 00000000..197c9cd9 --- /dev/null +++ b/src/ore_cllw_var_8/operators_test.sql @@ -0,0 +1,234 @@ +\set ON_ERROR_STOP on + +SELECT create_table_with_encrypted(); +SELECT seed_encrypted_json(); + + +-- ------------------------------------------------------------------------ +-- ------------------------------------------------------------------------ +-- +-- ore_cllw_var_8 < ore_cllw_var_8 +-- +-- Test data is '{"hello": "world", "n": 42}' + +-- Paths +-- $ -> bca213de9ccce676fa849ff9c4807963 +-- $.hello -> a7cea93975ed8c01f861ccb6bd082784 +-- $.n -> 2517068c0d1f9d4d41d2c666211f785e +-- +-- +DO $$ +DECLARE + sv eql_v2_encrypted; + term eql_v2_encrypted; + BEGIN + + -- json n: 20 + sv := get_numeric_ste_vec_20()::eql_v2_encrypted; + + -- extract the term at $.n + term := sv->'a7cea93975ed8c01f861ccb6bd082784'; + + -- -- -- -- $.n + PERFORM assert_result( + format('eql_v2_encrypted < eql_v2_encrypted with ore_cllw_var_8 index term'), + format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_var_8(e->''a7cea93975ed8c01f861ccb6bd082784'') < eql_v2.ore_cllw_var_8(%L::eql_v2_encrypted)', term)); + + -- other values are "one" and "three" + PERFORM assert_count( + format('eql_v2_encrypted < eql_v2_encrypted with ore_cllw_var_8 index term'), + format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_var_8(e->''a7cea93975ed8c01f861ccb6bd082784'') < eql_v2.ore_cllw_var_8(%L::eql_v2_encrypted)', term), + 2); + + -- Check the $.n path + -- Returned encrypted does not have ore_cllw_var_8 and raises an exception + PERFORM assert_exception( + format('eql_v2_encrypted < eql_v2_encrypted with ore_cllw_var_8 index term'), + format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_var_8(e->''2517068c0d1f9d4d41d2c666211f785e'') < eql_v2.ore_cllw_var_8(%L::eql_v2_encrypted)', term)); + + END; +$$ LANGUAGE plpgsql; + + + + +-- -- ------------------------------------------------------------------------ +-- -- ------------------------------------------------------------------------ +-- -- +-- -- ore_cllw_var_8 <= ore_cllw_var_8 +-- -- +-- -- Test data is '{"hello": "world", "n": 42}' + +-- -- Paths +-- -- $ -> bca213de9ccce676fa849ff9c4807963 +-- -- $.hello -> a7cea93975ed8c01f861ccb6bd082784 +-- -- $.n -> 2517068c0d1f9d4d41d2c666211f785e +-- -- +-- -- +DO $$ +DECLARE + sv eql_v2_encrypted; + term eql_v2_encrypted; + BEGIN + + -- json n: 20 + sv := get_numeric_ste_vec_20()::eql_v2_encrypted; + + -- extract the term at $.n + term := sv->'a7cea93975ed8c01f861ccb6bd082784'; + + -- -- -- -- $.n + PERFORM assert_result( + format('eql_v2_encrypted <= eql_v2_encrypted with ore_cllw_var_8 index term'), + format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_var_8(e->''a7cea93975ed8c01f861ccb6bd082784'') <= eql_v2.ore_cllw_var_8(%L::eql_v2_encrypted)', term)); + + PERFORM assert_count( + format('eql_v2_encrypted <= eql_v2_encrypted with ore_cllw_var_8 index term'), + format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_var_8(e->''a7cea93975ed8c01f861ccb6bd082784'') <= eql_v2.ore_cllw_var_8(%L::eql_v2_encrypted)', term), + 3); + + -- Check the $.hello path + -- Returned encrypted does not have ore_cllw_var_8 + PERFORM assert_exception( + format('eql_v2_encrypted <= eql_v2_encrypted with ore_cllw_var_8 index term'), + format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_var_8(e->''2517068c0d1f9d4d41d2c666211f785e'') <= eql_v2.ore_cllw_var_8(%L::eql_v2_encrypted)', term)); + + END; +$$ LANGUAGE plpgsql; + + + +-- -- ------------------------------------------------------------------------ +-- -- ------------------------------------------------------------------------ +-- -- +-- -- ore_cllw_var_8 >= ore_cllw_var_8 +-- -- +-- -- Test data is '{"hello": "world", "n": 42}' + +-- -- Paths +-- -- $ -> bca213de9ccce676fa849ff9c4807963 +-- -- $.hello -> a7cea93975ed8c01f861ccb6bd082784 +-- -- $.n -> 2517068c0d1f9d4d41d2c666211f785e +-- -- +-- -- +DO $$ +DECLARE + sv eql_v2_encrypted; + term eql_v2_encrypted; + BEGIN + + -- json n: 30 + sv := get_numeric_ste_vec_30()::eql_v2_encrypted; + + -- extract the term at $.n + term := sv->'a7cea93975ed8c01f861ccb6bd082784'; + + -- -- -- -- $.n + PERFORM assert_result( + format('eql_v2_encrypted >= eql_v2_encrypted with ore_cllw_var_8 index term'), + format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_var_8(e->''a7cea93975ed8c01f861ccb6bd082784'') >= eql_v2.ore_cllw_var_8(%L::eql_v2_encrypted)', term)); + + PERFORM assert_count( + format('eql_v2_encrypted >= eql_v2_encrypted with ore_cllw_var_8 index term'), + format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_var_8(e->''a7cea93975ed8c01f861ccb6bd082784'') >= eql_v2.ore_cllw_var_8(%L::eql_v2_encrypted)', term), + 2); + + -- Check the $ path + -- Returned encrypted does not have ore_cllw_var_8 + PERFORM assert_exception( + format('eql_v2_encrypted >= eql_v2_encrypted with ore_cllw_var_8 index term'), + format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_var_8(e->''bca213de9ccce676fa849ff9c4807963'') >= eql_v2.ore_cllw_var_8(%L::eql_v2_encrypted)', term)); + + END; +$$ LANGUAGE plpgsql; + + + +-- -- ------------------------------------------------------------------------ +-- -- ------------------------------------------------------------------------ +-- -- +-- -- ore_cllw_var_8 > ore_cllw_var_8 +-- -- +-- -- Test data is '{"hello": "world", "n": 42}' + +-- -- Paths +-- -- $ -> bca213de9ccce676fa849ff9c4807963 +-- -- $.hello -> a7cea93975ed8c01f861ccb6bd082784 +-- -- $.n -> 2517068c0d1f9d4d41d2c666211f785e +-- -- +-- -- +DO $$ +DECLARE + sv eql_v2_encrypted; + term eql_v2_encrypted; + BEGIN + + -- json hello: one + sv := get_numeric_ste_vec_10()::eql_v2_encrypted; + + -- extract the term at $.n + term := sv->'a7cea93975ed8c01f861ccb6bd082784'; + + -- -- -- -- $.n + PERFORM assert_result( + format('eql_v2_encrypted > eql_v2_encrypted with ore_cllw_var_8 index term'), + format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_var_8(e->''a7cea93975ed8c01f861ccb6bd082784'') > eql_v2.ore_cllw_var_8(%L::eql_v2_encrypted)', term)); + + PERFORM assert_count( + format('eql_v2_encrypted > eql_v2_encrypted with ore_cllw_var_8 index term'), + format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_var_8(e->''a7cea93975ed8c01f861ccb6bd082784'') > eql_v2.ore_cllw_var_8(%L::eql_v2_encrypted)', term), + 2); + + -- Check the $ path + -- Returned encrypted does not have ore_cllw_var_8 + PERFORM assert_exception( + format('eql_v2_encrypted > eql_v2_encrypted with ore_cllw_var_8 index term'), + format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_var_8(e->''bca213de9ccce676fa849ff9c4807963'') > eql_v2.ore_cllw_var_8(%L::eql_v2_encrypted)', term)); + + END; +$$ LANGUAGE plpgsql; + + + + +-- -- ------------------------------------------------------------------------ +-- -- ------------------------------------------------------------------------ +-- -- +-- -- ore_cllw_var_8 = ore_cllw_var_8 +-- -- +-- -- Test data is '{"hello": "world", "n": 42}' + +-- -- Paths +-- -- $ -> bca213de9ccce676fa849ff9c4807963 +-- -- $.hello -> a7cea93975ed8c01f861ccb6bd082784 +-- -- $.n -> 2517068c0d1f9d4d41d2c666211f785e +-- -- +-- -- +DO $$ +DECLARE + sv eql_v2_encrypted; + term eql_v2_encrypted; + BEGIN + + -- json n: 20 + sv := get_numeric_ste_vec_10()::eql_v2_encrypted; + -- extract the term at $.n + term := sv->'a7cea93975ed8c01f861ccb6bd082784'; + + PERFORM assert_result( + format('eql_v2_encrypted = eql_v2_encrypted with ore_cllw_var_8 index term'), + format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_var_8(e->''a7cea93975ed8c01f861ccb6bd082784'') = eql_v2.ore_cllw_var_8(%L::eql_v2_encrypted)', term)); + + PERFORM assert_count( + format('eql_v2_encrypted = eql_v2_encrypted with ore_cllw_var_8 index term'), + format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_var_8(e->''a7cea93975ed8c01f861ccb6bd082784'') = eql_v2.ore_cllw_var_8(%L::eql_v2_encrypted)', term), + 1); + + -- Check the $.n path + -- Returned encrypted does not have ore_cllw_var_8 and raises exception + PERFORM assert_exception( + format('eql_v2_encrypted = eql_v2_encrypted with ore_cllw_var_8 index term'), + format('SELECT e FROM encrypted WHERE eql_v2.ore_cllw_var_8(e->''2517068c0d1f9d4d41d2c666211f785e'') = eql_v2.ore_cllw_var_8(%L::eql_v2_encrypted)', term)); + + END; +$$ LANGUAGE plpgsql; diff --git a/src/ore_cllw_var_8/types.sql b/src/ore_cllw_var_8/types.sql new file mode 100644 index 00000000..4714cc67 --- /dev/null +++ b/src/ore_cllw_var_8/types.sql @@ -0,0 +1,7 @@ +-- REQUIRE: src/schema.sql + +-- Represents a ciphertext encrypted with the CLLW ORE scheme for a variable output size +-- Each output block is 8-bits +CREATE TYPE eql_v2.ore_cllw_var_8 AS ( + bytes bytea +); diff --git a/src/schema.sql b/src/schema.sql new file mode 100644 index 00000000..3fc12843 --- /dev/null +++ b/src/schema.sql @@ -0,0 +1 @@ +CREATE SCHEMA IF NOT EXISTS eql_v2; diff --git a/src/ste_vec/functions.sql b/src/ste_vec/functions.sql new file mode 100644 index 00000000..4d930b94 --- /dev/null +++ b/src/ste_vec/functions.sql @@ -0,0 +1,154 @@ +-- REQUIRE: src/schema.sql +-- REQUIRE: src/encrypted/types.sql + + +-- +CREATE FUNCTION eql_v2.ste_vec(val jsonb) + RETURNS eql_v2_encrypted[] + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + DECLARE + sv jsonb; + ary eql_v2_encrypted[]; + BEGIN + + IF val ? 'sv' THEN + sv := val->'sv'; + ELSE + sv := jsonb_build_array(val); + END IF; + + SELECT array_agg(elem::eql_v2_encrypted) + INTO ary + FROM jsonb_array_elements(sv) AS elem; + + RETURN ary; + END; +$$ LANGUAGE plpgsql; + + +-- extracts ste_vec index from an eql_v2_encrypted value + +CREATE FUNCTION eql_v2.ste_vec(val eql_v2_encrypted) + RETURNS eql_v2_encrypted[] + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN (SELECT eql_v2.ste_vec(val.data)); + END; +$$ LANGUAGE plpgsql; + + + +CREATE FUNCTION eql_v2.selector(val jsonb) + RETURNS text + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + IF val ? 's' THEN + RETURN val->>'s'; + END IF; + RAISE 'Expected a selector index (s) value in json: %', val; + END; +$$ LANGUAGE plpgsql; + + +-- extracts ste_vec index from an eql_v2_encrypted value + +CREATE FUNCTION eql_v2.selector(val eql_v2_encrypted) + RETURNS text + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN (SELECT eql_v2.selector(val.data)); + END; +$$ LANGUAGE plpgsql; + + + +CREATE FUNCTION eql_v2.is_ste_vec_array(val jsonb) + RETURNS boolean + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + IF val ? 'a' THEN + RETURN (val->>'a')::boolean; + END IF; + + RETURN false; + END; +$$ LANGUAGE plpgsql; + + +-- extracts ste_vec index from an eql_v2_encrypted value + +CREATE FUNCTION eql_v2.is_ste_vec_array(val eql_v2_encrypted) + RETURNS boolean + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN (SELECT eql_v2.is_ste_vec_array(val.data)); + END; +$$ LANGUAGE plpgsql; + + + +-- Returns true if b is contained in any element of a +CREATE FUNCTION eql_v2.ste_vec_contains(a eql_v2_encrypted[], b eql_v2_encrypted) + RETURNS boolean + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + DECLARE + result boolean; + _a eql_v2_encrypted; + BEGIN + + result := false; + + FOR idx IN 1..array_length(a, 1) LOOP + _a := a[idx]; + result := result OR (eql_v2.selector(_a) = eql_v2.selector(b) AND _a = b); + END LOOP; + + RETURN result; + END; +$$ LANGUAGE plpgsql; + + +-- Returns truy if a contains b +-- All values of b must be in a +CREATE FUNCTION eql_v2.ste_vec_contains(a eql_v2_encrypted, b eql_v2_encrypted) + RETURNS boolean + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + DECLARE + result boolean; + sv_a eql_v2_encrypted[]; + sv_b eql_v2_encrypted[]; + _b eql_v2_encrypted; + BEGIN + + -- jsonb arrays of ste_vec encrypted values + sv_a := eql_v2.ste_vec(a); + sv_b := eql_v2.ste_vec(b); + + -- an empty b is always contained in a + IF array_length(sv_b, 1) IS NULL THEN + RETURN true; + END IF; + + IF array_length(sv_a, 1) IS NULL THEN + RETURN false; + END IF; + + result := true; + + -- for each element of b check if it is in a + FOR idx IN 1..array_length(sv_b, 1) LOOP + _b := sv_b[idx]; + result := result AND eql_v2.ste_vec_contains(sv_a, _b); + END LOOP; + + RETURN result; + END; +$$ LANGUAGE plpgsql; diff --git a/src/ste_vec/functions_test.sql b/src/ste_vec/functions_test.sql new file mode 100644 index 00000000..5f1f92ba --- /dev/null +++ b/src/ste_vec/functions_test.sql @@ -0,0 +1,92 @@ +\set ON_ERROR_STOP on + +SELECT create_table_with_encrypted(); +SELECT seed_encrypted_json(); + + +DO $$ + DECLARE + e eql_v2_encrypted; + sv eql_v2_encrypted[]; + BEGIN + + SELECT encrypted.e FROM encrypted LIMIT 1 INTO e; + + sv := eql_v2.ste_vec(e); + ASSERT array_length(sv, 1) = 3; + + -- eql_v2_encrypted that IS a ste_vec element + e := get_numeric_ste_vec_10()::eql_v2_encrypted; + + sv := eql_v2.ste_vec(e); + ASSERT array_length(sv, 1) = 3; + + END; +$$ LANGUAGE plpgsql; + + +DO $$ + DECLARE + e eql_v2_encrypted; + sv eql_v2_encrypted[]; + BEGIN + e := '{ "a": 1 }'::jsonb::eql_v2_encrypted; + ASSERT eql_v2.is_ste_vec_array(e); + + + e := '{ "a": 0 }'::jsonb::eql_v2_encrypted; + ASSERT NOT eql_v2.is_ste_vec_array(e); + + e := '{ }'::jsonb::eql_v2_encrypted; + ASSERT NOT eql_v2.is_ste_vec_array(e); + END; +$$ LANGUAGE plpgsql; + + +-- ------------------------------------------------------------------------ +-- ------------------------------------------------------------------------ +-- +-- eql_v2_encrypted contains itself +-- +-- +DO $$ + DECLARE + a eql_v2_encrypted; + b eql_v2_encrypted; + BEGIN + + a := get_numeric_ste_vec_10()::eql_v2_encrypted; + b := get_numeric_ste_vec_10()::eql_v2_encrypted; + + ASSERT eql_v2.ste_vec_contains(a, b); + ASSERT eql_v2.ste_vec_contains(b, a); + + END; +$$ LANGUAGE plpgsql; + + +-- ------------------------------------------------------------------------ +-- ------------------------------------------------------------------------ +-- +-- eql_v2_encrypted contains a term +-- +-- +DO $$ + DECLARE + a eql_v2_encrypted; + b eql_v2_encrypted; + term eql_v2_encrypted; + BEGIN + + a := get_numeric_ste_vec_10()::eql_v2_encrypted; + b := get_numeric_ste_vec_10()::eql_v2_encrypted; + + -- $.n + term := b->'2517068c0d1f9d4d41d2c666211f785e'; + + ASSERT eql_v2.ste_vec_contains(a, term); + + ASSERT NOT eql_v2.ste_vec_contains(term, a); + END; +$$ LANGUAGE plpgsql; + diff --git a/src/unique/functions.sql b/src/unique/functions.sql new file mode 100644 index 00000000..d65e0c62 --- /dev/null +++ b/src/unique/functions.sql @@ -0,0 +1,30 @@ +-- REQUIRE: src/schema.sql +-- REQUIRE: src/unique/types.sql + +-- extracts unique index from an encrypted column + +CREATE FUNCTION eql_v2.unique(val jsonb) + RETURNS eql_v2.unique_index + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + IF val ? 'u' THEN + RETURN val->>'u'; + END IF; + RAISE 'Expected a unique index (u) value in json: %', val; + END; +$$ LANGUAGE plpgsql; + + +-- extracts unique index from an encrypted column + +CREATE FUNCTION eql_v2.unique(val eql_v2_encrypted) + RETURNS eql_v2.unique_index + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + BEGIN + RETURN (SELECT eql_v2.unique(val.data)); + END; +$$ LANGUAGE plpgsql; + + diff --git a/src/unique/functions_test.sql b/src/unique/functions_test.sql new file mode 100644 index 00000000..baccea4e --- /dev/null +++ b/src/unique/functions_test.sql @@ -0,0 +1,14 @@ +\set ON_ERROR_STOP on + +DO $$ + BEGIN + PERFORM assert_result( + 'Extract unique index term from encrypted', + 'SELECT eql_v2.unique(''{"u": "u"}''::jsonb)'); + + PERFORM assert_exception( + 'Missing unique index term in encrypted raises exception', + 'SELECT eql_v2.unique(''{}''::jsonb)'); + + END; +$$ LANGUAGE plpgsql; diff --git a/src/unique/types.sql b/src/unique/types.sql new file mode 100644 index 00000000..3286091c --- /dev/null +++ b/src/unique/types.sql @@ -0,0 +1,3 @@ +-- REQUIRE: src/schema.sql + +CREATE DOMAIN eql_v2.unique_index AS text; diff --git a/src/version.template b/src/version.template new file mode 100644 index 00000000..d98c76b1 --- /dev/null +++ b/src/version.template @@ -0,0 +1,13 @@ +-- AUTOMATICALLY GENERATED FILE +-- Source is version-template.sql +-- REQUIRE: src/schema.sql + +DROP FUNCTION IF EXISTS eql_v2.version(); + +CREATE FUNCTION eql_v2.version() + RETURNS text + IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT '$RELEASE_VERSION'; +$$ LANGUAGE SQL; + diff --git a/src/version_test.sql b/src/version_test.sql new file mode 100644 index 00000000..cb717454 --- /dev/null +++ b/src/version_test.sql @@ -0,0 +1,9 @@ +\set ON_ERROR_STOP on + +DO $$ + BEGIN + PERFORM assert_result( + 'eql_v2.version()', + 'SELECT true WHERE eql_v2.version() = ''DEV'''); + END; +$$ LANGUAGE plpgsql; diff --git a/tasks/000-version-template.sql b/tasks/000-version-template.sql deleted file mode 100644 index dba8e9eb..00000000 --- a/tasks/000-version-template.sql +++ /dev/null @@ -1,9 +0,0 @@ -DROP FUNCTION IF EXISTS cs_eql_version(); - -CREATE FUNCTION cs_eql_version() - RETURNS text - IMMUTABLE STRICT PARALLEL SAFE -AS $$ - SELECT '$RELEASE_VERSION'; -$$ LANGUAGE SQL; - diff --git a/tasks/build.sh b/tasks/build.sh index bb4ad314..42af26f0 100755 --- a/tasks/build.sh +++ b/tasks/build.sh @@ -1,58 +1,87 @@ #!/usr/bin/env bash #MISE description="Build SQL into single release file" #MISE alias="b" -#MISE sources=["sql/*.sql"] +#MISE sources=["src/**/*.sql"] #MISE outputs=["release/cipherstash-encrypt.sql","release/cipherstash-encrypt-uninstall.sql"] #USAGE flag "--version " help="Specify release version of EQL" default="DEV" #!/bin/bash -set -euxo pipefail +# set -euxo pipefail mkdir -p release rm -f release/cipherstash-encrypt-uninstall.sql rm -f release/cipherstash-encrypt.sql -rm -f sql/000-version.sql +rm -f release/cipherstash-encrypt-uninstall-supabase.sql +rm -f release/cipherstash-encrypt-supabase.sql -RELEASE_VERSION=${usage_version} -sed "s/\$RELEASE_VERSION/$RELEASE_VERSION/g" tasks/000-version-template.sql > sql/000-version.sql +rm -f src/version.sql +rm -f src/deps-supabase.txt +rm -f src/deps-ordered-supabase.txt -# Collect all the drops -# In reverse order (tac) so that we drop the constraints before the tables -grep -h -E '^(DROP)' sql/0*-*.sql | tac > release/cipherstash-encrypt-tmp-drop-install.sql -# types are always last -cat sql/666-drop_types.sql >> release/cipherstash-encrypt-tmp-drop-install.sql +RELEASE_VERSION=${usage_version:-DEV} +sed "s/\$RELEASE_VERSION/$RELEASE_VERSION/g" src/version.template > src/version.sql -# Build cipherstash-encrypt.sql -# drop everything first -cat sql/666-drop-operators.sql > release/cipherstash-encrypt.sql -cat release/cipherstash-encrypt-tmp-drop-install.sql >> release/cipherstash-encrypt.sql -# cat the rest of the sql files -cat sql/0*-*.sql >> release/cipherstash-encrypt.sql +find src -type f -path "*.sql" ! -path "*_test.sql" | while IFS= read -r sql_file; do + echo $sql_file -# Collect all the drops -# In reverse order (tac) so that we drop the constraints before the tables -grep -h -E '^(DROP|ALTER DOMAIN [^ ]+ DROP CONSTRAINT)' sql/0*-*.sql | tac > release/cipherstash-encrypt-tmp-drop-uninstall.sql -# types are always last -cat sql/666-drop_types.sql >> release/cipherstash-encrypt-tmp-drop-uninstall.sql + echo "$sql_file $sql_file" >> src/deps.txt + while IFS= read -r line; do + # echo $line + # Check if the line contains "-- REQUIRE:" + if [[ "$line" == *"-- REQUIRE:"* ]]; then + # Extract the required file(s) after "-- REQUIRE:" + deps=${line#*-- REQUIRE: } -# Build cipherstash-encrypt-uninstall.sql -# prepend the drops to the main sql file -cat sql/666-drop-operators.sql >> release/cipherstash-encrypt-uninstall.sql -cat release/cipherstash-encrypt-tmp-drop-uninstall.sql >> release/cipherstash-encrypt-uninstall.sql + # Split multiple REQUIRE declarations if present + for dep in $deps; do + echo "$sql_file $dep" >> src/deps.txt + done + fi + done < "$sql_file" +done -# uninstall renames configuration table -cat sql/666-rename_configuration_table.sql >> release/cipherstash-encrypt-uninstall.sql +cat src/deps.txt | tsort | tac > src/deps-ordered.txt + +cat src/deps-ordered.txt | xargs cat | grep -v REQUIRE >> release/cipherstash-encrypt.sql + +cat tasks/uninstall.sql >> release/cipherstash-encrypt-uninstall.sql + + +# Supabase specific build which excludes operator classes as they are not supported +find src -type f -path "*.sql" ! -path "*_test.sql" ! -path "**/operator_class.sql" | while IFS= read -r sql_file; do + echo $sql_file + + echo "$sql_file $sql_file" >> src/deps-supabase.txt + + while IFS= read -r line; do + # echo $line + # Check if the line contains "-- REQUIRE:" + if [[ "$line" == *"-- REQUIRE:"* ]]; then + # Extract the required file(s) after "-- REQUIRE:" + deps=${line#*-- REQUIRE: } + + # Split multiple REQUIRE declarations if present + for dep in $deps; do + echo "$sql_file $dep" >> src/deps-supabase.txt + done + fi + done < "$sql_file" +done + + +cat src/deps-supabase.txt | tsort | tac > src/deps-ordered-supabase.txt + +cat src/deps-ordered-supabase.txt | xargs cat | grep -v REQUIRE >> release/cipherstash-encrypt-supabase.sql + +cat tasks/uninstall.sql >> release/cipherstash-encrypt-uninstall-supabase.sql -# remove the drop file -rm release/cipherstash-encrypt-tmp-drop-install.sql -rm release/cipherstash-encrypt-tmp-drop-uninstall.sql set +x echo @@ -60,5 +89,10 @@ echo '###############################################' echo "# ✅Build succeeded" echo '###############################################' echo -echo 'Installer: release/cipherstash-encrypt.sql' -echo 'Uninstaller: release/cipherstash-encrypt-uninstall.sql' +echo 'Installer:' +echo ' release/cipherstash-encrypt.sql' +echo ' release/cipherstash-encrypt-supabase.sql' +echo +echo 'Uninstaller:' +echo ' release/cipherstash-encrypt-uninstall.sql' +echo ' release/cipherstash-encrypt-uninstall-supabase.sql' diff --git a/tasks/postgres.toml b/tasks/postgres.toml index 5df6d898..3232ffdd 100644 --- a/tasks/postgres.toml +++ b/tasks/postgres.toml @@ -10,3 +10,10 @@ run = """ {% set default_service = "postgres-" ~ get_env(name="POSTGRES_VERSION",default="17") %} echo docker compose up {{arg(name="service",default=default_service)}} {{option(name="extra-args",default="")}} | bash """ + +["postgres:reset"] +description = "Reset database" +run = """ +mise run postgres:down +mise run postgres:up --extra-args "--detach --wait" +""" diff --git a/tasks/reset.sh b/tasks/reset.sh index 2d6257e5..079bdfdd 100755 --- a/tasks/reset.sh +++ b/tasks/reset.sh @@ -27,7 +27,4 @@ fail_if_postgres_not_running cat release/cipherstash-encrypt-uninstall.sql | docker exec -i ${container_name} psql ${connection_url} -f- # Wipe test data -cat tests/999-wipe-test-data.sql | docker exec -i ${container_name} psql ${connection_url} -f- - -# Install -cat release/cipherstash-encrypt.sql | docker exec -i ${container_name} psql ${connection_url} -f- +cat tasks/reset.sql | docker exec -i ${container_name} psql ${connection_url} -f- diff --git a/tests/999-wipe-test-data.sql b/tasks/reset.sql similarity index 62% rename from tests/999-wipe-test-data.sql rename to tasks/reset.sql index 3fdf7003..3178228a 100644 --- a/tests/999-wipe-test-data.sql +++ b/tasks/reset.sql @@ -5,3 +5,8 @@ DROP SCHEMA public CASCADE; CREATE SCHEMA public; GRANT ALL ON SCHEMA public TO postgres; GRANT ALL ON SCHEMA public TO public; + +DROP SCHEMA eql_v2 CASCADE; +CREATE SCHEMA eql_v2; +GRANT ALL ON SCHEMA eql_v2 TO postgres; +GRANT ALL ON SCHEMA eql_v2 TO public; diff --git a/tasks/test.sh b/tasks/test.sh index 57013a00..c86909c2 100755 --- a/tasks/test.sh +++ b/tasks/test.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash #MISE description="Build, reset and run tests" +#USAGE flag "--test " help="Test to run" default="false" #USAGE flag "--postgres " help="Run tests for specified Postgres version" default="17" { #USAGE choices "14" "15" "16" "17" #USAGE } @@ -25,27 +26,49 @@ fail_if_postgres_not_running () { run_test () { echo echo '###############################################' - echo "# ${1}" + echo "# Running Test: ${1}" echo '###############################################' echo - cat $1 | docker exec -i ${container_name} psql $connection_url -f- + + cat $1 | docker exec -i ${container_name} psql --variable ON_ERROR_STOP=1 $connection_url -f- } # setup fail_if_postgres_not_running -mise run build -mise run reset --postgres ${POSTGRES_VERSION} - -# tests -run_test tests/version.sql -run_test tests/core.sql -run_test tests/core-functions.sql -run_test tests/config.sql -run_test tests/encryptindex.sql -run_test tests/operators-eq.sql -run_test tests/operators-match.sql -run_test tests/operators-ore.sql -run_test tests/aggregate-ore.sql +mise run build --force +mise run reset --force --postgres ${POSTGRES_VERSION} + +echo '/////////////////////////////////////////////////////////' +cat release/cipherstash-encrypt.sql +echo '/////////////////////////////////////////////////////////' + + +# Install +# cat release/cipherstash-encrypt.sql | docker exec -i ${container_name} psql ${connection_url} -f- +if cat release/cipherstash-encrypt.sql | docker exec -i ${container_name} psql ${connection_url} -f- | grep -q "ERROR"; then + echo + echo '******************************************************' + echo '* ❌ ERROR installing release/cipherstash-encrypt.sql' + echo '******************************************************' + echo + + exit 1 +fi + + +cat tests/test_helpers.sql | docker exec -i ${container_name} psql ${connection_url} -f- +cat tests/ore.sql | docker exec -i ${container_name} psql ${connection_url} -f- + +if [ $usage_test = "false" ]; then + find src -type f -path "*_test.sql" | while read -r sql_file; do + echo $sql_file + run_test $sql_file + done +else + find src -type f -path "*$usage_test*" | while read -r sql_file; do + run_test $sql_file + done +fi echo echo '###############################################' diff --git a/tasks/uninstall.sql b/tasks/uninstall.sql new file mode 100644 index 00000000..2ab226e4 --- /dev/null +++ b/tasks/uninstall.sql @@ -0,0 +1,10 @@ + +DO $$ +BEGIN + EXECUTE format('ALTER TABLE IF EXISTS %I RENAME TO %I_%s', 'eql_v2_configuration','eql_v2_configuration_', to_char(current_date,'YYYYMMDD')::TEXT); +END +$$; + +ALTER TABLE public.eql_v2_configuration DROP CONSTRAINT IF EXISTS eql_v2_configuration_data_check; + +DROP SCHEMA IF EXISTS eql_v2 CASCADE; diff --git a/tests/core-functions.sql b/tests/core-functions.sql deleted file mode 100644 index 03187405..00000000 --- a/tests/core-functions.sql +++ /dev/null @@ -1,78 +0,0 @@ -\set ON_ERROR_STOP on - - -DO $$ - BEGIN - ASSERT (SELECT EXISTS (SELECT cs_unique_v1('{"u": "u"}'::jsonb))); - ASSERT (SELECT EXISTS (SELECT cs_match_v1('{"m": []}'::jsonb))); - ASSERT (SELECT EXISTS (SELECT cs_ste_vec_v1('{"sv": [[]]}'::jsonb))); - ASSERT (SELECT EXISTS (SELECT cs_ore_64_8_v1('{"o": []}'::jsonb))); - - END; -$$ LANGUAGE plpgsql; - -DO $$ - BEGIN - -- sanity check - PERFORM cs_ore_64_8_v1('{"o": []}'::jsonb); - - BEGIN - PERFORM cs_ore_64_8_v1('{}'::jsonb); - RAISE NOTICE 'Missing index. Function call should have failed.'; - ASSERT false; - EXCEPTION - WHEN OTHERS THEN - ASSERT true; - END; - END; -$$ LANGUAGE plpgsql; - -DO $$ - BEGIN - -- sanity check - PERFORM cs_ste_vec_v1('{"sv": [[]]}'::jsonb); - - BEGIN - PERFORM cs_ste_vec_v1('{}'::jsonb); - RAISE NOTICE 'Missing index. Function call should have failed.'; - ASSERT false; - EXCEPTION - WHEN OTHERS THEN - ASSERT true; - END; - END; -$$ LANGUAGE plpgsql; - - -DO $$ - BEGIN - -- sanity check - PERFORM cs_unique_v1('{"u": "u"}'::jsonb); - - BEGIN - PERFORM cs_unique_v1_v0('{}'::jsonb); - RAISE NOTICE 'Missing index. Function call should have failed.'; - ASSERT false; - EXCEPTION - WHEN OTHERS THEN - ASSERT true; - END; - END; -$$ LANGUAGE plpgsql; - - -DO $$ - BEGIN - -- sanity check - PERFORM cs_match_v1('{"m": []}'::jsonb); - - BEGIN - PERFORM cs_match_v1('{}'::jsonb); - RAISE NOTICE 'Missing index. Function call should have failed.'; - ASSERT false; - EXCEPTION - WHEN OTHERS THEN - ASSERT true; - END; - END; -$$ LANGUAGE plpgsql; diff --git a/tests/core.sql b/tests/core.sql deleted file mode 100644 index 5a02fcd9..00000000 --- a/tests/core.sql +++ /dev/null @@ -1,219 +0,0 @@ -\set ON_ERROR_STOP on - --- Create a table with a plaintext column -DROP TABLE IF EXISTS users; -CREATE TABLE users -( - id bigint GENERATED ALWAYS AS IDENTITY, - name_encrypted cs_encrypted_v1, - PRIMARY KEY(id) -); - - -TRUNCATE TABLE users; - - -INSERT INTO users (name_encrypted) VALUES ( - '{ - "v": 1, - "k": "ct", - "c": "ciphertext", - "i": { - "t": "users", - "c": "name" - }, - "m": [1, 1], - "u": "text", - "o": ["a"] - }'::jsonb -); - -DO $$ - BEGIN - - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE cs_ciphertext_v1(name_encrypted) = 'ciphertext')); - - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE cs_match_v1(name_encrypted) = '{1,1}')); - - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE cs_unique_v1(name_encrypted) = 'text')); - - -- ORE PAYLOAD ABOUT TO CHANGE - -- ASSERT (SELECT EXISTS (SELECT id FROM users WHERE cs_ore_64_8_v1(name_encrypted) = '{a}')); - - END; -$$ LANGUAGE plpgsql; - - -TRUNCATE TABLE users; - -INSERT INTO users DEFAULT VALUES; - -SELECT id FROM users; - -DO $$ - BEGIN - ASSERT (SELECT EXISTS (SELECT id FROM users)); - END; -$$ LANGUAGE plpgsql; - - --- ----------------------------------------------- ---- --- cs_encrypted_v1 type --- Validate configuration schema --- Try and insert many invalid configurations --- None should exist --- --- ----------------------------------------------- -TRUNCATE TABLE users; - -\set ON_ERROR_STOP off -\set ON_ERROR_ROLLBACK on - -DO $$ - BEGIN - RAISE NOTICE 'cs_encrypted_v1 constraint tests: 10 errors expected here'; - END; -$$ LANGUAGE plpgsql; - - --- no version -INSERT INTO users (name_encrypted) VALUES ( - '{ - "v": 1, - "k": "ct", - "c": "ciphertext", - "i": { - "t": "users", - "c": "name" - } - }'::jsonb -); - --- no ident details -INSERT INTO users (name_encrypted) VALUES ( - '{ - "v": 1, - "k": "ct", - "c": "ciphertext" - }'::jsonb -); - --- no kind -INSERT INTO users (name_encrypted) VALUES ( - '{ - "v": 1, - "c": "ciphertext", - "i": { - "t": "users", - "c": "name" - } - }'::jsonb -); - - - --- bad kind -INSERT INTO users (name_encrypted) VALUES ( - '{ - "v": 1, - "k": "vtha", - "c": "ciphertext", - "i": { - "t": "users", - "c": "name" - } - }'::jsonb -); - --- pt -INSERT INTO users (name_encrypted) VALUES ( - '{ - "v": 1, - "k": "pt", - "i": { - "t": "users", - "c": "name" - } - }'::jsonb -); - ---pt with ciphertext -INSERT INTO users (name_encrypted) VALUES ( - '{ - "v": 1, - "k": "pt", - "c": "ciphertext", - "i": { - "t": "users", - "c": "name" - } - }'::jsonb -); - --- ct without ciphertext -INSERT INTO users (name_encrypted) VALUES ( - '{ - "v": 1, - "k": "ct", - "i": { - "t": "users", - "c": "name" - } - }'::jsonb -); - - --- ct with plaintext -INSERT INTO users (name_encrypted) VALUES ( - '{ - "v": 1, - "k": "ct", - "p": "plaintext", - "i": { - "t": "users", - "c": "name" - } - }'::jsonb -); - - --- ciphertext without ct -INSERT INTO users (name_encrypted) VALUES ( - '{ - "v": 1, - "c": "ciphertext", - "i": { - "t": "users", - "c": "name" - } - }'::jsonb -); - --- ciphertext with invalid q -INSERT INTO users (name_encrypted) VALUES ( - '{ - "v": 1, - "c": "ciphertext", - "i": { - "t": "users", - "c": "name" - }, - "q": "invalid" - }'::jsonb -); - --- Nothing should be in the DB -DO $$ - BEGIN - ASSERT (SELECT NOT EXISTS (SELECT * FROM users c)); - END; -$$ LANGUAGE plpgsql; - - -\set ON_ERROR_STOP on -\set ON_ERROR_ROLLBACK off - - - - diff --git a/tests/operators-eq.sql b/tests/operators-eq.sql deleted file mode 100644 index ad51ee54..00000000 --- a/tests/operators-eq.sql +++ /dev/null @@ -1,267 +0,0 @@ --- Create a table with a plaintext column -DROP TABLE IF EXISTS users; -CREATE TABLE users -( - id bigint GENERATED ALWAYS AS IDENTITY, - name_encrypted cs_encrypted_v1, - PRIMARY KEY(id) -); - - -INSERT INTO users (name_encrypted) VALUES ( - '{ - "v": 1, - "k": "ct", - "c": "ciphertext", - "i": { - "t": "users", - "c": "name" - }, - "m": [1, 2, 3], - "u": "unique-text" - }'::jsonb -); - - - --- UNIQUE eq = OPERATORS -DO $$ - BEGIN - -- SANITY CHECK - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE cs_unique_v1(name_encrypted) = cs_unique_v1('{"u":"unique-text"}'))); - - ASSERT (SELECT EXISTS ( - SELECT id FROM users WHERE name_encrypted = '{ - "v": 1, - "k": "ct", - "c": "ciphertext", - "i": { - "t": "users", - "c": "name" - }, - "u": "unique-text" - }'::jsonb - )); - - -- cs_encrypted_v1 = jsonb - ASSERT (SELECT EXISTS ( - SELECT id FROM users WHERE name_encrypted = '{"u": "unique-text"}'::jsonb - )); - - -- jsonb = cs_encrypted_v1 - ASSERT (SELECT EXISTS ( - SELECT id FROM users WHERE '{"u": "unique-text"}'::jsonb = name_encrypted - )); - - -- cs_encrypted_v1 = text - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted = 'unique-text'::text)); - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted = 'unique-text'::cs_unique_index_v1)); - - -- text = cs_encrypted_v1 - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE 'unique-text'::text = name_encrypted)); - - -- cs_encrypted_v1 = cs_encrypted_v1 - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted = '{ - "v": 1, - "k": "ct", - "c": "ciphertext", - "i": { - "t": "users", - "c": "name" - }, - "u": "unique-text" - }'::cs_encrypted_v1)); - - END; -$$ LANGUAGE plpgsql; - - --- UNIQUE inequality <> OPERATORS -DO $$ - BEGIN - -- SANITY CHECK - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE cs_unique_v1(name_encrypted) != cs_unique_v1('{"u":"random-text"}'))); - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE cs_unique_v1(name_encrypted) <> cs_unique_v1('{"u":"random-text"}'))); - - -- cs_encrypted_v1 = jsonb - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted != '{"u":"random-text"}'::jsonb)); - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted <> '{"u":"random-text"}'::jsonb)); - - -- cs_encrypted_v1 = text - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted != 'random-text'::text)); - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted <> 'random-text'::text)); - - -- cs_encrypted_v1 = cs_encrypted_v1 - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted != '{ - "v": 1, - "k": "ct", - "c": "ciphertext", - "i": { - "t": "users", - "c": "name" - }, - "u": "random-text" - }'::cs_encrypted_v1)); - - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted <> '{ - "v": 1, - "k": "ct", - "c": "ciphertext", - "i": { - "t": "users", - "c": "name" - }, - "u": "random-text" - }'::cs_encrypted_v1)); - - - END; -$$ LANGUAGE plpgsql; - - -TRUNCATE TABLE users; - --- --- Example ORE values are generated from an array in the form `vec![0, 1, 2, 3, 4, 5]`; --- --- JSON values are JSON escaped on top of a PostgreSQL escaped Record --- --- PostgreSQL value is ("{""(\\""\\\\\\\\x000102030405\\"")""}") --- --- -INSERT INTO users (name_encrypted) VALUES ( - '{ - "v": 1, - "k": "ct", - "c": "ciphertext", - "i": { - "t": "users", - "c": "name" - }, - "m": [1, 2, 3], - "o":"(\"{\"\"(\\\\\"\"\\\\\\\\\\\\\\\\x12121212121259bfe28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea5c9fdb3cc34da8b152b995957591880c523beb1d3f12487c38d18f62dd26209a727674e5a5fe3a3e3037860839afd801ff4a28b714e4cde8df10625dce72602fdbdcc53d515857f1119f5912804ce09c6cf6c2d37393a27a465134523b512664582f834e15003b7216cb668480bc3e7d1c069f2572ece7c848b9eb9a28b4e62bfc2b97c93e61b2054154e621c5bbb7bed37de3d7c343bd3dbcf7b4af20128c961351bf55910a855f08a8587c2059a5f05ca8d7a082e695b3dd4ff3ce86694d4fe98972220eea1ab90f5de493ef3a502b74a569f103ee2897ebc9ae9b16a17e7be67415ee830519beb3058ffc1c1eb0e574d66c8b365919f27eb00aa7bce475d7bdaad4ed800f8fc3d626e0eb842e312b0cc22a1ccf89847ebb2cd0a6e18aec21bd2deeec1c47301fc687f7f764bb882b50f553c246a6da5816b78b3530119ea68b08a8403a90e063e58502670563bd4d\\\\\"\")\"\"}\")" - }'::jsonb -); - --- ORE eq = OPERATORS -DO $$ - DECLARE - ore_cs_encrypted cs_encrypted_v1; - ore_json jsonb; - ore_record text; - BEGIN - ore_cs_encrypted := '{ - "v": 1, - "k": "ct", - "c": "ciphertext", - "i": { - "t": "users", - "c": "name" - }, - "m": [1, 2, 3], - "o":"(\"{\"\"(\\\\\"\"\\\\\\\\\\\\\\\\x12121212121259bfe28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea5c9fdb3cc34da8b152b995957591880c523beb1d3f12487c38d18f62dd26209a727674e5a5fe3a3e3037860839afd801ff4a28b714e4cde8df10625dce72602fdbdcc53d515857f1119f5912804ce09c6cf6c2d37393a27a465134523b512664582f834e15003b7216cb668480bc3e7d1c069f2572ece7c848b9eb9a28b4e62bfc2b97c93e61b2054154e621c5bbb7bed37de3d7c343bd3dbcf7b4af20128c961351bf55910a855f08a8587c2059a5f05ca8d7a082e695b3dd4ff3ce86694d4fe98972220eea1ab90f5de493ef3a502b74a569f103ee2897ebc9ae9b16a17e7be67415ee830519beb3058ffc1c1eb0e574d66c8b365919f27eb00aa7bce475d7bdaad4ed800f8fc3d626e0eb842e312b0cc22a1ccf89847ebb2cd0a6e18aec21bd2deeec1c47301fc687f7f764bb882b50f553c246a6da5816b78b3530119ea68b08a8403a90e063e58502670563bd4d\\\\\"\")\"\"}\")" - }'; - - ore_json := '{"o":"(\"{\"\"(\\\\\"\"\\\\\\\\\\\\\\\\x12121212121259bfe28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea5c9fdb3cc34da8b152b995957591880c523beb1d3f12487c38d18f62dd26209a727674e5a5fe3a3e3037860839afd801ff4a28b714e4cde8df10625dce72602fdbdcc53d515857f1119f5912804ce09c6cf6c2d37393a27a465134523b512664582f834e15003b7216cb668480bc3e7d1c069f2572ece7c848b9eb9a28b4e62bfc2b97c93e61b2054154e621c5bbb7bed37de3d7c343bd3dbcf7b4af20128c961351bf55910a855f08a8587c2059a5f05ca8d7a082e695b3dd4ff3ce86694d4fe98972220eea1ab90f5de493ef3a502b74a569f103ee2897ebc9ae9b16a17e7be67415ee830519beb3058ffc1c1eb0e574d66c8b365919f27eb00aa7bce475d7bdaad4ed800f8fc3d626e0eb842e312b0cc22a1ccf89847ebb2cd0a6e18aec21bd2deeec1c47301fc687f7f764bb882b50f553c246a6da5816b78b3530119ea68b08a8403a90e063e58502670563bd4d\\\\\"\")\"\"}\")"}'; - - - ore_record = '("{""(\\""\\\\\\\\x12121212121259bfe28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea5c9fdb3cc34da8b152b995957591880c523beb1d3f12487c38d18f62dd26209a727674e5a5fe3a3e3037860839afd801ff4a28b714e4cde8df10625dce72602fdbdcc53d515857f1119f5912804ce09c6cf6c2d37393a27a465134523b512664582f834e15003b7216cb668480bc3e7d1c069f2572ece7c848b9eb9a28b4e62bfc2b97c93e61b2054154e621c5bbb7bed37de3d7c343bd3dbcf7b4af20128c961351bf55910a855f08a8587c2059a5f05ca8d7a082e695b3dd4ff3ce86694d4fe98972220eea1ab90f5de493ef3a502b74a569f103ee2897ebc9ae9b16a17e7be67415ee830519beb3058ffc1c1eb0e574d66c8b365919f27eb00aa7bce475d7bdaad4ed800f8fc3d626e0eb842e312b0cc22a1ccf89847ebb2cd0a6e18aec21bd2deeec1c47301fc687f7f764bb882b50f553c246a6da5816b78b3530119ea68b08a8403a90e063e58502670563bd4d\\"")""}")'; - - -- SANITY CHECK - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE cs_ore_64_8_v1(name_encrypted) = cs_ore_64_8_v1(ore_json))); - - ASSERT (SELECT EXISTS ( - SELECT id FROM users WHERE name_encrypted = ore_cs_encrypted::jsonb - )); - - -- -- cs_encrypted_v1 = jsonb - ASSERT (SELECT EXISTS ( - SELECT id FROM users WHERE name_encrypted = ore_json::jsonb - )); - - -- -- jsonb = cs_encrypted_v1 - ASSERT (SELECT EXISTS ( - SELECT id FROM users WHERE ore_json::jsonb = name_encrypted - )); - - -- -- cs_encrypted_v1 = ore_64_8_v1 - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted = ore_record::ore_64_8_v1)); - - -- -- -- ore_64_8_v1 = cs_encrypted_v1 - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE ore_record::ore_64_8_v1 = name_encrypted)); - - -- -- -- cs_encrypted_v1 = cs_encrypted_v1 - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted = ore_cs_encrypted::cs_encrypted_v1)); - - END; -$$ LANGUAGE plpgsql; - -TRUNCATE TABLE users; - -INSERT INTO users (name_encrypted) VALUES ( - '{ - "v": 1, - "k": "ct", - "c": "ciphertext", - "i": { - "t": "users", - "c": "name" - }, - "m": [1, 2, 3], - "o":"(\"{\"\"(\\\\\"\"\\\\\\\\\\\\\\\\x12121212121259bfe28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea5c9fdb3cc34da8b152b995957591880c523beb1d3f12487c38d18f62dd26209a727674e5a5fe3a3e3037860839afd801ff4a28b714e4cde8df10625dce72602fdbdcc53d515857f1119f5912804ce09c6cf6c2d37393a27a465134523b512664582f834e15003b7216cb668480bc3e7d1c069f2572ece7c848b9eb9a28b4e62bfc2b97c93e61b2054154e621c5bbb7bed37de3d7c343bd3dbcf7b4af20128c961351bf55910a855f08a8587c2059a5f05ca8d7a082e695b3dd4ff3ce86694d4fe98972220eea1ab90f5de493ef3a502b74a569f103ee2897ebc9ae9b16a17e7be67415ee830519beb3058ffc1c1eb0e574d66c8b365919f27eb00aa7bce475d7bdaad4ed800f8fc3d626e0eb842e312b0cc22a1ccf89847ebb2cd0a6e18aec21bd2deeec1c47301fc687f7f764bb882b50f553c246a6da5816b78b3530119ea68b08a8403a90e063e58502670563bd4d\\\\\"\")\"\"}\")" - }'::jsonb -); - --- ORE eq = OPERATORS -DO $$ - DECLARE - ore_cs_encrypted cs_encrypted_v1; - ore_json jsonb; - ore_record text; - BEGIN - ore_cs_encrypted := '{ - "v": 1, - "k": "ct", - "c": "ciphertext", - "i": { - "t": "users", - "c": "name" - }, - "m": [1, 2, 3], - "o":"(\"{\"\"(\\\\\"\"\\\\\\\\\\\\\\\\x1212121212125932e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea5c9fdb3cc34da8b152b995957591880c523beb1d3f12487c38d18f62dd26209a727674e5a5fe3a3e3037860839afd8011f94b49eaa5fa5a60e1e2adccde4185a7d6c7f83088500b677f897d4ffc276016d614708488f407c01bd3ccf2be653269062cb97f8945a621d049277d19b1c248611f25d047038928d2efeb4323c402af4c19288c7b36911dc06639af5bb34367519b66c1f525bbd3828c12067c9c579aeeb4fb3ae0918125dc1dad5fd518019a5ae67894ce1a7f7bed1a591ba8edda2fdf4cd403761fd981fb1ea5eb0bf806f919350ee60cac16d0a39a491a4d79301781f95ea3870aea82e9946053537360b2fb415b18b61aed0af81d461ad6b923f10c0df79daddc4e279ff543a282bb3a37f9fa03238348b3dac51a453b04bced1f5bd318ddd829bdfe5f37abdbeda730e21441b818302f3c5c2c4d5657accfca4c53d7a80eb3db43946d38965be5f796b\\\\\"\")\"\"}\")" - }'; - - ore_json := '{"o":"(\"{\"\"(\\\\\"\"\\\\\\\\\\\\\\\\x1212121212125932e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea5c9fdb3cc34da8b152b995957591880c523beb1d3f12487c38d18f62dd26209a727674e5a5fe3a3e3037860839afd8011f94b49eaa5fa5a60e1e2adccde4185a7d6c7f83088500b677f897d4ffc276016d614708488f407c01bd3ccf2be653269062cb97f8945a621d049277d19b1c248611f25d047038928d2efeb4323c402af4c19288c7b36911dc06639af5bb34367519b66c1f525bbd3828c12067c9c579aeeb4fb3ae0918125dc1dad5fd518019a5ae67894ce1a7f7bed1a591ba8edda2fdf4cd403761fd981fb1ea5eb0bf806f919350ee60cac16d0a39a491a4d79301781f95ea3870aea82e9946053537360b2fb415b18b61aed0af81d461ad6b923f10c0df79daddc4e279ff543a282bb3a37f9fa03238348b3dac51a453b04bced1f5bd318ddd829bdfe5f37abdbeda730e21441b818302f3c5c2c4d5657accfca4c53d7a80eb3db43946d38965be5f796b\\\\\"\")\"\"}\")"}'; - - - ore_record = '("{""(\\""\\\\\\\\x1212121212125932e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea5c9fdb3cc34da8b152b995957591880c523beb1d3f12487c38d18f62dd26209a727674e5a5fe3a3e3037860839afd8011f94b49eaa5fa5a60e1e2adccde4185a7d6c7f83088500b677f897d4ffc276016d614708488f407c01bd3ccf2be653269062cb97f8945a621d049277d19b1c248611f25d047038928d2efeb4323c402af4c19288c7b36911dc06639af5bb34367519b66c1f525bbd3828c12067c9c579aeeb4fb3ae0918125dc1dad5fd518019a5ae67894ce1a7f7bed1a591ba8edda2fdf4cd403761fd981fb1ea5eb0bf806f919350ee60cac16d0a39a491a4d79301781f95ea3870aea82e9946053537360b2fb415b18b61aed0af81d461ad6b923f10c0df79daddc4e279ff543a282bb3a37f9fa03238348b3dac51a453b04bced1f5bd318ddd829bdfe5f37abdbeda730e21441b818302f3c5c2c4d5657accfca4c53d7a80eb3db43946d38965be5f796b\\"")""}")'; - - -- SANITY CHECK - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE cs_ore_64_8_v1(name_encrypted) <> cs_ore_64_8_v1(ore_json))); - - ASSERT (SELECT EXISTS ( - SELECT id FROM users WHERE name_encrypted <> ore_cs_encrypted::jsonb - )); - - -- -- -- cs_encrypted_v1 <> jsonb - ASSERT (SELECT EXISTS ( - SELECT id FROM users WHERE name_encrypted <> ore_json::jsonb - )); - - -- -- -- jsonb <> cs_encrypted_v1 - ASSERT (SELECT EXISTS ( - SELECT id FROM users WHERE ore_json::jsonb <> name_encrypted - )); - - -- -- -- cs_encrypted_v1 <> ore_64_8_v1 - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted <> ore_record::ore_64_8_v1)); - - -- -- -- -- ore_64_8_v1 <> cs_encrypted_v1 - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE ore_record::ore_64_8_v1 <> name_encrypted)); - - -- -- -- -- cs_encrypted_v1 <> cs_encrypted_v1 - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted <> ore_cs_encrypted::cs_encrypted_v1)); - - END; -$$ LANGUAGE plpgsql; - diff --git a/tests/operators-match.sql b/tests/operators-match.sql deleted file mode 100644 index 58743d4d..00000000 --- a/tests/operators-match.sql +++ /dev/null @@ -1,173 +0,0 @@ -\set ON_ERROR_STOP on - - --- Create a table with a plaintext column -DROP TABLE IF EXISTS users; -CREATE TABLE users -( - id bigint GENERATED ALWAYS AS IDENTITY, - name_encrypted cs_encrypted_v1, - PRIMARY KEY(id) -); - -TRUNCATE TABLE users; - -INSERT INTO users (name_encrypted) VALUES ( - '{ - "v": 1, - "k": "ct", - "c": "ciphertext", - "i": { - "t": "users", - "c": "name" - }, - "m": [1, 2, 3], - "u": "unique-text", - "o": ["a"] - }'::jsonb -); - - - --- MATCH ~~ OPERATORS -DO $$ - BEGIN - -- SANITY CHECK - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE cs_match_v1(name_encrypted) ~~ cs_match_v1('{"m":[1,2]}'))); - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE cs_match_v1(name_encrypted) ~~* cs_match_v1('{"m":[1,2]}'))); - - -- cs_encrypted_v1 = jsonb - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted ~~ '{"m":[1,2]}'::jsonb)); - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted ~~* '{"m":[1,2]}'::jsonb)); - - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE '{"m":[1,2,3,6,7,8,9]}'::jsonb ~~ name_encrypted)); - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE '{"m":[1,2,3,6,7,8,9]}'::jsonb ~~* name_encrypted)); - - -- cs_encrypted_v1 = text - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted ~~ ARRAY[1,2]::smallint[])); - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted ~~* ARRAY[1,2]::smallint[])); - - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted ~~ ARRAY[1,2]::cs_match_index_v1)); - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted ~~* ARRAY[1,2]::cs_match_index_v1)); - - -- cs_encrypted_v1 = cs_encrypted_v1 - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted ~~ '{ - "v": 1, - "k": "ct", - "c": "ciphertext", - "i": { - "t": "users", - "c": "name" - }, - "m": [1, 2] - }'::cs_encrypted_v1)); - - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted ~~* '{ - "v": 1, - "k": "ct", - "c": "ciphertext", - "i": { - "t": "users", - "c": "name" - }, - "m": [1, 2] - }'::cs_encrypted_v1)); - - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE '{ - "v": 1, - "k": "ct", - "c": "ciphertext", - "i": { - "t": "users", - "c": "name" - }, - "m": [1, 2, 3, 4, 5] - }'::cs_encrypted_v1 ~~ name_encrypted)); - - - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE '{ - "v": 1, - "k": "ct", - "c": "ciphertext", - "i": { - "t": "users", - "c": "name" - }, - "m": [1, 2, 3, 4, 5] - }'::cs_encrypted_v1 ~~* name_encrypted)); - - END; -$$ LANGUAGE plpgsql; - - - - --- MATCH ~~ OPERATORS -DO $$ - BEGIN - -- SANITY CHECK - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE cs_match_v1(name_encrypted) LIKE cs_match_v1('{"m":[1,2]}'))); - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE cs_match_v1(name_encrypted) ILIKE cs_match_v1('{"m":[1,2]}'))); - - -- cs_encrypted_v1 = jsonb - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted LIKE '{"m":[1,2]}'::jsonb)); - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted ILIKE '{"m":[1,2]}'::jsonb)); - - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE '{"m":[1,2,3,6,7,8,9]}'::jsonb LIKE name_encrypted)); - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE '{"m":[1,2,3,6,7,8,9]}'::jsonb ILIKE name_encrypted)); - - -- cs_encrypted_v1 = text - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted LIKE ARRAY[1,2]::smallint[])); - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted ILIKE ARRAY[1,2]::smallint[])); - - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted LIKE ARRAY[1,2]::cs_match_index_v1)); - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted ILIKE ARRAY[1,2]::cs_match_index_v1)); - - -- cs_encrypted_v1 = cs_encrypted_v1 - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted LIKE '{ - "v": 1, - "k": "ct", - "c": "ciphertext", - "i": { - "t": "users", - "c": "name" - }, - "m": [1, 2] - }'::cs_encrypted_v1)); - - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted ILIKE '{ - "v": 1, - "k": "ct", - "c": "ciphertext", - "i": { - "t": "users", - "c": "name" - }, - "m": [1, 2] - }'::cs_encrypted_v1)); - - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE '{ - "v": 1, - "k": "ct", - "c": "ciphertext", - "i": { - "t": "users", - "c": "name" - }, - "m": [1, 2, 3, 4, 5] - }'::cs_encrypted_v1 LIKE name_encrypted)); - - - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE '{ - "v": 1, - "k": "ct", - "c": "ciphertext", - "i": { - "t": "users", - "c": "name" - }, - "m": [1, 2, 3, 4, 5] - }'::cs_encrypted_v1 ILIKE name_encrypted)); - - END; -$$ LANGUAGE plpgsql; \ No newline at end of file diff --git a/tests/operators-ore.sql b/tests/operators-ore.sql deleted file mode 100644 index 553157eb..00000000 --- a/tests/operators-ore.sql +++ /dev/null @@ -1,290 +0,0 @@ -\set ON_ERROR_STOP on - - --- Create a table with a plaintext column -DROP TABLE IF EXISTS users; -CREATE TABLE users -( - id bigint GENERATED ALWAYS AS IDENTITY, - name_encrypted cs_encrypted_v1, - PRIMARY KEY(id) -); - -TRUNCATE TABLE users; - -INSERT INTO users (name_encrypted) VALUES (NULL); - --- User with "LOW" value -INSERT INTO users (name_encrypted) VALUES ( - '{ - "v": 1, - "k": "ct", - "c": "ciphertext", - "i": { - "t": "users", - "c": "name" - }, - "m": [1, 2, 3], - "o": ["12121212121259bfe28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea5c9fdb3cc34da8b152b995957591880c523beb1d3f12487c38d18f62dd26209a727674e5a5fe3a3e3037860839afd801ff4a28b714e4cde8df10625dce72602fdbdcc53d515857f1119f5912804ce09c6cf6c2d37393a27a465134523b512664582f834e15003b7216cb668480bc3e7d1c069f2572ece7c848b9eb9a28b4e62bfc2b97c93e61b2054154e621c5bbb7bed37de3d7c343bd3dbcf7b4af20128c961351bf55910a855f08a8587c2059a5f05ca8d7a082e695b3dd4ff3ce86694d4fe98972220eea1ab90f5de493ef3a502b74a569f103ee2897ebc9ae9b16a17e7be67415ee830519beb3058ffc1c1eb0e574d66c8b365919f27eb00aa7bce475d7bdaad4ed800f8fc3d626e0eb842e312b0cc22a1ccf89847ebb2cd0a6e18aec21bd2deeec1c47301fc687f7f764bb882b50f553c246a6da5816b78b3530119ea68b08a8403a90e063e58502670563bd4d"] - }'::jsonb -); - --- ORE eq < OPERATORS -DO $$ - DECLARE - ore_cs_encrypted cs_encrypted_v1; - ore_json jsonb; - ore_record text; - BEGIN - ore_cs_encrypted := '{ - "v": 1, - "k": "ct", - "c": "ciphertext", - "i": { - "t": "users", - "c": "name" - }, - "m": [1, 2, 3], - "o": ["1212121212125932e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea5c9fdb3cc34da8b152b995957591880c523beb1d3f12487c38d18f62dd26209a727674e5a5fe3a3e3037860839afd8011f94b49eaa5fa5a60e1e2adccde4185a7d6c7f83088500b677f897d4ffc276016d614708488f407c01bd3ccf2be653269062cb97f8945a621d049277d19b1c248611f25d047038928d2efeb4323c402af4c19288c7b36911dc06639af5bb34367519b66c1f525bbd3828c12067c9c579aeeb4fb3ae0918125dc1dad5fd518019a5ae67894ce1a7f7bed1a591ba8edda2fdf4cd403761fd981fb1ea5eb0bf806f919350ee60cac16d0a39a491a4d79301781f95ea3870aea82e9946053537360b2fb415b18b61aed0af81d461ad6b923f10c0df79daddc4e279ff543a282bb3a37f9fa03238348b3dac51a453b04bced1f5bd318ddd829bdfe5f37abdbeda730e21441b818302f3c5c2c4d5657accfca4c53d7a80eb3db43946d38965be5f796b"] - }'; - - ore_json := '{"o": ["1212121212125932e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea5c9fdb3cc34da8b152b995957591880c523beb1d3f12487c38d18f62dd26209a727674e5a5fe3a3e3037860839afd8011f94b49eaa5fa5a60e1e2adccde4185a7d6c7f83088500b677f897d4ffc276016d614708488f407c01bd3ccf2be653269062cb97f8945a621d049277d19b1c248611f25d047038928d2efeb4323c402af4c19288c7b36911dc06639af5bb34367519b66c1f525bbd3828c12067c9c579aeeb4fb3ae0918125dc1dad5fd518019a5ae67894ce1a7f7bed1a591ba8edda2fdf4cd403761fd981fb1ea5eb0bf806f919350ee60cac16d0a39a491a4d79301781f95ea3870aea82e9946053537360b2fb415b18b61aed0af81d461ad6b923f10c0df79daddc4e279ff543a282bb3a37f9fa03238348b3dac51a453b04bced1f5bd318ddd829bdfe5f37abdbeda730e21441b818302f3c5c2c4d5657accfca4c53d7a80eb3db43946d38965be5f796b"]}'; - - ore_record = '("{""(\\""\\\\\\\\x1212121212125932e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea5c9fdb3cc34da8b152b995957591880c523beb1d3f12487c38d18f62dd26209a727674e5a5fe3a3e3037860839afd8011f94b49eaa5fa5a60e1e2adccde4185a7d6c7f83088500b677f897d4ffc276016d614708488f407c01bd3ccf2be653269062cb97f8945a621d049277d19b1c248611f25d047038928d2efeb4323c402af4c19288c7b36911dc06639af5bb34367519b66c1f525bbd3828c12067c9c579aeeb4fb3ae0918125dc1dad5fd518019a5ae67894ce1a7f7bed1a591ba8edda2fdf4cd403761fd981fb1ea5eb0bf806f919350ee60cac16d0a39a491a4d79301781f95ea3870aea82e9946053537360b2fb415b18b61aed0af81d461ad6b923f10c0df79daddc4e279ff543a282bb3a37f9fa03238348b3dac51a453b04bced1f5bd318ddd829bdfe5f37abdbeda730e21441b818302f3c5c2c4d5657accfca4c53d7a80eb3db43946d38965be5f796b\\"")""}")'; - - - -- SANITY CHECK - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE cs_ore_64_8_v1(name_encrypted) < cs_ore_64_8_v1(ore_json))); - - -- NULL VALUES SHOULD BE IGNORED - ASSERT (SELECT (SELECT COUNT(*) FROM (SELECT id FROM users WHERE cs_ore_64_8_v1(name_encrypted) < cs_ore_64_8_v1(ore_json)) as count) = 1); - - ASSERT (SELECT EXISTS ( - SELECT id FROM users WHERE name_encrypted < ore_cs_encrypted::jsonb - )); - - -- -- -- cs_encrypted_v1 < jsonb - ASSERT (SELECT EXISTS ( - SELECT id FROM users WHERE name_encrypted < ore_json::jsonb - )); - - -- -- -- jsonb < cs_encrypted_v1 - -- genrating ORE data for tests is fiddly, hence the IS FALSE here - ASSERT (SELECT EXISTS ( - SELECT id FROM users WHERE (ore_json::jsonb < name_encrypted) IS FALSE - )); - - -- -- -- -- cs_encrypted_v1 < ore_64_8_v1 - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted < ore_record::ore_64_8_v1)); - - -- -- -- -- -- ore_64_8_v1 < cs_encrypted_v1 - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE (ore_record::ore_64_8_v1 < name_encrypted) IS FALSE)); - - -- -- -- -- cs_encrypted_v1 <> cs_encrypted_v1 - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted < ore_cs_encrypted::cs_encrypted_v1)); - - END; -$$ LANGUAGE plpgsql; - - - --- ORE eq <= OPERATORS -DO $$ - DECLARE - ore_cs_encrypted cs_encrypted_v1; - ore_json jsonb; - ore_record text; - BEGIN - ore_cs_encrypted := '{ - "v": 1, - "k": "ct", - "c": "ciphertext", - "i": { - "t": "users", - "c": "name" - }, - "m": [1, 2, 3], - "o": ["1212121212125932e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea5c9fdb3cc34da8b152b995957591880c523beb1d3f12487c38d18f62dd26209a727674e5a5fe3a3e3037860839afd8011f94b49eaa5fa5a60e1e2adccde4185a7d6c7f83088500b677f897d4ffc276016d614708488f407c01bd3ccf2be653269062cb97f8945a621d049277d19b1c248611f25d047038928d2efeb4323c402af4c19288c7b36911dc06639af5bb34367519b66c1f525bbd3828c12067c9c579aeeb4fb3ae0918125dc1dad5fd518019a5ae67894ce1a7f7bed1a591ba8edda2fdf4cd403761fd981fb1ea5eb0bf806f919350ee60cac16d0a39a491a4d79301781f95ea3870aea82e9946053537360b2fb415b18b61aed0af81d461ad6b923f10c0df79daddc4e279ff543a282bb3a37f9fa03238348b3dac51a453b04bced1f5bd318ddd829bdfe5f37abdbeda730e21441b818302f3c5c2c4d5657accfca4c53d7a80eb3db43946d38965be5f796b"] - }'; - - ore_json := '{"o": ["1212121212125932e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea5c9fdb3cc34da8b152b995957591880c523beb1d3f12487c38d18f62dd26209a727674e5a5fe3a3e3037860839afd8011f94b49eaa5fa5a60e1e2adccde4185a7d6c7f83088500b677f897d4ffc276016d614708488f407c01bd3ccf2be653269062cb97f8945a621d049277d19b1c248611f25d047038928d2efeb4323c402af4c19288c7b36911dc06639af5bb34367519b66c1f525bbd3828c12067c9c579aeeb4fb3ae0918125dc1dad5fd518019a5ae67894ce1a7f7bed1a591ba8edda2fdf4cd403761fd981fb1ea5eb0bf806f919350ee60cac16d0a39a491a4d79301781f95ea3870aea82e9946053537360b2fb415b18b61aed0af81d461ad6b923f10c0df79daddc4e279ff543a282bb3a37f9fa03238348b3dac51a453b04bced1f5bd318ddd829bdfe5f37abdbeda730e21441b818302f3c5c2c4d5657accfca4c53d7a80eb3db43946d38965be5f796b"]}'; - - ore_record = '("{""(\\""\\\\\\\\x1212121212125932e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea5c9fdb3cc34da8b152b995957591880c523beb1d3f12487c38d18f62dd26209a727674e5a5fe3a3e3037860839afd8011f94b49eaa5fa5a60e1e2adccde4185a7d6c7f83088500b677f897d4ffc276016d614708488f407c01bd3ccf2be653269062cb97f8945a621d049277d19b1c248611f25d047038928d2efeb4323c402af4c19288c7b36911dc06639af5bb34367519b66c1f525bbd3828c12067c9c579aeeb4fb3ae0918125dc1dad5fd518019a5ae67894ce1a7f7bed1a591ba8edda2fdf4cd403761fd981fb1ea5eb0bf806f919350ee60cac16d0a39a491a4d79301781f95ea3870aea82e9946053537360b2fb415b18b61aed0af81d461ad6b923f10c0df79daddc4e279ff543a282bb3a37f9fa03238348b3dac51a453b04bced1f5bd318ddd829bdfe5f37abdbeda730e21441b818302f3c5c2c4d5657accfca4c53d7a80eb3db43946d38965be5f796b\\"")""}")'; - - - -- SANITY CHECK - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE cs_ore_64_8_v1(name_encrypted) <= cs_ore_64_8_v1(ore_json))); - - ASSERT (SELECT EXISTS ( - SELECT id FROM users WHERE name_encrypted <= ore_cs_encrypted::jsonb - )); - - -- -- -- cs_encrypted_v1 <= jsonb - ASSERT (SELECT EXISTS ( - SELECT id FROM users WHERE name_encrypted <= ore_json::jsonb - )); - - -- -- -- jsonb <= cs_encrypted_v1 - -- genrating ORE data for tests is fiddly, hence the IS FALSE here - ASSERT (SELECT EXISTS ( - SELECT id FROM users WHERE (ore_json::jsonb <= name_encrypted) IS FALSE - )); - - -- -- -- -- cs_encrypted_v1 <= ore_64_8_v1 - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted <= ore_record::ore_64_8_v1)); - - -- -- -- -- -- ore_64_8_v1 <= cs_encrypted_v1 - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE (ore_record::ore_64_8_v1 <= name_encrypted) IS FALSE)); - - -- -- -- -- cs_encrypted_v1 <= cs_encrypted_v1 - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted <= ore_cs_encrypted::cs_encrypted_v1)); - - END; -$$ LANGUAGE plpgsql; - - - --- User with "HIGH" value -INSERT INTO users (name_encrypted) VALUES ( - '{ - "v": 1, - "k": "ct", - "c": "ciphertext", - "i": { - "t": "users", - "c": "name" - }, - "m": [1, 2, 3], - "o": ["1212121212125932e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea5c9fdb3cc34da8b152b995957591880c523beb1d3f12487c38d18f62dd26209a727674e5a5fe3a3e3037860839afd8011f94b49eaa5fa5a60e1e2adccde4185a7d6c7f83088500b677f897d4ffc276016d614708488f407c01bd3ccf2be653269062cb97f8945a621d049277d19b1c248611f25d047038928d2efeb4323c402af4c19288c7b36911dc06639af5bb34367519b66c1f525bbd3828c12067c9c579aeeb4fb3ae0918125dc1dad5fd518019a5ae67894ce1a7f7bed1a591ba8edda2fdf4cd403761fd981fb1ea5eb0bf806f919350ee60cac16d0a39a491a4d79301781f95ea3870aea82e9946053537360b2fb415b18b61aed0af81d461ad6b923f10c0df79daddc4e279ff543a282bb3a37f9fa03238348b3dac51a453b04bced1f5bd318ddd829bdfe5f37abdbeda730e21441b818302f3c5c2c4d5657accfca4c53d7a80eb3db43946d38965be5f796b"] - }'::jsonb -); - - - --- ORE eq < OPERATORS -DO $$ - DECLARE - ore_cs_encrypted cs_encrypted_v1; - ore_json jsonb; - ore_record text; - BEGIN - ore_cs_encrypted := '{ - "v": 1, - "k": "ct", - "c": "ciphertext", - "i": { - "t": "users", - "c": "name" - }, - "m": [1, 2, 3], - "o": ["12121212121259bfe28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea5c9fdb3cc34da8b152b995957591880c523beb1d3f12487c38d18f62dd26209a727674e5a5fe3a3e3037860839afd801ff4a28b714e4cde8df10625dce72602fdbdcc53d515857f1119f5912804ce09c6cf6c2d37393a27a465134523b512664582f834e15003b7216cb668480bc3e7d1c069f2572ece7c848b9eb9a28b4e62bfc2b97c93e61b2054154e621c5bbb7bed37de3d7c343bd3dbcf7b4af20128c961351bf55910a855f08a8587c2059a5f05ca8d7a082e695b3dd4ff3ce86694d4fe98972220eea1ab90f5de493ef3a502b74a569f103ee2897ebc9ae9b16a17e7be67415ee830519beb3058ffc1c1eb0e574d66c8b365919f27eb00aa7bce475d7bdaad4ed800f8fc3d626e0eb842e312b0cc22a1ccf89847ebb2cd0a6e18aec21bd2deeec1c47301fc687f7f764bb882b50f553c246a6da5816b78b3530119ea68b08a8403a90e063e58502670563bd4d"] - }'; - - ore_json := '{"o": ["12121212121259bfe28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea5c9fdb3cc34da8b152b995957591880c523beb1d3f12487c38d18f62dd26209a727674e5a5fe3a3e3037860839afd801ff4a28b714e4cde8df10625dce72602fdbdcc53d515857f1119f5912804ce09c6cf6c2d37393a27a465134523b512664582f834e15003b7216cb668480bc3e7d1c069f2572ece7c848b9eb9a28b4e62bfc2b97c93e61b2054154e621c5bbb7bed37de3d7c343bd3dbcf7b4af20128c961351bf55910a855f08a8587c2059a5f05ca8d7a082e695b3dd4ff3ce86694d4fe98972220eea1ab90f5de493ef3a502b74a569f103ee2897ebc9ae9b16a17e7be67415ee830519beb3058ffc1c1eb0e574d66c8b365919f27eb00aa7bce475d7bdaad4ed800f8fc3d626e0eb842e312b0cc22a1ccf89847ebb2cd0a6e18aec21bd2deeec1c47301fc687f7f764bb882b50f553c246a6da5816b78b3530119ea68b08a8403a90e063e58502670563bd4d"]}'; - - - ore_record = '("{""(\\""\\\\\\\\x12121212121259bfe28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea5c9fdb3cc34da8b152b995957591880c523beb1d3f12487c38d18f62dd26209a727674e5a5fe3a3e3037860839afd801ff4a28b714e4cde8df10625dce72602fdbdcc53d515857f1119f5912804ce09c6cf6c2d37393a27a465134523b512664582f834e15003b7216cb668480bc3e7d1c069f2572ece7c848b9eb9a28b4e62bfc2b97c93e61b2054154e621c5bbb7bed37de3d7c343bd3dbcf7b4af20128c961351bf55910a855f08a8587c2059a5f05ca8d7a082e695b3dd4ff3ce86694d4fe98972220eea1ab90f5de493ef3a502b74a569f103ee2897ebc9ae9b16a17e7be67415ee830519beb3058ffc1c1eb0e574d66c8b365919f27eb00aa7bce475d7bdaad4ed800f8fc3d626e0eb842e312b0cc22a1ccf89847ebb2cd0a6e18aec21bd2deeec1c47301fc687f7f764bb882b50f553c246a6da5816b78b3530119ea68b08a8403a90e063e58502670563bd4d\\"")""}")'; - - -- SANITY CHECK - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE cs_ore_64_8_v1(name_encrypted) > cs_ore_64_8_v1(ore_json))); - - ASSERT (SELECT EXISTS ( - SELECT id FROM users WHERE name_encrypted > ore_cs_encrypted::jsonb - )); - - -- -- -- cs_encrypted_v1 > jsonb - ASSERT (SELECT EXISTS ( - SELECT id FROM users WHERE name_encrypted > ore_json::jsonb - )); - - -- -- -- jsonb > cs_encrypted_v1 - -- genrating ORE data for tests is fiddly, hence the IS FALSE here - ASSERT (SELECT EXISTS ( - SELECT id FROM users WHERE (ore_json::jsonb > name_encrypted) IS FALSE - )); - - -- -- -- -- cs_encrypted_v1 > ore_64_8_v1 - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted > ore_record::ore_64_8_v1)); - - -- -- -- -- -- ore_64_8_v1 > cs_encrypted_v1 - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE (ore_record::ore_64_8_v1 > name_encrypted) IS FALSE)); - - -- -- -- -- cs_encrypted_v1 >> cs_encrypted_v1 - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted > ore_cs_encrypted::cs_encrypted_v1)); - - END; -$$ LANGUAGE plpgsql; - - - - --- User with "HIGH" value -INSERT INTO users (name_encrypted) VALUES ( - '{ - "v": 1, - "k": "ct", - "c": "ciphertext", - "i": { - "t": "users", - "c": "name" - }, - "m": [1, 2, 3], - "o": ["1212121212125932e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea5c9fdb3cc34da8b152b995957591880c523beb1d3f12487c38d18f62dd26209a727674e5a5fe3a3e3037860839afd8011f94b49eaa5fa5a60e1e2adccde4185a7d6c7f83088500b677f897d4ffc276016d614708488f407c01bd3ccf2be653269062cb97f8945a621d049277d19b1c248611f25d047038928d2efeb4323c402af4c19288c7b36911dc06639af5bb34367519b66c1f525bbd3828c12067c9c579aeeb4fb3ae0918125dc1dad5fd518019a5ae67894ce1a7f7bed1a591ba8edda2fdf4cd403761fd981fb1ea5eb0bf806f919350ee60cac16d0a39a491a4d79301781f95ea3870aea82e9946053537360b2fb415b18b61aed0af81d461ad6b923f10c0df79daddc4e279ff543a282bb3a37f9fa03238348b3dac51a453b04bced1f5bd318ddd829bdfe5f37abdbeda730e21441b818302f3c5c2c4d5657accfca4c53d7a80eb3db43946d38965be5f796b"] - }'::jsonb -); - - - --- ORE eq >= OPERATORS -DO $$ - DECLARE - ore_cs_encrypted cs_encrypted_v1; - ore_json jsonb; - ore_record text; - BEGIN - ore_cs_encrypted := '{ - "v": 1, - "k": "ct", - "c": "ciphertext", - "i": { - "t": "users", - "c": "name" - }, - "m": [1, 2, 3], - "o": ["12121212121259bfe28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea5c9fdb3cc34da8b152b995957591880c523beb1d3f12487c38d18f62dd26209a727674e5a5fe3a3e3037860839afd801ff4a28b714e4cde8df10625dce72602fdbdcc53d515857f1119f5912804ce09c6cf6c2d37393a27a465134523b512664582f834e15003b7216cb668480bc3e7d1c069f2572ece7c848b9eb9a28b4e62bfc2b97c93e61b2054154e621c5bbb7bed37de3d7c343bd3dbcf7b4af20128c961351bf55910a855f08a8587c2059a5f05ca8d7a082e695b3dd4ff3ce86694d4fe98972220eea1ab90f5de493ef3a502b74a569f103ee2897ebc9ae9b16a17e7be67415ee830519beb3058ffc1c1eb0e574d66c8b365919f27eb00aa7bce475d7bdaad4ed800f8fc3d626e0eb842e312b0cc22a1ccf89847ebb2cd0a6e18aec21bd2deeec1c47301fc687f7f764bb882b50f553c246a6da5816b78b3530119ea68b08a8403a90e063e58502670563bd4d"] - }'; - - ore_json := '{"o": ["12121212121259bfe28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea5c9fdb3cc34da8b152b995957591880c523beb1d3f12487c38d18f62dd26209a727674e5a5fe3a3e3037860839afd801ff4a28b714e4cde8df10625dce72602fdbdcc53d515857f1119f5912804ce09c6cf6c2d37393a27a465134523b512664582f834e15003b7216cb668480bc3e7d1c069f2572ece7c848b9eb9a28b4e62bfc2b97c93e61b2054154e621c5bbb7bed37de3d7c343bd3dbcf7b4af20128c961351bf55910a855f08a8587c2059a5f05ca8d7a082e695b3dd4ff3ce86694d4fe98972220eea1ab90f5de493ef3a502b74a569f103ee2897ebc9ae9b16a17e7be67415ee830519beb3058ffc1c1eb0e574d66c8b365919f27eb00aa7bce475d7bdaad4ed800f8fc3d626e0eb842e312b0cc22a1ccf89847ebb2cd0a6e18aec21bd2deeec1c47301fc687f7f764bb882b50f553c246a6da5816b78b3530119ea68b08a8403a90e063e58502670563bd4d"]}'; - - - ore_record = '("{""(\\""\\\\\\\\x12121212121259bfe28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea5c9fdb3cc34da8b152b995957591880c523beb1d3f12487c38d18f62dd26209a727674e5a5fe3a3e3037860839afd801ff4a28b714e4cde8df10625dce72602fdbdcc53d515857f1119f5912804ce09c6cf6c2d37393a27a465134523b512664582f834e15003b7216cb668480bc3e7d1c069f2572ece7c848b9eb9a28b4e62bfc2b97c93e61b2054154e621c5bbb7bed37de3d7c343bd3dbcf7b4af20128c961351bf55910a855f08a8587c2059a5f05ca8d7a082e695b3dd4ff3ce86694d4fe98972220eea1ab90f5de493ef3a502b74a569f103ee2897ebc9ae9b16a17e7be67415ee830519beb3058ffc1c1eb0e574d66c8b365919f27eb00aa7bce475d7bdaad4ed800f8fc3d626e0eb842e312b0cc22a1ccf89847ebb2cd0a6e18aec21bd2deeec1c47301fc687f7f764bb882b50f553c246a6da5816b78b3530119ea68b08a8403a90e063e58502670563bd4d\\"")""}")'; - - -- SANITY CHECK - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE cs_ore_64_8_v1(name_encrypted) >= cs_ore_64_8_v1(ore_json))); - - ASSERT (SELECT EXISTS ( - SELECT id FROM users WHERE name_encrypted >= ore_cs_encrypted::jsonb - )); - - -- -- -- cs_encrypted_v1 >= jsonb - ASSERT (SELECT EXISTS ( - SELECT id FROM users WHERE name_encrypted >= ore_json::jsonb - )); - - -- -- -- jsonb >= cs_encrypted_v1 - -- genrating ORE data for tests is fiddly, hence the IS FALSE here - ASSERT (SELECT EXISTS ( - SELECT id FROM users WHERE (ore_json::jsonb >= name_encrypted) IS FALSE - )); - - -- -- -- -- cs_encrypted_v1 >= ore_64_8_v1 - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted >= ore_record::ore_64_8_v1)); - - -- -- -- -- -- ore_64_8_v1 >= cs_encrypted_v1 - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE (ore_record::ore_64_8_v1 >= name_encrypted) IS FALSE)); - - -- -- -- -- cs_encrypted_v1 >= cs_encrypted_v1 - ASSERT (SELECT EXISTS (SELECT id FROM users WHERE name_encrypted >= ore_cs_encrypted::cs_encrypted_v1)); - - END; -$$ LANGUAGE plpgsql; diff --git a/tests/ore.sql b/tests/ore.sql new file mode 100644 index 00000000..4b70c8a9 --- /dev/null +++ b/tests/ore.sql @@ -0,0 +1,107 @@ +DROP TABLE IF EXISTS ore; +CREATE TABLE ore +( + id bigint, + e eql_v2_encrypted, + PRIMARY KEY(id) +); + +INSERT INTO ore(id, e) VALUES (1, '{"o": ["12121212595a5ac3e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608fd5926b0ea7f04cca1b87b0519a2e2a4d00f609e27862073387f46123dffa5bbdb4e77437f92bdcd757d85ef4430f9c2c7296402d5d5487c048b63cced26aab9381e94204f0046a1264343e482a98e9d262cfee1f88a6d213c930761f0869b9286bd9f3844d8ea8cbfdfe6a5b5f49178a4454b0662ccf83f13bbb42fb9541f73dc9166e720d793ab3821ce3690abfe8f5e2cb9c60f1ff10868246e3e7e5c33838ffd28876219c64819022a7d2a90184f29cddf68fb7f7da4d3003a1499863651f90f950f1cc3d1ce8e392f7c06d9760a96301de48f8e0ad785fa35e31b55cf114a3ac1b6cbbae2da0639524fce7774ea504898bb801304f7a5489663b673d74a51ad7a74a22837fcacbded4402444c38846715e54df923927d19f0111e623d7f"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (2, '{"o": ["12121212595a5a4be28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608939c5758ae24a6e713702cb3ced9da7d0cf0712b96032c1bacbbd8266112ee4aa6dfc36d439ffcba1ea6a25056ba409c5099a9552f02352694e6728d2e3e33865d9ebfef3a0321f5f85fa31addfe73cc6f3f60b3668f05aa57188dc8d48750b6039403541e96c511f90a35708d0d572d3c6c85bc54d3dfb10d1840700163d1dfd41a65bd14df7217c9a3b6e8a8ef1970074056182ff1b11190ed3ef0340aae45e9c5ce871b7fa392b46296e1bd5d0c1d2978fb3590eb252bcd1af19f13a2bb91d7045b02b61ff1b2930906740a1be374cd2ad984e01b80c442e81a1a9a05f44e0a5d5e0a5a70075aafed8e3fb1c11d144a89ff1452010203a43eef37f1678e7e5ad882f294136cd6c5ee20d77b8ed926bb5fa5e59f47b59e550405c6b81a87d2"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (3, '{"o": ["12121212595a5ac2e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac60802e63b60e032e28b0ecf7edce63d50883ae1e5242fd8afc3075a6f4a8cac76a9e6da2b4916c55639902dd78a1de1ba7a25ce1889f015f3dbd1742fdb4ee988ce8096d17eb589b60caa209787c2ffb165ca154326c55626bacc2ea31061d00fc8d16823d513ca18b1b99baae15c9cc19274245436d3e55fa27dbb527d1c0d52129cc0e854d3b81fcfb716276ef4069980de3275ad2780bdb5f6c53fdc4ae0c3fbf571a5c8039109c8175f7198eb1f6efcba06d88a2e254ea9073b2bf354646defdb9694e37ed19d46ae8ef242ed7f894602f7244146ad293366f1164a235c4db0b9ae7bdb7f366622fb2f0a1c81184aafcb2bb12312b9310c150b435fe6d1401a0e92ebba9b38ecbba83823e4e40ebff0d0d894177987c4b84dc1554f22bc0506"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (4, '{"o": ["12121212595a5abee28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6085024c4dcf4b0e1ab322977a6162fa77b014ce230c08eebe043bc9255ea60fb55c262ee12501d1098d1a8bad015e9ece90b09544664e8cca25ddb4bb320eba1acc7ef54faefe1177230aa39851259e6291b9a8145d2a4702162afcddfdb50753984bdf888e7d558ad443687da9a4dfacb465a7338f9f75c9744853e92e911637b084f4f145fe28270b8cbf37ed9b30795273574294bd53189016437f1fb87597411cece7f6129f4df930a50bd2986c1d54e33f152775107d6189652e1e63446e04fbe36b3122fa55bf92944a91f659790386e6664f50c5b174667025b421001ba2f7fe33eacf0ee51c84e4075718baa277aa938366049c5485b71748b1adb1f84e07e7cba2742c94df09e412994373fd8e931bc5d199da98200a57151b9bbeb06"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (5, '{"o": ["12121212595a5af5e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac60892d84cf8720a2c385989fdbead27bf30530fa2d6712dbc175c7a88dc0c2af165331a3acda8907bed40bb8c35bd0472d3f6565e1710dd9f96e039191928f67b193418e96e605d1232a3536f1a8115c69ae6b61a7f58fe2aabd15eb6d30a44c009afe8237dca80204329de0fff7bd162c449010cb7954cc18607da440ac3fb8b96825f3b1ca88dda889fe2ea710ae2f5518946e3f21f39e44ed067e4e0269f0e5ceb82121e2b05e4418b4492fa291e7450915bdbcb9d30f0bf3b858174c90c5139963b391a85b17909da824b8a8205cd5340ff4faa804e16546654007398047d13bd25bd95767c7875c974558a78d05e84042d0c054f5a41c114d8b5b26a7332736c30ed4e99b87b65aa6cf9beebd48a98937c1e557617eb869549c2f9a301d2bc"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (6, '{"o": ["12121212595a5a73e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac60812d3458204dd27f84bc377ec78b6773cc5228806d0f5ef7951f195e9c7d6bccfda79782b18115450414e098473642023d4b9d2026b073b77760b881ee97098836b45000cd6f2ab15596d80d592c282e3eb792ec5029db5bdb619125a27c01dca1e2e9fe5fb5472dc490dcf24f27634963ad891c90608c647b47c669dfb359669af44b081efe69ffd8c1d2881254d0d2cc2dc57b3baed483d1559c55d042453c7e987df535d3e31fb65f5ea9efc2adb8136e52e8ee68ab3974fe0f365b5a60838ef45bea5a8638e879fa353e09318dc4131f73f1a0e5bb18a4de93dd308b946c47ce2522738e7b04bf2bc099fbbba8d859917383f9ba7fcd2aaba37af8466fd06f2ce8cd033e96ab2658a12b7c029c8f875331dcf7e98f459c9582e46d0bb4102"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (7, '{"o": ["12121212595a5a44e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608265136512346b7f35fa6101d960029053ba709bcf2aa369a1ef42ff3c98ddfaa2bdaf0f60314ab54ae6faeb6595565c16c465e6ed0021c6e4b5443d3864019cd8cf3a1b16e663d79b2b4e977a46500026337b098cb4e050eac7b269f45fe23acb89104fce7278bc7bd96ca1b9a128c7cc762f984ff17b6be2f2a6c42b3f9f493c39523994530b7669f1512d0e6ff37495aa676440fc4edc51757ccb6b31b2600e05eff8861f55fd1e0569101dec4128c36d7aaacc40eb1c96c6c022f549ac63bb717511945e485231f1dafc3390c2785c4c9d6f9af5e3727c1c0070e1e0083dfa68a3c8e4104b9f2f848a1cf59a38430bca312483e53fb8804cac9bbd30128e1d196b53bc1426d8599a327c7173e44a32ff641d8f3d78045aa27e6d386462e65"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (8, '{"o": ["12121212595a5a81e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac60877cb19e4f4530ba87cc080344273602160e88fdc19c7a3e94e9fdc37cc831edeca205568817fa95044739cfcb42c96347ceeab68b13563b4ae72e4185acb320e09b783d4ad09848c1a6e313fa5a1d2ede1c1441d4f1605af2187be51a0846889c0c37cbf421c9693d60beaf8b09f9c39aad3d067ddcc14ed0b19c15f1d30d3f3f0ed1a8590d16ee313c77c4abefe03edd0f70034e9b3348dbfc33f127521d5292e24a2be53cd90523c93ac15f5914dddab9e35f25487aaf949f7e95c4794c7715a4e8112230d5889a7ad9f6ead13443f3351bd86061d66d288447477274573c5c1626e0e7a9321eac9b7c7f7d8a13c7d8379cfe0c5b2e1cc349449923a1a15ab46e05a4dd402ac228d2d4ba613fbf828f900d7bf8d2c22c8a61e25c1c8b8fcf0"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (9, '{"o": ["12121212595a5af8e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6083073a4e90950b588083bb498080cf13ebfdb7c7144f39b5e6d1a632e85048ee5c3572676cd9cda6fe1a69247ec1d5228955a8b9812518e233bc5a5b27cfb82e46b9bd3310907dcd9ba617c738db7ddb9ac1c41164f9c46aa02733c8337a016b7fe40b6c459d567f8758182853494451a467f47f4fa5a2dee4b892833c8a5bf0617f28f64cb7ba887d67aab62dfbada70c0e9def4d5175551ad1f404c099bb3e81d38a91f441426cb485652c8fb33ed204ea51961a544cfd8778680988faa9527701148b8991c5bc16340284e584682d9d34fc68c8e6c73cf0902fa335da9b77e0c9315717c7aeece71c5b792d1ed612014d7e43e19c4b9f1d71062531597ad28cb2b5ec67f15bf09406554eb4e54a0bcbaa65518f32440d4299675b0184aab14"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (10, '{"o":["12121212595a5a55e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6084c991b192cc668d5f7805e2716b7a35a504a59e094e2422e64a083d629a56b9305d1de9b906cf439e7f7f373ef5f2c000fd0c50989b93409d431ab9465fe7c468fddb9bf9df86f2ea52bf6370c96f124d7fd48cf60b28161ac0dd22c2653a1ada262f352e857791de914238a32e7305d797a3e58f0f5d7632fb4543e08143615c8dbf6cdbb6fbe5e242f59cbd70a3f514c2a8ec6cca5c95f1745d9e583f11a24378bd91ef3d9184322f38227f4951b51684ca19bc719b0513343b58971afcf8f4b6c6a015d182c5fc1aa58884a45ab0b8176266bcd7dda0b10d93bd239d4306748d75f666a40ec7d1cfdc3a476ea960ac1bf3434633caf61d663540ae76588d7a7bc72656981d167c04c810223a08c6e83f6943ac4b43f8276f09566311c56ed"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (11, '{"o":["12121212595a5aade28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac60812ae946d007186aa28d5df568c18a26f4eb91f26e138532c4cae2e5f4f8e5f0a8b2c7847a9689afdb4d722eae00987bc8010c068c5aafd5d18710c8b55c5aac15cc0eceea8ca4d4f4ce2b5977d75571dfbcf63f1dc86e746c906288ca8c7fc9284060c75c3b8996b8bca1337733af315dfd612016428d04b9ea7da8e39c4f04e2d6c4b6f8654d0ff95686f5d0cb355782a9ead6d446d4510d492a52ff04571d156f75e55e28913c2bcc20ff1e74ecf445b255706d701653f72e1b709234fa8b771f098803eccce5931733d4010f7c6bb681562ed05e34b751ecc6411cd802cf73c77aca459553d9931bed592919a99892afbf22a1c3874a4664f4beade8f7c5117854328a7302db1b21624c0a1ba416a7d556b95b22e17608530ddf2cc0817ad"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (12, '{"o":["12121212595a5a37e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6083964fa917aaefe83f03b14350aecc5aa62fa253cb96d4c841064a995ff5f742ef6523a773ee0321a619d444d17dc457d8ab76763c2c136f70d225355574ded0e205bf8b115ce7c88516a9f83dcf69cfe2a717a109d7c6a7bc31dcecf2a2d61be17cca9e2d101b37504fcfaf815bab890faeb230df5f8dfb39cd0b6d94456ebac59bb5bdc84d264e89d53571d25fed202f25f2bd9403a840d7697bb3a746053f58899551f084bbca57ba0cdd2676d0816dd45cf2ebfb239bf6bfe5628a81bb792c3f27937ea53cc3a3140bb91f7982e604a0bc5a31dff2d571f040911764579b3580687e33dad2f6efc5a281ca75ffa710e24200a7ffb54c79f5ed74d1bca52c1d431a687338fa5738ca2a74ab115dddc1084f710ef19bf093d604270bf593772"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (13, '{"o":["12121212595a5a7de28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6089bee44ee735c9a1930b465d2532ace2a4a528273b643ba43c76f34f2c9f88a9123adaa86e9b8ee7db501c3dfa023d8caed8b4bf015937c27170c623c32e4f834eff482d9d844cce3274d4d69c9d6042d19699c006e0e7199029549fc14fb4594a5e3390a134b7a00825ce318219e5e70ace9cba634cef967dcacc2a4b2fe8154ec3bddb96526821154598d05e1f5cd1b7f974d3b213427b4ed073051de3a023a2af4a1c355a1e8ddc635779863b5b8f98af8a2a35bceed8faf4b0386e9f7eba0792e99e214b5247a5c70c11caa238392984212381927e8cbc2602cf03195dc2e5c12590e5e8d14fd25efcb440d240fb7971623413a84bf4c95efc02f0826c6498a73ed3654789eb0400f04473d2f19cc029114ed39e1e8d0b2253a74b41fe599"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (14, '{"o":["12121212595a5a4de28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6088db498842d2fe4b5bb5ff5bfbfabe3d05568f68642093f332e409a596866676f09c8254553fcb551b836aa3747abccc41a29e106a124cdf4683a18a6d5ccc8d44c44a5506d28a49bbdffda9ab3f99b6a67f009858e182584e5c5cdf65dc7656ec17724d1b6ed7396316b4c0496f023f9584a1174e40ef628790399dc46caf1417745a2fbe274543a2eeb43ec8c399ccdeb57b9e90856f7f8edcba92d5e2b0c88da4f5fa34f6e694c1f9537b16993b10262c57db87fd6c37d0449731daae7d02042bf24da6b6aef3212e6565f8145b289550ac1c0ddc6c4e933549cd9ecf54011af462624f1b2d3de00b3c33b3b252577cf801f6e076e9b56d65099d28c728e63547a8e944a6c01f492fa165246c4de2b8de923568734a8d06328fb9aaba4580e"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (15, '{"o":["12121212595a5afee28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608dd110843cd5b367ee9b253db9b4ab9a8f2dd146bf38210b85055dd4d93cad8688581c0e32f90da216faa28b3db6c48b83d9b0fca3013d65257a60344b25bfa45bfe1346fb73b547a16fdfbbaed41e1a9ded261814746c82c82f84e9353aa28105312923e56831a8ce83da2f2fc7e56f6bb7d0fb6b467365f6a79131eb708779a8f19dac72365e197199bacf324f4d75c067ba77aa009802bbee2e2bcafd6ee22015b4ebb97cbd9f369c47f4c0aa276563bab26e44a99c29a2cdb14734fc0d1d43be7a2aeff73aeed83e9bf97b2c70250e8403c3b9cff8ffd76b9c1be25b6114ee3c34a242abca1be5a2a2e31708945b5d66d70717dc2e0899c3ec505ba07503e5f8b6809ae7d59ef1ff4e9d203ca3c03b13202ea5b28b050bcd5b6f582c5f7dd"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (16, '{"o":["12121212595a5af7e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608bbcc86e718c5e06fdbc881a9c4aa5d4f2b2562e92c71d26e16c86f5b3d1c2a5da7ad5344dd7f7af19dbb1ef27209e0b3e20be19003c72b259085fc8e234f84a245b38a48d5b5debb0c26e05e27d0035e26e3a54333350796c1848ac5a21c569b6d9fabc34a1375853ee44be2a471b54a50ff1d68ab19f113a1e7ae4225f00bcaac2af828fc770849aab7b05feab9b83b3ec2efed57de4e351ab6a4998e28670c17d23dbd0af60878efc4c7144838dde5149c86b78610d1870b2cfba9e4bf5480ba802c758e815499b2f02853fe839f6821acafb42523dddad7b819902c8a67f2635628c7d2cbd9eff5422eadbbdf5bf89663ee7615c3deffe8bf1bdeb8fb7b113507039f9ff9092e382077bf87d964ea7f08738cf40b2fdf61eca35dbd130be0"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (17, '{"o":["12121212595a5ab2e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608f9cf8ecf6c5553cfc040b0951875d97fe1dfa058177c3af3dad91b1d08c6ac10221852c5fd9296a6fc69b06ba7db4da7ee857d547dda2d640b43d500a16f765f7f3c09e7d6f7ab4470354af8e7553dc9eca23a662d2dd7f8a9c5b4518cbc511feb5551b14fe1dcb5b70921d9fa73504fc83969cb6e20ff2b0cf22ade7cc65c5874b1a5c6ca48d958bd6bd8dc7b09780146634b76854901539cd18644b77663fdf6a2c215c82e6a5eb9fe4beb4cabc710f791e9b9cc9c4d271ed3e9e098862ca0ac248946f9baaa5838e56dc48064214d4511234d6cbd0dccd7b48d64dcba0f28c79373e7404038c47ee4b7e760191359d0f059bccabd41913dd3e089d8d193b01eaac0410c4e0c2710c7db9cd67acacf4acec63387f08a61c813409c4b78e90b"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (18, '{"o":["12121212595a5a9be28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608f44342f45c45d5fc9772a281caa03d7b99a17ddd05559606e5681b8f6524956c3f0d44a12e6470703f825090ac60060b50bac7737946f0a5acd2e7f61690cac3c7a72c26903f0cf9283b3b8359338cbebb006d110f6df2a9fd7b55f5d3af36be148472e28bad04c209a8d17690e3996ac69cd927f65ae461b0573fcd1105d060f7f085044a1a5d735389772977625466647f35491718a525a0bcfeca2d4b52fdcee17aed656f8c2d3af16f9c8e46dff6dbf16092e4b96f0ede4352b5cbc7d835ff11cb5774badd0693fd206ee47bee0e62a8ab31be17ed1197f60e8752080c4eb8f10b12901b3e5d2391d89da58d609fb8ed987e14d0a9b02b3b9751cad9a493784d52c0caf3b195e4a6c255ace3377b24b66b5761e3f8dad7e7b4adedf636e2"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (19, '{"o":["12121212595a5a35e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac60812711d062a62191633ac00869aa9e67ea37858b349f72f830ffa67febfea3a642845b1d9cd136319aa725db4f136f6c83b8723901a899864bec8a040b37e491ea0ea5b2a831656e4825168937b9c3f3d4d6280728ffe6e7ec2abeea39ffc0367575cc7eb4e92c6feb1452d5eba9cd7464ee97fd7443f07d2c2efe222a9d8041ad3ab5a660a3cfc066b5b4b23d89fd5fd082448c6f17ddc2044e2e00e9effe30840e4affe52d3864af0ea6158696a9b55f811de218840b19d086e66a8ce27fc7ce5e26419f4f42d296d818ccada04104cb357b59f81a3d2920d005b79c58459efc9856f46f51b65ce127a5837238987260800399737dc5cfc6974dcd92ffff61afd9fef9328034998fc58d6b115fee98c5dca80396ed8fc6b06ffb8826a08aff5"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (20, '{"o":["12121212595a5a49e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608ab3b5551aa688cfa7bade03123724bd6b5fb0d25a13e70570039781e56498c41ccd9e2e9c8cdc2a54b017177c7a5baef62e5b9e354bd7b88f92bf4f73e799fc4ddafd55a1ae1ead63eef174b0bc21bf6c913cac1ae18f1f7f7d3dacfc8ebd1e1a8e1393bbc82145e5004a531a09232d91e21b4e99a020ed2d3764fdca288fd417b242d4c43153abfb67527c4e6e347ced2038ad7632ea360ab0633e8dc656627d2889c57f63baa98e4ba26837b934f82d5a60d2fea0729fe45e6b785624e1d6561fb6a91492e39a9881c27803a58770cdf87b98feff0917052e41547ddccd64adb3d17bafa1cf86d956106135b5b9b7c342669fa545750f654158a3e962d0a262a0ca7bd1f50629161a8c92ae5b81e3e51a893b7fdd042e00abd0fbcf2858788"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (21, '{"o":["12121212595a5af0e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac60804abc5a60dbd4ed7568553dfcb85dc84cfcc41b2912a6d30574354ae52b3c17ccf4b5eac4f93134d4893399491b782400178bd8a05c5def9aab7e864d43a5f3adfd1f62e2ea5b89e5910e0686e66ec81cfd52e4e85f7e2daa6b4e458cd0ee9cd67b76902f73f1fb059ea40300e1e56c06002d0d0af9d9565c5e0ae44ae399d3290e28ab04ead6bee3dd02b1f1694e4e771cff29b0d423e5bbfda44838e49010174be93f8f50570156820acf661d53a9321db37581f0b0923c4350cd3e855ce647cbac349fe682e0691266a95f70ddffa49a5a8023896c9a94ccef3499689e92b7fd7c38819ffc76b36dd54b44fb32daf2469a8db124385afb04063fc8edf1a81d60d96f7289d4797027c2a75bc2f7106a6f9be2d0c545dc1c47786b7a0cd61b1"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (22, '{"o":["12121212595a5a21e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6085d8adec3b41473d5a69f9234af42d4903fe0a4f67bc0654f21a73d332b8851ccc40029df24c62e3524f06eaa1d3b8ad01f4804f51a1f8106d065813e532adf030b21d532239bb4a82d8446fe9c6f35f4a7037d97dc7221f29e5b33ceb6986d52662a9063d7574305d343911454eea629d4a826b12bc936b0ae9d58b089cdf64794d25cbbbca9a56f2168fd94b371bb03a59bfe3f0182c1923fdef07d150caa94da48cad9dddbbc60c3478905c48115f0d87cac5da0444f3c20a69c39cd11d1d0c7ce469f725bc01be0de062b1caad2ab3899cfb68c9fa09ec87fdf4eea18ad440e2d7e09a2d6be994d60a42e2caae1b35ae5362367f8eb3dc48540ce4fb41ac4d645bf0cc34bb79904e6de0c63df82b06d33d96aa83aa9befded52bb01888d48"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (23, '{"o":["12121212595a5a7fe28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac60812a235a3345ccb72a1be3d1a985f488653445143ea3d127be0ebbc3b7b0c347ab87a9dbcfaba249e5e3ccc5815cf57d187b6e32b3d356500280877352a991f88c4111dbaac7b840c7ac8ee63ced77ec19cf97a940204e7284bb9da933c0e1bb66db03b557f0722f880856a945944c20834cf377e2d4b18cd52abad1a40367dc79d97f7862eda7f50296996242e0353b3ed346f091f4bc160d408ffbb59cb29fcebd0118df969f53642444983c64a88790f523248a7f3b609e21baca3460bf5448264dccd4d52a7ad70300491efc6ef9d676a0690ac2f3ac8349357d327bc986d0da033458884fd1eceb818bbf1b03e2fdaee3c5abf68de14820111df2e1591f69f20fff47ba873a9e84ddfd8e6b9a3c80e5c0b25514299ab7da85cda7685895c"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (24, '{"o":["12121212595a5acae28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608e04d89feef36edb385d8b267270ec8d69bed51f3c47b9a12a8049378f5e6f72a283c6b60f09e7cef270f700e8e0787f4e91e4d276cd8ce5fcd7f793d0c1a87ec4b4fbe12ab46beb3265cf3ebf736ed030639ddf959947eef38c7fb5b0e99844a697ec8ed0468d07e5a287dd480b9a299998bbb40eb5ee737024d3a983326c1cedac42ce3f9e889f190658def7ebd2d0677da0032733f0f9d7f3c9012ce91a7bf396dd1254229f9dc7af917c653262d9fca2fa1547faf0d4542efddddf27f195e71e1c9a18e01d80bc8b9e513b27b2ff35da30a7457ab767cbe52ec0cb0e53563959bf37ae4bd762db7dee3e026fcf14ce5818866ad495d0e316db41e1cf5ca6b57a1cf76dba2140787c55d30508dd3aa784958cacc06d106c0ef7839554e2696"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (25, '{"o":["12121212595a5a3ee28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6081934233eb14cba68036b3084ea3ff86f79d616697d2946c16e1c696db7b192993794b0571c250fd0f808e094504841a3fe8b33a278b35ffa3fc9233da6c1770326f9b97cee04e4e33db80a84e552875c59e7189dcee7b8397b7070e4090aaa85fedb0dd782f63f00019ef9d9915b3a58911e7f4aeac520c46691cf6cec77169cf49910124a0eca57ef1eaa45273546ca66d18835f624c8372dca1441fe911ac73c33d436f3ed26611f75b81923adc8847bd209d3693c990a75072b12265928ab17fc8c35f861d24438af3ffc546c50e633bc0fe16385fda3186be686516919b018d11ccfccd67f41f6cd626b76499c3432445dd39d2e84b700883969e73316e3e69867b775e192e7d49ec286a3c9656e7954e9dd3540ae9e14da0e365033b833"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (26, '{"o":["12121212595a5a52e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6087f88ed0cf2524182b7582562f7f1951f87a91569375806210e462c6c1c60182aff137e1a919694db53cb38e75948d6edc9bd925ed6bdf3c05ae0f729770282a2f156a2b7b97b22de8298140efdee854bc77a7f9d8b07d992346e42babcaa1fd0268b5fb50fd14d1d155b52f8f0f9e15868b0f7b6d3a2081e3b9c59976c602567bfd94d2b7f1edf6a54578687bdd8eee6908c06e83f4028d63996112a7f5d92ba03590c2bf45eb2a1d2f23c61da46768f23f5e5069f342f88c246d1e443c5cbf4f01aa4119021dd33dfbb3278e00a612074acbfe582e45bc377a82bc498a00fd388da52270e23980e8839543463de0c35d28033104eff8b7f312cc214fffa454eec455b4512bbfaa40ceeab04d5959dd18dbd662083f37ccdd0f2269d39e338c9"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (27, '{"o":["12121212595a5a36e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6083432f74ee724a3f43ca64fb6a8f07cef2f6cd1e218247421a8dc015790efacb13c5b5bd215870cf88ab2af31bc36e2145fc85087cd15ea13e87533ff67f33206a3a5a839d7ceb47599232e30d97064fbcc7b9685a7b9cc205db204a97d3b02ffec703a9a862883689aa84e8df04064773d3f8be85a50c560be2fb1f93a70f1171776c017e21cc80b03d3987e941efada62096588b7f2ebeef81642e8d81dc49b1ed4cd84ae20d880b320dac7b1e98557a5c3878e11ea308be4d4c1d4a5f00da54bff65c028eaa47f680b8beede529aca24f68b7c37b8dd67d34e8918201b7a28dcfc5ae8392ac46278107d697eaf181b18c0267979341f655adbeb9e0b7885d1094159e23deac3aa0c624a4b6fa14d9c16fa57bb84390c81d33583460cd123d2"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (28, '{"o":["12121212595a5a08e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6084d254beb0a03b41e20da2d30e75ff1698ca5b9875ab4342042d34da848b0efff95d18d406976b04e5339b00a3c39ba0ac31d0a5323f44a477f45ff65f5657d664500f7fdf8472c3a287567acfddb576871eee993219cb61e413052fe09278a6446983f395b67da5e133ec7b68854aa0a648df5f54e7990d8decd063192f3fc67bd83781b9558dd9cd3c4ecf068ca26aeadac1428eedd80905994ad6c4f56a0d6065323992da1cd1d231b2bc08b54faecdfe2858a0f03b35abe2758630d5dcc3817e4ab14832ca926a1aa9d2701c73c945a24169f48b4a39990a1c8e8daf1ac0598be830286615765aae46880f7011b6eba2ea5ec5e612f867452fcd948c5d6857a365e575b887e2800fbc44a9bbd0fa7c8ac11c5b42556c65297098040af427b"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (29, '{"o":["12121212595a5ae2e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608a89a05423b5f713668b7dd8741e4f0ff876e6493792d4119304f7e6d781165a2422c59c11a9925276b8f0c99822c12472b496a74d9ffad1b68ce38d97bd2675e555ae5c23132b2d89e51f8d1686dad326c5ede29c7f353d371950e9ea38a74fe3ecaedefe9b638b9c4506616c633a0b1d82a8a0517d5c0119c584460a0fab290cb6808bae3ab39f46b990c3a28e336361987970538f06eacf1d7904de1242a6cf8262a77b9ad391f9bcb41c24eebe75c281ceeb0a3227731c47c5650a249722d7744204a27deff9f427d5a1690f0e2d5ed509bf7759ab2ec126aa98c67284ff69ce5036d2cd68ffec9accde76ae0ddbbcb89cf8d06b090461f3f599deeb8876a443d7b2039af7830c249282cb59532bb9e5518338d393f3a54eedc750effa82b"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (30, '{"o":["12121212595a5a5ce28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6089f846a3875eee65df34b717fa4e68b4ef6761796a66a65f602d92224519465a1860670aef7224a48a645df3b304bf80f984536ea27120f5a92c0f95eccb74a8e3d0e46503789b2f2357c05a1b4cc7acf5140cd599b8e02d998c8a726b417c580565fed0d85f085bb3717727ce837d6f92cdf8e41ad4bfccc2b3e3b7c9bb91703b9e71e3c9723585e1f931135e5f2b56435a7c47ae871052cdfc2ec3b8620e9421409c241b744c604d38df37bb88ab4615c70cfb42efe4364e1a17b93c103b024d3ce26ce9c80804e3fcc8291ad0998ba7d19894d3b647af8763caecd51333d4f0b86154c12dab52af33925dfd6f59fa03b9d875ca1ce861d03a8abc05c26c652d827a0cba57c50513577fd691306ac6e8dfdb30e4c5b6be5365b8e0bc70128f6"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (31, '{"o":["12121212595a5a17e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608d6807bc98560de6edc0b849cb93b0cf3306939211bdedbe829ca1d35b534072ccb79bea416ccfadebf4387a2947134c991c60d9ab966935d289900e73cdee1cfd77a51dd7562eb039796f88fdabb84b3e419b3fd07a49bf64f15fda048d0d2e2bfb9009863208adad05a6af385fe9d315924787bf5ddbfc7916ef0294eae9ca836047891e325b69a08ebe9176ad55ea773d332acc985b3ae10b95fa00f2414284d8fde47a51e3d2edcb6c3aa8b96c6d507090756e789414f1285a88c2d7e31baca4fafd9ff60af1145168e0362a5012eba145e3f9ba489883dba2079ff422a54234da1934c243bd0b4bca72909bd6e1b827812f8d2500c5433877d9b9699d58f53f8bf2b595d09878ded5fae3193fab51632cca4ef5384f597096faf6fe73106"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (32, '{"o":["12121212595a5a85e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608cd0b93a2679cb66192a03971d07f1498144ec06f9a5e750e897579f6e011726d4a861fb20cae060be40c2b91be5fd8543918ccfd1916dbcb57765739b7363e085541c7ccfd7e57ae8ca95ada3ae02a48a91cd32c3e5de5321ca916d5bc47e7f4b2b71dca62bcd06302925f4d92011a5c317941e18424330d8bba352857ad7ca72193412ea2e8ee88b1a60d31482b17feee8e324809cc751fcbefc57822d4c5603019c4ac6115d89e82d7f66518d94ff94e1399c9fe22e02db4f25830a1f414aa77ed05c63028053056a09dca948698c38ba84b7663ab797f57968b964770329119004af69cc81b9ec26e179cab935107b9f8695c7488918897655432aa52bd0a1be26bf8377a0a0c6e627eef48ef1a19549e733b34d628473121642ba9b56c0e"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (33, '{"o":["12121212595a5a9de28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608ba0ca10ee733518a183c0295526d28ef865a5ff08db0f96464b16f844bdbcfdebaad1116fa7321674b2d64cffa40f3a0f08659cb21359c88301b252fe8e3fbc52d9c298850435acc041c1956ee2a4f4962b151669026c23627b83983c5056681f42d9eb036189d80c329e4f60e9f4f6ee682ffe26543ca54f2a555543a0f669db2fcde91566b138ce7953414b3a4c6771590da2edee8afcbb29b651606e2b6d08f29be08fdb40ef38b399d26184cbbad8603cc7c99791d3ce161351b87afe3f8e7161152853054f7d54ebeda3f858810f1795e7c2bf3ff2381ac8f1cd42452cb156efb9cfccbb53a34ed77002e0cf2a77c9755072f1542f0c97dbfa0d4fb4214f8f82b1fee3aaa4b12cabb245a1f6e84b20aee646d6bbc144774ecdc7054d3b9"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (34, '{"o":["12121212595a5af4e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac60812684bfca63c75d67d3616d0345f9d529a0990ae6137c5d7347b556bf47cc7150155757272cc784900b261854408ba880b2edbfcecbaf16de8a64810ae70ee8791dd0ad28e442c781dad3137b5236e2530d7223c6d395c7c6c781ee4fd05bb27f0b447204e18af2ffd760843192c1d90eb24e832e86d6a61e7992650258693cf954a0ff18c4ba359c405b83fa10e58ef28efb344e22d432804312abe1857abaef2c7d95149ce7612ddd65a71e4cf8ee31acd05852ab326f136e34051b9be4405f2ae682a196483135339d01ae339e84576e632c159ce950ff0da51bc7d1282d207ec9a940aaa1c9bb44202170f42744c434bfc22d73d855b74a86885d400bbb8de0d064620b68ff84fedbf04aac157d05c9ae61b8ec0113c1f5cd1bd17844e70"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (35, '{"o":["12121212595a5ac7e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6089f0266290af1e93cd259e4d490834aa938ebad56ab1ffe198efd11d947b21eabb09d2d0231747556437c00f59095c15668aa9ba12a33e322da3bdbf93228fe6ed53bfc69974ea1f34d8ba4ccfb37ed9a8c743bb95bb4f168acdf5db38badc25c00e1d8f81b96eb5ae0167025d3914fe36c8e95ea9f579ab09919a6f124ed00377cd22b478656a346f0b2b18936f1b09bb371f2c368b4e4afff3a9c9982e1149d9ff10cf9fabc1b756aa87691695ed26652ab04f9be960e3a64890ad34087ee0de4f9a55ac7afe0515004d94fd014447d4cf46272a70662503202ffbd881ca84e1073d92fe5defedb1a9f06865cb500820dd859e7f61314e7f05e026ae4d3b68fb099675b3c038681f37b68e3bb0747c1be057cb429714a01f733018d1c14865e"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (36, '{"o":["12121212595a5a28e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608ec46c05270892e44b88993aebfb1c53f229c5f83e348e057d5184aa16e3170ce2993d196ce3bb799cfd5ba85563f119543772402ce2035d15b23533503df287032799a201178b14db79ac58da9133228bfb189c7aaa4808de7142e6dd41bab1380abd5665b5244bf19e13bc00f0922166bd72698eb7daaafedd4109fba1e5cbccdab1ebcb073e000aaf792209d21c34fc61bf9ab62304a885f20344ee5cd3c32fbcfaf4364c14328f83b835b021e284eb78b185c711548c115b92ab4c853b4bdb6bdbc4ce1aa679d0c53a51403abfe8205d1238fee18fc4d169265f6091858cfbbeafc6c1523a3694bad6fed96855cf5bed530d50f6dfd783d47c3bc63571750d17852e0523afecd3a900cb918b95126777ff22d4ac841a7089014708b650cc2"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (37, '{"o":["12121212595a5a4ee28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac60803a7c697edddbb7c86409d206c268ffbcb3e7364b29620ba781cf80ac8fa3717bc5bc31057c31b4a1495b33aa57ec583dd6cc3b49ca8f419253c2d3db4998dbc462a8c662a9e4866dcd1604bc01ba56235f11d886700751933f9809dc9f474866a78777817ef2a0f8c72c6bcb3086beac84c7985c9b5f42994d1758ee5ba820bf2c9da1cc372a048c04ff8a4d92e640a19416b4f35ea67053ee62e0568882abf84ee886ca02b9669a88fba26f427b7edb9964766ea4146beff197df15291c74c3065e8606fb21d3dab058b3943696c2a475e6ba0acaf703cddd3b7946de7202f38ac303086df01fd89e97ca9338cce9e478e6526494df92d49895a6f881e74634b21bf4ffd4234bf2f37dd567ec806bc9e2fd22f89215702991f56528fe30458"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (38, '{"o":["12121212595a5ab3e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac60867fe5abf16f8909d1de739e4d96323f1f2e0de0d2a92a430955d3d2fae7d4eaf04e805599fbab3ef17cb9858e1d9479d8fe28fd36c8952a1bed0718801561c369e057eb10c3dd4da59b4959abb7b87f53bc507a5661d72ac196430ae80bf6a7427c81c596f6d8bb921927eff04a763d7aecb63288f8eec631fa26f7a0fd72805c12ee871dbd93bbde9a005778c9e925b12c76ea7744cd0c0151eeea23c315435267e3ebad8d310c187531467a8413c3829dc561e154fa742e9849e60020151892598fc72076ae89a533d0852cc3ed9fd264784ff3d77cc75eb7bb8a4cf14ec98577c23944d0ced93b9ad13f6b7890c1b2fff46ca3a047b484d320c3a19af611774596a3a89684bf4e6918872ae4bdadafd4f79b98760773eb6454f7001e64f2a"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (39, '{"o":["12121212595a5a6ce28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608f466a1157a21d3ff122633fe87f70781e6353d9113a56249ff9670656988e04d564c545f80d6db83551a0884a1a29a919cae40bcecae048c80fc3cde03ee2a81083eb0c60db78d3cc8853854d8193019c22c962f09fdd30882b877bc34dca231b63232651d2b43f5710f983d91de356190d5bb5bcf4fa3de7ef07227dc9ad05e077fde240862ab5de7302c00a86d8b41ebbb4caeda03216d308eb047ef9c56630e6dd1370c4e7bc70d3a8ed70ac10cda006a330c5245236baa06a77fea836d6bca5c7ee113887f75339bcb7fab79e75fe03dfdf2f7c83e63f328eec9cd82228b9dfb8ccc036bb57556c4d8eec4c74a65abe427c7b7799ad4abe568741f5735a33d007e0da6030012d09451e56669e61064508ac0bb0cfa577ee730ba6dad4395"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (40, '{"o":["12121212595a5ad1e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608560fddaabf3d070f774dffcdf712875d60ef5770c207d59468393a10b33f9fdc0907814db93af3795232fceed6dad56541a3952fea5408a86741dc47f992cfc2bba16a1b274711a4371934e4f2206a523607011ab1a1899be3f3ac630c513209f8fb10af2b19fd8f68d9f707538c3d1f38779e978dd92a843ff782d31f98f22c6028e8537a9a0ab813bd31535e442efee8cbf6f7c2ac4862d24c1e2150c6650bdb985962a7f034413a5f316e0381d1f841cb0ee57cb251a19614fe3d7ff83620df707e535efcc29d222557f1d93c0a5f48a3668e33841eca3ff0c92252b0aeb5f7725fa9b5544e49642fd3593c4b811051d99b72f69b92a514e1a9f2cb019204acbd2ef60e6c85cf4a232af76d87294d3c57d20db38f4adfd3bbdec405fa46d5"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (41, '{"o":["12121212595a5a77e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608fa20d4cff2117c1ab0f3e34689151c874113010d27b43606c6ea5cfbd69e1630f3e97b7450b0622c04b8a7be80d3f452550dd6c1bb98f87d122e6e7f96dea9126b5f95fd3dd91219792db509dde872057138ace049f54f1af3cc6270e2969a8676aa1e6955884afb435b1ad123383f4a4ef2535502fbcbc7035f13d918d70d5fe78bf014b0f22fb06afb297b9b1c52eed7247493e1e94d6648c2f999d173877dcb9e52252b0acfb28d42e9e367817f6c9e91da67dc29bcd8ce778d288f7fe67216e649675241d42aff58b4ab78d8de37070b0fd105a7fda1ae7a88bb6750a0154a42f3cb48a286306c091e3da10fa4cb8374f4491b44355db3f23db788308eed375e5ef3cacbd1cc8e034761427129407d329724103b2420a8fd75dfd7d9353d"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (42, '{"o":["12121212595a5af2e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6085a19514bfd236b8ddca554ba17f8feb376b31dfbcc36ec06156e0b053dfeaf7012a8b052e238077209f83d309c868d4d8e7fdcb381134a3978fa641537ae65c0949daf13cc8590ca1f61137bd4ec43230a982dca0944a52bd411d48ab502b706b5853b33d6aa2c44dfb80d549f74ba335d09888100116183562cf374273598fd115e361eafd60d5208ef6784062aee2830d30741132d7d7f6b729f821af3ccaf1e4c8b09aa4766811802d5b984a4a36fea46ede81b90472b1961e66b0d929e006c6f32b101af446eac50bfc343b7e6ed51911d46a1a5e7604c462f0d4026dcd445238d88084fde245e8e4b0899faefad797f4e12aaf11d7871fdf0e08cee94045794afd10bab001da445ba313963bc306e48df0fb1b7ad8763f846f74dbdd87d"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (43, '{"o":["12121212595a5ad0e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6083de4d0ad4d58caae0374ed4341bdefc211f7bd2d169615a6b9d2f8b24f98720d6fe8670e70c9687bb3921dc194a5466ade2276cf72839ef22e43d25d99c2e2fab5cb9b96abe0833c5468298ea69612ed0ba8988643603d9c8afc24820e9b444e7f4170200254e8ebb437bcba0b5eb838a726698877af8373196db1161ca10954a406edd1b94fd5dee4e81eb6267b3b70bf6153e9aa65df1ed73a7581864288901e5a2ef692b15de6c65aabe986a38b86145b801b8b361b5445aa65178fb347fe9d1e27e897186769c446ed52c710405f868291c2ac2fcf99ee1dbbe1ffa788be879295420fed7dea42ee2e829e852a921b154c8e3acb956d5577e103ca5e46f84e3137c3f1c9f8d63546de9ae4f58defb4830d709f95fa1d7a0572f959fbd972"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (44, '{"o":["12121212595a5a87e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6083abc6d206bf9a64047cb9bb374c417a5abf4adc6efd15204f6be62439a5bff7ad7f204664a56e32993fe4d92b0835be94bfcd2ce136119bfb42b598beb13a36d41ec6cc92b702142d05089159bdc85ec53e16b2fa050856eca273d4ec56c9bd55a36e67aadb3025008eae8388a19e82036374bb2bf5cfd5502c463705f73ce5d16e3639603a79eb42a8bda8406fb2e3c53c40ae11b23701e2bb894552ede85a51ce0a0e3eed080188f8de5c953261b58c0eeda582e5d89e697a628cae13e76765531c21bc064592e9dcd7523fd89baf836bc484557d32e5019e85f0849fe696bfdf3bee5dd533e968ab863963e50c289ee222a32728b7ea3532f29c035fef7e2074e0fa6598d1c09e9695f9c185d04c29583b7878f785b7947dbec07a7eecf3f"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (45, '{"o":["12121212595a5a75e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608c85d698b63d0d4bd9fe24599164177e9f0d1c0430ee9d087a766ea06c11fdf0dce5867036bce769ed6f099251a2810e482b19b3d63f17dc1f677f07115aaec7ca47104cbbdd99cbc218e8d34ffbd357d03d7a52f9405a556fe8484b9ecbdec30d81fa40bc541c63193840e6d93fa303a5f0d41f28c70edeac3da56ed9e71b3c194b3f80d50093e9ad29db48e71b14273aa60de7cde4addf966c48ffd515001a1a8c757271344409847e20417e38dd5749b55b94c662e7d7c131cbaa6b3da7b75e209123aae38358d59b2a71a00baebb8e6f70c6b669cf2428a7d1155bfb92bf5c3a7b2b4934ea04ae9fd63d4fa7e2c2573b7875d3dfd686be36ae8d62df16e6d75c96024fa4da71aa6c504f45008ae7e540a418d9e04a1a0926f67933985ffb5"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (46, '{"o":["12121212595a5a64e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608ad92d20dc9d0d9b88cbba28265ab87d5a505eee05a534cbfc5fe99d2c46f83949fce84a80344d45b4499c2fcb28b7ec68b874fa4426394ada6fd9f86630676b0fc9451e57269f895921b88668c33c37c2b61c449e2dcacf74262ffac1db83f7a1b4a1b6e67e3e19cf53a172720904e9c2a8b9beb88532a39a19b0ba6acf960642dc2b348308e0128f811fcb2cc17d125f5aca1afea9292eba0212f08474bba5d7b1b517f6d6c7d394260776a34fbf96ef10facef778a455f17396ce5809eb8889de2b27bb7711ced45856c84689fb7b42e8a8283217a9ee233703bd84986e2498df33381446b5df263ff69ef196b42d2ebe5965a4797afaee37ec7822821a51f6a237d116a4dab2c7b1a7642a74bb607361afc2e3895dae9912fd7a9a1edf23f"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (47, '{"o":["12121212595a5a8be28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6083aa53f74454308ebc41d934dd084bb0fc78629cf36173109225a1b80a801732fd4a364c245ff16ca3e136dc4e04f667a8b61bf741eb3acfacbf8f6fa888070eda034f92bd441b26792e4f074bdbd4c3ad7b3463c1607526dd42499c839f278d2f17db155bd77d53aae2c2bd71f548d24fd007807a0a796d79ae096196e4a81ba7d45ecc190aa321f6c68b8410c781483757066c4379addc27e4dadb202f460609e5b2a2551ca469a3bef90dfe8a6b4ab9ce157fc8c59745c2ba6635a2d8977e2a9b068d35ead54b7b741237f0160b49f29b75da4ebb513b0320c062d9e1b98f96aeb19c494d4eef81e3c42afb7c60f3f5a014daf329322b9fe4b87230b11427ff7d3a495019490f51eadd4dfa6130d06937d5b7f522cd9015491ac4f29fe94a6"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (48, '{"o":["12121212595a5ad7e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6085d9bcff0431446cfc3ad5eb9840d2ef075468ea07ebdcc99352b75f068b46575bc429571bc22dab59071d1141322fa355694c43c5ebaf34a925601afcd10d84fd8cf97f449c57c30975fb374dbac4d96f86f6092157e4d5009e42e10915ca8b30f3faf7a87c2ea4e991629b0dc23bbe0c5234b1ae9a02ea2d58334a7deee6b2f115504bf0bc4fe4458c9e3fb0d87bedd98227ebae475082716dbab73599833f112d29a7178f0608fcdb5c7e5fe2828a19b3d04587106e32140aedb0fe891b53964e432ac269d9f822e493ba057551a593087400ebdb5f605173e93e7d485eb8568c00989c94e69f911ae068b4c17dcee83e6cbc5ed8209b01663d3c8d1242f1cec129cd23e29a118253d207747175186fb8535b9ecba9e28cfb57694f0ca3fb1"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (49, '{"o":["12121212595a5a18e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6082afaaab4d80c8215ec275d6fef9966f3f9b8eecfe0bf1b98b5eb5df20e1b403e57d686c0afbaeeaaa99d84a0fb270c63389907df306574d26214aca2926bd99333798c9faab81cede7fabea04bb50460f9f3145288831b1df53b0a418a309ebcbda88e97d54b98118e4f04a14320073717dd617b03a617516413496e27979f606627f775e149ab726e88941707cbb33086c27b002f23fd0beffffbeac0fb8490b6cbe25564317a4ea9c4f3e07a1c8b9fa7db061fde3fc7fa71cce962e453ef1384db55b602ca761a82ad2ba1f6b62b73cb3df5a3e6edea57ac485b58a2ccf3b53cd597ecbc1e5e1d9717a6f7817465f03a0c12e1439d17e7925f2a5b4a02a56fe708557b8b2e1fa0853deef15f6e1ac38d2b74f013fda8164f1285ea0ea80407"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (50, '{"o":["12121212595a5a07e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac60841af602aced8a62ab042a6f22a3c1fdb5b9859a1e6e85cefc433bfd8fcbbdc47b3374354fe38521d254cc2a496ca664debd204ec424f44fc709f6861bcbfc39d7608325b48734219112d5e4d8da731040a04b9ad2006fffa5adbad01de7233d606268f1ee14ee6af8a38a2af2efccc758e1a500b7a833e6b7c99bdf5495aeaae6af856bfb13a330cba9b441d62d5a3fdc2a1127a5ea77742f66ca39652a2a2725a93a1245bf1b5002ad6b97a98047f926a8f26d21ff200e6005bccd63c776af50aad1777010da05fb25903ae9ff64cce639e7fc76649759df976b26b4d0a5588b8cff1dfac8285a1e3068cd7f8fe122e9cff1d8211ba55c88990d1f043c9ad291b0ff3c1e63855d1067cf25c88ce30558056fde9fada0f63f4a1c0e4e81dee3f"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (51, '{"o":["12121212595a5a59e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac60863ed61eb81958f7784fa071871c28557c805e4abba3b7968c7bc21a96f61d711b12dc8e03f060596cb46d421cc722c5a89595054dc1d4519782697b37301ca72a3d822296999f7fed75d33f5be8854ba0ee706ea83b9f7a13b8770cde7e20dee3fbecac4f32cc28783a2567b9f0fe0e1490dc607c19b8fbe58cd9792d9a5f6716ec302be3ce2f011d4c57d8cc64eb2b2d574c6a1458f2f2cb28aa62406b3fd019be5aa7c051599fe4d7e848d323e8eee58c712bb943835cb6489a8fd942f7ec25f7d19c8b2dde872373e25194b21fa69bc38027dc521c1bec9b2ed579d5e9acbbe7b3773b399fde778aaefebae32941df13a8c4d79a55d334a7111a2eed57a1f00f06fbc521bddd3a9cc4fff64d049c5fd42a1635a27facd0a6e9b9fd7c80fea"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (52, '{"o":["12121212595a5a4ce28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608f1473ca8879bdba2927642a91d8c556cce8510778855c7dee6382fe7db826c59b2b71b681c3ef4d0fa99f77aba070f36bb5fa45741ad6f9a84bd6d6212702ccaf5c9bb9e12a06d3aad121283ca78a836ecdc8cca3096a48d2b56b0a8809eca9ce3edc7e33f86309949e667c40e1452fff1824b4d5f18539de234af527a0648c21a25d1b1146fb1f9a55bfcbd69328c505319bc1090903e971b88bdbcdcc28b09c3ac71f6958bc3bfa62381e0e3b17010169c4cd93306503313c7468a43cdf58eb1bcb5e38ebfee5a2559b0aa5198e99e0f69ae14020dcf39e36f4f16d40f7319c011b363542f68a219ad5fdf8e792501b19d7d00a2f1a35367fb843b8e6b7cd10eded7926beb77a379970200b5a1e64abec442b20c2e90fdbac1e9a4d359f47e"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (53, '{"o":["12121212595a5a0ee28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608e8c8d17981708842cac2b5828eec7e2071007f0bca227e0a9c5eb54c6b3e548afc8666de574d5cb8109befe18d146b5cddcb214462db79d364f1156cbaa0166d923a3dd8e6c242e4d099e284ecc4388d44521bde5d515338f0761fd084e5c5a3df36abc0e572f3dff60a3776d1db71a3443b0aa8b889016535d8c20dc7a65b4f0ab6e40fdf5880e30c10b53c3f79b6e1ec694c86f1471a1854e3ce8063d2712293cf5fe59cc0991b165f37034d069b9216c18d4ea90bbbf0fb161596f5a0fc02958e253f501c5a5dc98b112bee45c66ffe628c0a260c40eec801f45f7639762f5c423d30d4cc0fbdb10df3af1d7fec77e0d5f545c50e7c69cbf8688400e6b3e6c9d24eda85de4361b7e8f041c68075304b52bce3c0b66127e6462992cfca7314"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (54, '{"o":["12121212595a5a2ee28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6088c2f84ec2dbc05729b95870fdfd21058c8fdd974846a1e9f13b58db3ebbd3e128e21f3b6cbb63b38d69e032deedd501a66a16bfa3b66638071959eb576f98ff4972d8a268956a387a86065dc6f9a515d604349495a7e9e0c29e014b3a5d4dafc08234076d97e0cc54ae7d3606d8620d1dbdf13cbde3c2ffdd93f7c261de7103672579121c3b3e2ea774c9ec7b8e8d5d07ce4c6f17ec74090acb867b907b596830660a9c109ef44faf7a323a09b04b404664475db25197353f501c4fc6b3c8ed22ab7f86f96d7041b69a8620a1b01aed92594e267edddd5e2aa05f1f48cfce0536c8375492f2e9838413f554fb58b651291b6298420b7b03d380c8b2064f56b0db3b9da0fee829b17a3fa11103bae7ec27e05c4d376fb9a9f5796a70148a6d0a2"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (55, '{"o":["12121212595a5a71e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608645e8f4afabba00300b7f3e2e37522e2bc04cd0ca7a8d60a753c8505f1a46365e9f85000987aae27bd4426a51d47d5b6308ab92c27d3f793030b0ebe47a7fbeb19c98ab80da00eea8bedfffac9edfcad6451138f3f232ae8324c2ca2435d80abe66abd7ba0b0b642e64b90f4c9d97f3e4a09cee68cd4ed655b5872e060f90d9e0dad0de69359d2c2ceff77812f12a6aa899c476cc09fdd57bea9d73e283385ff52a0e9d09d4177d312b6fa1f8780b1ce5f76b3be1bf9ef173b11ae8e0e3b78261c26ac9609985625a5f5ef739c28691521620e1d828084db8c1050083bc0543d0186b1c01f2b178ea8c7229f553490b89b4302e3299f1d6a1c9e6af003c8fa820ade559985a3d991fcef9bc18368ab50e373b0463619be64c8b3f17139ff1535"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (56, '{"o":["12121212595a5a40e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6086b5918cb5257baa1624fbcacc9adc351a5b929a7c6e9b1fa2d9a9ed087e2efb1698a6d0df9e4f9adce8ee553ce7e686e8b5c2087956c86e76a7306a669bf38e645e525bf5e92cd21ec8f8be9664c3c7e050397d5e2923c27739ea54c6c55faff1dae6a78d49ccf551e31c32f0933ee8fb28aacc2360ec2446f2b94b8225ea0004e563cf53610f278e518ebe74858ef7a92ba9b8f47da56c32260dc19855fc37bcfca88ccdc5c48fb07e42085b76ecb14dc85dbf58121e4956dd824d8e54cfc955b3e3b221e6be78d2aa5f2000532f40c95851f980443493b30fa194107dedcf4f1526a8f80e50d2feeb55d54935c6209ad4f5944e3b49e9d4fdad2b16b43dfab11b9f53c792b342a7f37ba3f0b25f0ebf05007f0ab2303a60a009d20e7db17f9"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (57, '{"o":["12121212595a5a2de28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608dc576a3eba080c96ead6765731fa877fb3f94088c7dee0166895729ec58139a59139e8e7f7da5d7812daadd5b66599690bcaf63b46ef37bc05d04fe7805fa80202b95224800317a0ea473ceb3b199d7a133cb1fb318588cb85cf1e9a1e38382adf926e07ef9436cca27f0fb2961f935e2dd419bb5b11d58e12975c85d2378cdfc286a266a660d6e1f5210aa43941991e112068818a759a26790b37a7373d18fa67708874743ba2be08d7acbfbb4523786d3e5f02b6ac75dc522fa16875346b26542374c4b4c7a55fc8957ce2be2717bec48106a70050abb583b5025299789f014f7f0100b47f671c7cc5d658a08f2e0314f4967d398cb0ccc06d9a51b507d135a05f1ebc51b740e404d33fb9d08a7253717bbb45206313286ddad645e3de7164"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (58, '{"o":["12121212595a5a74e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6087e8d899715947ed540834b7975ef3f795b1ef1f27b1aae63b2e4c6d43d63e0100f446eae611ef15c71fd858c0e83ec48627a79aa260ee9be5ac167b13530b060c17bb0fc124ebf14875455a081478195c453cc4ff014641f56f6abd8b8389940568bdb8050810e94601b4ff8b00bd399d6c90e243f5bf8d0801b45ebc35f3160a3f5b3e27fa26e40548b74d47da5e483b291c7b75eb973164cdbc45b8afbbc196f675dbfa7a827afc980d1d28e2e2e6133d4723c392080277c1eea4f921e3d49b7eca9a8e5480e595fc1de33245a0eb6efac1c66a59a48a535588c122e2e6f26df49955076e52007fe10d045661a3ea6061f461f5d4b98b2048800ded36cd9aa3082656ea6dbc87b13655e30cee58f647cc5de98b5be50e5d16370a9bada3975"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (59, '{"o":["12121212595a5ad5e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608b3cb8f8319791849970b6e09f85f607d63f1ac1cc0a512843d1e709004aed97bdf0712fa8bba3fa1c7cffc9d34233edb7e04fc8f10756e3913827ef53f605c0ecfcc71f699b3149c81e144907ac2d97e93af6d4992d744722b3a038be60403bf375ab9e71cbd9f68009c3b6836291e4d4b1f1cd296faee714b3bbb9d538da5c6f9098849e86628e555340566ead8d4a7372daf18b44cdd9def547d7321c809651495a36d33e3bc1b55f13b05c8b59cbac791b7c14c36976e954d49b9519e947e1ef32343536d261ae5cbda2e4b69d982921f59f82b7f57396e6556d6b81f874e589983a087c31661a48c18ef968b907713cd491c58a02d9b2516872aaf73c2199f72e6781351bd2cbda028de63f9addbe805d3ccb7dfb06347798f4c88a1b760"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (60, '{"o":["12121212595a5a70e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608ac8ad13491d87d9a2f430f3d96c336bc4eac39f0836c3c38fe33ef81ee4e49e31145c16caf6e2053f83b262b237f1d7a1bb3060c2070652b99ef3510e745d453139fcb46e464d98e9f2ae3eb61490400b836d1039e5315ddf5539e6293ca107f178372e00d563034eb583b8c595c31d61d7915549c8bcdc5fcf88409fb88d7fb593aac7ddc2e56e91e02132cbba3a41cdc6c9b4acb6aef69aaf00ba2333fdd5b53f7252b2a6cab60a339c263372c9322cc3c3f5011f10acf2053bff7b26b7ac105cef80cfd26f2048400ad23a61ea8d0501a94daca598633025ec3c5b15f2ba4124d03863b093ddf9bc2e3942a5b40131d021f235e59fb184d87538a0208f6123c7a13cec5919ce7b9a9af306a757682c2e7a1e571a44c6fc3c5c9cbfbe50b4b"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (61, '{"o":["12121212595a5a8de28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6083e4b5bb284d662e515d7b545cc831c35c9b2cbc086c6e29dc94dff83394933929d3fc6b1fd3a645417654edda2953f3f16ad21ef35fea1c8422a34399105d102689be97267b464aadfefb6861aa0af8718c0160329e7fd7d43ade7d069c0c6d0db75fa04ab6842080c954b53f9bf0dbed78de95eea43a65c6747df18bbe084d4d87ecb93a1556719d26df2c87cc4425d8d936bc9fd6107455f361478df9878224e257720d66d5eb4d583d814c4420d50a7d7e384b15b01afb0b23d86c7e9ce43ff2aad85ac8228352988b81d89ed031edd9b2155246f00ab0d41505c17a3d6ef2598028d37ef0c8a8e4f5b2d992366e76e3389f61e83e58b68548edc6e5e4f7e885c95209fd397cbe9fb985d84283f89355a79619e5259e4f311abe7618b74ce"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (62, '{"o":["12121212595a5a45e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac60848126a3261f98a97f42ec091675899d478eaa6b69205a4a123c2d22fe9662f1db94fbf492ae7ca4bbf8f6696c63543b19cdf4100df00d43a81de411d0f43d0a01c1d2502f12156a9b66bc2b45747f083f8e636e180a99e7fd3d290c52c664ec87b5d2bfba62997ccf23ea96ac9b75ab5ce88d536878c8ef9d0f92878ba72c9bfbcacdd9b13035df968ebd8719622570b0e26035bafe49acd52ad7406cf5fffb0cc6c4200257e08f2dc4a8cbe94e5f8fe9a4c9af2b331f1876593f917cbed3bc7a1e9fdbbd43dab8764755c2890f32da57941f1b855ea122e25fd435d11a3ac6d77760a567a2c51c782c73e5a30839a81c52b514e4b5ade134a0daca6bf7bb951add342a0fc982c52d6853365e075475bafd61c87044db417c2afa989b7deb731"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (63, '{"o":["12121212595a5a63e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6084e845bff7d579058fff53b2a9d899e9c97c18242022ceda02095295fe1c84f8154dc8724aeccb0f780befd095be4a04b3b3f6ef536d67a00b68f56c194e6c47eff1bcae91ab417afd3b999743bdaa70456e84dc64d6f64b913fe66f487954d1f7e84832c7e85a537a155f5d78627386a91fdab2bbe52e5ba140b027da31a51fb265073330e337ccc62d69bbee76341d4d7a03fa7d7cd703ab6d25177bdc69fb3aaef966e7a65a793c4b832cbe45033eca235b26b5cbc00b75c8c4d3c51baa4bcc367463e760701b34a418aca9b6865fd3084ff2ed154a7c42fe61814e6a90dfbc60a82a5ef0208b860b816a47b1d644deb8ccb853a36d90a98140e99b8ab4835d46a73f5b36d6d3769160b6a2941b73f1f74d65d0fd9fffac226f0b1918be6c6"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (64, '{"o":["12121212595a5a04e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6080e0873ae2aab8fd95951d84631468e7b7023c86c02ec1af3d116a4e8ee6c4d48e353affba835f76314ecbcdb0ced470bae0d8cbae20001df19781d4a7b02627f54cf282c875a3ed525964a80cef924bccf9921bfb5303c68462f546b4dcda1f523e76fd7f14f55340a9f74edbc326050f2e6294305ca19f03e9fcf6e18f225a01b6287a2cb1d6127906ef8e486975a2de72a1043152038ff451d554a349305c1a70cad0754092d2111a76e2b67bd6a71a222c42e66de9b320bd20d267d7b5ff02a9574cc06d1845cba172b2952fc8131f05ea4875b018ca9e06ca1040338d27e432e52f9be6e1cbe0d752607321db7655608ae8a92736a33a96e8682e19bdd7aab9a2a3278634987eeb53d4c3c719ad3e2bb588cc41213298768d3481b4d0cb8"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (65, '{"o":["12121212595a5a50e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608ef8da631d4a57a9329345116ddc613dd61a3b9a522f2ffe35c77c7f927b04a72771ef1bd5bc2daaab6f40b7f65a9c93d11d9237ed19dded59fadaf1d0563017fda7c83637a18fff84cfc2cec37571aea7c7e6576e21846ef7249e95fc981fdd2c7b0ea0730e2fcfd76c4da56bb787453dcaf20f690c491e167666fb4f385be64b75044ebe895bb5b3166fb30485604bb4ea7dadf47c1845beb79281cb5a5a23f0a43dfd614904107b0496995906bc3528d587e13740e3bfc0f064e79f56446e3541874c69ee729de5249000b0e145009cf91e26c43a95a84abf43e75a175cc44b6a7ffdd85f575682b0d7947bf36d7d25daf049e1377aa31022b4c5d4eb80510b9d8a2acc32dc1643e3cd94435972cd25bd74961b3dd9806af2b9c095b15e571"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (66, '{"o":["12121212595a5aaae28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6088f5ddfcbeb9527f931cdabd57d45e31659002350921c18e9ca8c4e16f1588554ca5bdb4a3f08ef3f14cc3947111e74e958f64149cc27aa78dfde38a3c4670f4f2e4510e7977718b0dbe2911e8dadfd74ab07659b07c26cdbb2930b804ba0453780a13f2c4fa86657d1b5004a82d7ab30ca81ce24c2f48bcbb9764ec855bb834799711b35452e937363e5dd51b22d387816ee669703e8580f539ab151a04a0304c5d18833572ce552313e7683f19e32bd30d9e62830f8fccb7439b15112e55ec76edd09b09a8a0c475556b22e8621f7e6ed728d216e3a6cf913fd394e736e71bbbc6cbe191a2a2efbe118a4a175af2cfc3e94a72f8107cf94a84997daab4ac789c28a7cd90d0d0bd438e9ba32c9d37a19a37980835532a6854e2e7e1094ff2016"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (67, '{"o":["12121212595a5a5be28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6088a089d0d7b4af521dd7ae55dd38085c34083c4395fb86e1a2fb1c88e89ba17dc4a757f850064febe6af43c117ade5ecc1da5e9fb828f7327d8fce73003a35f4fca590018743209ff0ca7916251dcd1e5017dd0c3ceac88464d7442a7a6450fc010eaac74cd85564617349df7e1ac34333ccd714a067392cf9c66fd820b436504361ed62179b5ff831ef0bb44f5c80a3718b708bec96f7db9075d92af334d337251251a576f65ade782dfd1655a7928768760cd02cb6bd5bf77f9336d53f5f52570021b6c8ffbf07f8d924b624ab5c2f6dca446f355e6ac90804bec100ca7a72e16e99cdaa989b6db640bb2db34d82422e81ed7e6fcb9119cd55bf27bc703b9890249733a40c6541e3bc66f76d0c99e46d1836dbb81a704e4b800d903d4557063"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (68, '{"o":["12121212595a5aa8e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6087c20a49f8c5c81f37ddb1b959b058d7e13cdfd78d51cacf671da0b3aaeb5f0814e775373cca77cbcce6db62ae2e1bf3979d0a6d1c24740338c7501f182d3263aa4bc1eaac8414f3edfa290ee2b708db43462e12a556ebce4890e2538f26f516c69d5fdfadd6b898571fe2d3cbf5e16560f153a2774cbe807256445f688ef7081b0c1fc7b1198dd72635ef42bc4bec3903236b97d57e74139bbb756d77f1519a6a0b39b0122c77551ff83e474a66e71ec43e5b5dc25a91e8c9b317e24460b5f52d360dd64cc9aa574299526e12852f9d47d5679e8b73da9c7515cbd3959a4a2eadfcf4223054064a78efba5c7f3ac16542cdb005aa0a028eff7ab39865bb87ebea27abbc546374f9c9f689da890c307dd692328dfe880e3ae22021475cb1e8fd0"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (69, '{"o":["12121212595a5a2fe28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac60869defc15af40a1c5756e2e33554567ca4e27fe1070d98c656204b3d6e22b00a0ad7ec069545d62cf8c570196d0c99e3f1e00aae4aee8515ce18b7d6ff43e015072e1ee3e745d57db2781dd29b3bfacab9b0bf7719b1bb844814e3c2473ef4e9c76d61ee7c129ce9a3ac81edc7cddb120c14c0c75bdbb5d0fb47d43da48abafd8e83e5ab31c274e7b666230ea57847adf99d5e304d4a07c02194644d78e1f965cdc56b3c87c42495b1ddcd8fb26aeed06d6dae5c3675c4342e7e45f558f408f0791c79267da123810ed47be4f97385b62cdf55fcb2a32a5e7bd81f4f7f7a3ea3e9cb60dafb97834f06be1beb0874e6c3f3f66726a933b9fda1019f8c2385bc9d444ee77297b2bae8831887a972ffd572d5cc48e8573a881f83df23f0238ebe291"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (70, '{"o":["12121212595a5a3fe28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6082f0d03c0b39078e550449f235ba8058ccf6bf563f5886dc6d3c5277333e6ade0d70cdb0161c9d8de759a42e8ff4153b2ee354c0f5269b01e5e06ebf7e87e45b6e084349d18253598cdcfa992f3bab7e349bb27cb9c6743a4b0435e8d3eb4b15890f13526f40a91dfaf6541c80e4cc8852c286dfc53528b3397453d396b35d4ea4b39122c86638c4366c8950311cbd850dd944ce93d5dc53b5cdebf7b5abb11acff13ace2f2468c9997c02a4934197a707b551865c5bf0b940394dbc1bbc36053926162341b431bcbdb4ac191fcd7951e26dbcce38c45ab0f1b7d6e2523557346a04d08c04ea9d0889a11b240a510f986e38689979e390002a7b01d2cc62ede3dc9b98c57f3411d4a3f8fba2c1189ed2bb4804ea0c1e8cd227a22c200aa04d03c"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (71, '{"o":["12121212595a5a99e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6088029f656403c03c5054d4bf4b8796978780254ecfc17d75cd9cded3fac0cabecb3ea4e36ade2a355bd34555baec5690b17feef1cd9c451c770ed053a73153c6b9cab738967ef3e911b39061629ba488b77fde2ab0effc30c9bd3af61b671574abf0d38af9909c0f1a3a0408c2ddd848db763c84c8268262bff32b4e4abdc1200e826cb27296cc37382b81804cddb44a947ede7dedcb8fb4a61c067d8b961ed541556fb0b8944e56c6e7675c4a4fba412dd4894c05b6ccc67c635b7d981e7cadf2f478f7be6bd86fe49af3a047846ba0217daec0f20a28b4e4bf11214d03d5627d2f2958cfa2487126bddebf9fc4705c2f2c6a8f3f62a0a1d6c51763c89d54d43d2338684aa7d063a319ecebd749598f697ae128190bbad8009cb5c5780ba5282"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (72, '{"o":["12121212595a5aa6e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608845845eb4932efd10c78a45acec01defe23ef213767976f57e54a274a4ef7f3b77ebcd4f8a9dd11b60684e52fea5c871806a08ee1e25e48ed7bcf08cec516483eaecd7e9ac9d20db53fc8c21ed4fecae6c1a7006b791fafa0ce476d0f907c3d54164e9f991f960e5393916cd279a531fa30805d191c2d762cc8add2696dd0926d8a2338c3902fd0f9d25141b2956233c3d0aea83d8a51189eba4bc1eba2fb900a5e8073620755d654169a92990841cbab82d8440a707d3171cc8ae731c3e16f3de6995e0dea6993942632f43de77e1ae9133cde814bb7f06388eb4f8cd2ea853fe173c3f014badc3066aedc31b8bf0356542e8fabf62c2c281b342e8ce44b02f3aec2401a29441ee5b71c99439a2e894d603decfad89cfca21c7a53814b943da"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (73, '{"o":["12121212595a5af9e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608211126ea5f555ed970f0069b8e28aeb919948ed7942f3b094cd76f492107df1f38770d5706f24885d31c8f908da7c646b11840597a041e5652e17cdf9ef1fcb146937dcefa7a98c01579779b20182e50d21ffcbcfc09dcb3b24721686849ca864d8167c216d6fb0e61361c9c07ea9bef84658c76296dafe53dc4d088a2d4b155c40dda26bd61f27f3103c8aaee61296538e17145f279b9ba705346ec237c0f2a708af577df747b3277ecc1545db06c8a33246d3c4c83d820715ce468dfe804f07e275c3b4aad5f5ce660ab2f8f8277013aeb238bb9ee81d08df81a5fb60d3a0c02bc52be8a0e9462d8de747e42dba8a149db0a215f9730a720098d528eb421f7bd2e7868db6b825fc13912e4d2376ccc0e1f29274408a6d5aabbc36147375cae"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (74, '{"o":["12121212595a5ac5e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6083b88285f8dc47b89b1ed77ec30e660f1f34670babf8807f26ed4e4e9fc99f0257cf7869228b0892339b352aa47252e1e2c52edd7ad7f59240cb3469f26cf9e69514a679f5b749639622a37fd22454503dce85c20da176dd76922ed64e5b5091620b8513d76a014c90d1ce3f06b65f96a75f127d0f9e9e7a2b77535744a9b9d616dd2e02b48f5f3f15b804d9525bf5ad08f155142b9d41b6937e05a235de4a626079a66c07e2a704a583c2cdd2f9cfb1ad48a8202f573d3687a2620eff46420e6ff7d7fc2833347a84b83713a0786675b40ead4b611f26bcb51bb901172cf0ca26796972d3b750fe066a2258094b9116b31ba5262c6359af4218d12a93d64cba7791a47d6c54123cbe195f50365c426c34d445a939ad388bb915a521b11deca7f"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (75, '{"o":["12121212595a5ab1e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608c7af1971e60d36f2392f8cc76b1b1e94521a67b21c388136ff47ae5005a8a255bf6bcc2954fd044c9c5a8e5ef464281ebe1d6352c3978f2edf9d64d904e6165ebbb47036552ad8dd9d67b089b8b475169f3d5d8fc38e88a92685abb5536f09360cd91b1d6eb4ca1faa7fa8bce538b342d6e29180dbd19d0645272b4982148ef019c2c2f10cc95ad3641226a955b6cfb3c98fe4079b11a319a93bfed36e9fad4614f325da28f4861ea81695407047a54341aa2f98220574e2c00385b96037aa58738ae74e23ef7ac012f09ecce288de93f9876fc0ef72a2a83ec7439802d4dd9ffe30725e683e531888b70fefd2c03986cbcd163f237af1642c7a273184960e7a46285d006428bb1c718d2cf464273edf47da6bf888ed9059e00f0ea97e5d2054"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (76, '{"o":["12121212595a5ad8e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac60887c0dca0293b009bee7181e481ffbc26f19fa23823c6f8d863d3b951416e43801ba0f10fafc02c6a7c50109a7d51e6efc8ca7481e7db63a32b289d5f6c2f1fd84ffc7a73a137a68a582c4405ab1ae477545bb097b707ccb29a92a8300e4992ea964ac03fa47522bea7673a1af4737338cfdc9a4f51c6b5f7f3ad2dbd12467c7e4f42b43e0724682984645ca9ff32a29b6f694aa1c580f0e0690077d173b89f99b22c96b6793aa618c02d3d7d99d0005ec38d8540e32a42d2ebd09ead9815c00cd5a6ab94741cdc4d60129e605123804fecbeac24662f5bba390ea2192c3b7e1be3912e8bb4a05192fbef501ed0fcd6aeb145f0cac0840435a9ed98176f892e762a711c3ab596d6ab57b73528f8a174101a89e6d9606cc2f36935bb422087a92c"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (77, '{"o":["12121212595a5a90e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6084114c5b3924e9e64c2695675af5d633b806b8411648b24443c95eaca6130a795beef9f5356fe1cf838261d8ed99b4cc005631b7b24c7cf14695e39db7924e4e0526a18d09e792960e4542e03c8f2994bdc13aab8309867104811ad3c17588189b322020919370081f4cd3f7ba259bbf24a807d865ff0ab499aeca82b4a337b73a1fc60bbc13b3d0a03f5fcdd20d7d8171608c74222fe338f18b544cb61d639f54e1274195564c6a5b0feba56e12f7513c5278bce09c4df07b16f4cb5920e1435fd581bd4d30976a4ddb79aa53356b1059eb74ebef89fd30e111261c0842bf9f60c04d2c54fb2724f3f6b6dda0a07825d4d08df314c44f4a1e3feb916c65edc7b64d0ad55d290882c67d7d8741dc1da671161dfe29ec144987a6bf5e0dcdd50ca"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (78, '{"o":["12121212595a5a4fe28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6082a3f37c6d3fcba41de6a9a3ac27a116bb18fc5a655b1b740ea367f20d57a1164796fcf14c0fd758b4b6f100c3b4d101ccfc7628fcf4ff0c8cd8655f333f9587454824fa77ee7fc162f727cf520ec9f2b8d6fc3a85099c9f7e68d0bf3312ffb67412df0ef1cb5a9bf14c436617f5786e11758ffca9c249d371a59ade168f1e6a559b2172db1fe36b0962fa2be57358f0936f72b4b4f031b1d982049922fa4f7d74d2401e034b64415370885f03bc8b4aab24231f3fdb90c568c9679d665f57288f8c65b04ecf0b4046c0105d08a687112109f29de5a74921230e3b758d5c3136f28522f2c04bb663d435a331aa1e744f84753dceecb82e665c637565b3fa3f384dca90edff467a096029c49eaede017e1e2fbe838110f0f150a1cdd7913d5c07a"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (79, '{"o":["12121212595a5aa7e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac60840b0f4b52838d2bacc4777810747092751e1352efaadc2fec7017a7b6e5f4f3cd12224630769ea14b9a7b3c0f2143243b00872f02c30b8537cd70d816c5dbe27fdad08a2bc06a6fd9765bc5d44fb715d26aa41f289866ce9ba18412d782732b448cf1f675098f31c55ef917d0bf83986ec14e6cba17cf39fce993855a2268d2593a360fcda660f5d1af4ffc8da1e8abca3b97016f99ffaf9ab0d64c1886d60a632040276469ec3af4364e3f5b01dbbf7b577c1b7abb07afacb7e15b8a4be7038a98d3551690c6eac31fb97d06d11bf9191197244a322d3bc5905ec5f350326cd8826349d82932b2e0988ecd3172ef9f78c815b7575b36b3abeab824081f4bc01810fed8d98c3a00754927be62cf84f9fc1463310888814b003377f23869d8e23"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (80, '{"o":["12121212595a5afde28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608598745df9d62deb0cf3f0164a5e77c7fa55183336e06aac2fce964bc0386a4ccb43d3c0fd0781b1083180099a10ec49573557ee3c3e7e263d97076c42eedac1a806cd679fe30b7192b18b07b38582deacb6d71ab796ef5c7ff83450016228dc35fe4695008e639f03f4e9312680d27d30a64986ee277131c7c7aaf56acea857304c765d3a49c24d23358f295f3387ecb454774e1759b07032e1ac43587f459991ca0d7f8f2a4eba77b27b49ee891cf61425625142578a87941ab12c8fbf5ecf2a473792086312d43b07306f0c84f0ea3aeef424bc4a60d3bf6cb58e344b55fc961d05fddc6b78cb6d6f497f7b0f52de383d851f436a72bb88a2afdda9e8e070828b7e6100987babb3b428b8fed41d7dd59b4bc370a54f3f62f50ed0081aa96d6"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (81, '{"o":["12121212595a5ab4e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac60803922f2e23957b6e2657e5d02771379e1390572f86081d784c25f66a4f8d8b68eb93e70e353c5bc14f9ef6fe0d7d0d76a91d2cf0f2a4394b71c1e479757bd08d1a7e649acbbbaec019af1200b794c4986adbe9c7d84bb751cb436cdce285ee928a9731995e3fd5ac0e67b1daafaa4f647e1bec1c78ed9bc5dcc3f478d7ccaed805cf8c66456c8ef850309dc620dfd1b856e526b2b15578845fac5572e1c9216d5320ae3c1ce874dc92835eb294fae4b75b851bef71e626d2df6b768f4d231600bf975c9d0911514ce8c13eed97a59fd6dcfd93a7a7062117af41542350c581738afd70fa19edb7e69bc0f14e91d3addd5272108e9b8ff6c01a29e323f4bc656b15e35a0bdedf1cbca0bf0da929bd05c7a2456a9fc003ed14f307234ba3ad81e7"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (82, '{"o":["12121212595a5a6de28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608233214760cc0e0e9e5102b0c6cae45ebecf7a74e8ac3c23352c6aa6a31df85493ded5276de8a726e8582fd90617ed162774ff2b68fd08aab11bcaa9777b6d236bf1e6672a003732650ba39d44211b6f5fa9388e30b283e4d7e2b5bdb6551ac347adc5f971e1e21074801defc1fd8e150b6d1ad83f11c86361bafdc320eb99132136eb3f304c06e956913cbbd3a5c427db1bb2a4f453c6ca874c15e9d93c49f7a61b22c45dafbe0719b81cdeaac84c00ee7c7319b6553a5dbc9c54f8d7f0ce9c9da57bdc4c2ec1a40cbea52b5b57f0b67f4e9849b29b3301758cfd0dd0342b09c24b4495c25dd3a49f0f3a60e729fb4593e35ce1809702b1d07327001fa494a15d7cb064cd44a347fde8c17ced82f379c5b3716f5541e181eba981173e72f8219"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (83, '{"o":["12121212595a5a3be28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac60851c52beb1f6937db40b83c84b4e389135529eb3d64c50581d002b01a7bc4d93f4d61d9f0d7e9b1415b2ec5d5d49d58a15727a5e762cf16400d3958b19e08edd615be506dbdfe107807acea24e687a2cc45b693a29c87711e5d29be7a3ec9c9239b0cc39c62613aefac58273b4c5148b5d52e53f952ad352eba31aa414ca7b049d9bb7be547a6c6d9c14fa643cfe2b480f240c223311e3546dda429c8fe8e8ca360b2c1f1b60586a208a3337370d78bc7aa252d0d55d0ec6f53245a7433fd17736efb3a13a90086cb40385f4cdd5d9d8dd117e55d90df8d8455e7d3b3e62b30dd778576a037f59dcbc39fdcfb59eb2064fb9c215ace2a79e4a1ccd6ff2515429c44211be11784514769d26fd69bc7c3c90e6ec3e8b520cedc7800784bff179275"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (84, '{"o":["12121212595a5a24e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac60831fb5bf57d446f704d45c6660561aeac67310bfb6473cbb1756ed507c11abc40c6227ed1fde0f4b9b5f55e93aac7259a399e86a77b0905ee6ac8dda59597c1850e08841405ab29fe29bc58045f770bea354e7caa91115d1091e926ecdefccc324427ca8601207a3f97933d5ee9edd1ae6edf24ee28a1584c9d6be079883ec3ec8c198fbb42caa7d03a4510a6cbf5b5ff996c2c6c146eed5a8c112460f1997dc87e2f7897125dea201fb075db9fb80d8d470e5f25eb72a26af9ddcb0dd41b8251482b9fc55934e19679d1b5d0699e42fe7ca2b11e0b0913964ce233bc7c3711b41dcb2557e1fb04b0d1fdd331c94ece36eddb0100ea668085694335cecd9ff709b7e7a213c222918c20bfe2d3cbdcb42ffbe3ac61797984062e84fe452211b4c9"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (85, '{"o":["12121212595a5a23e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608daf861d4875cffbacf53e0a3b193484021439717d3b7c484e7c979202703537ebe77c5359e14b94ac09cbf8d184ff31ae7f104da45c912fe9f4275ee5e019bb965a52ab9930f99a041271013bdc4f08a0388200bcea98e612070fc515e5f7190527da42a743bdba0ecd0873116f15717a355e7b3aec6dbd067572d6c3c191c880fad7184517a8009122559441e41b1f042f82a03fde6b495d4fe5a8dcc0dc0e01ceec7705b8e08f18664cf2e4bfc5f9e6ae9f132319460ce1d0f00142f481a6fb918aa8d6e0cc99046d8b7f4bc1db0e30a1addd157f4d8fdc324cb7fa06fdea23722886a4e414a1882945f8193bd3ee4b4041a2fe5fba8c6f9247d0bfa2944f4c68984e53416ed5445d15414f98385fb3a611a6b6230dd58221801b308c6b0fe"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (86, '{"o":["12121212595a5ac8e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608bb1da1d0d8ab1ee04a0df3bcb40c662baaaaa29f96dd0c1d66b6de690931358907cf6d070994ccdc562f4a62883cb265ba3dbf12937811c9b523572246b6fe83f4fa286248d2a777cfb1ecfd3f5e90d2ab0c005f2d858a5b5e65aec5f3b991e3e612e07a786519f4205335f19f784d0c7973f156d4e855821365a80cbf97aa199d447a35d02144c2a946269e240721239e6ecc76ab8e7d9fe10d51fe1db07df9e70e39e72f493c2d46958f5c4d0033f643ff32cf3cedbbd132f2d250fd92d025d4fc1aa307fde5768c9f19f18c1430b7e6ec0a9ecfa91c5acff8812a9a8f1bfff71043aecf497c5210ac7a3c98b6dccb32a4df31cd35acc3c222996d6204c3fe5be19bbc4c77c5024bcc06b60c8d080848c5f3bb15255df2a77ece5995d8caf7"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (87, '{"o":["12121212595a5a2be28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608e2cb4b483974ab01fc063b6834d226788613f6f7884831792c25e67e7da930107895a7a98717ced4e434c5a6ece6247107aa9650f32849d3aaf68336f664244e80e373d0920edf50825cef1e63dc6669b731a301ee8ae28d598d20713f7ac9d0d90656e1e26ffd1be3ffc2eeda7af524fbdef3b8c25534a5bb950721221208815ab9785eee236d15c1da6b1532a1799be74c017266487dddaf666c6a4fef865c10e5d4e4730b9c25bb2bd4082e716bd19a6a84bcbbb09fc00a799e547fc6abe05ff584310ff5e065f7b3656ac533e3299dfa96fbb8738b5f0ce6d62e49e6c9f73fc4ccb6ffc3bd4e0efdb70bea9fc50a4fe1e8934e2528f30587437d9655824b9bb5f88d5a5cb0fa14506bb8ddb46456eb60070dbcd9daea15df284db9e3e570"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (88, '{"o":["12121212595a5a91e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6085710fa8f45ea4a4648d155bbcf6f601fbc4b226c667928957372ca71698368a90b5bd3888c487a1d108cca74ff00c285ac80490c9e4ef2ba48decdb20a5462a5b6111bce5af4b7ec7f0fabaaead0c3861a34db9321c4f5e92b9a533051ea8e265d1944e9ac7a4d0aa62f45974c83c043a8e9b173017594db1df7d5790e75dc9d0328f3f64632644246fd18e3015b43c1ba6fddd1310326e4ff650b2dfe4ad83f5d8c658631a7c945335786c9fc6d21f8c463dc7c9a0467b0fa84da6e838c7af5775d91c2748204df12f7c9208c5839bf8d9dd006c372662c3bf8b8d50e8d075b0bfd15f78b4391dffaad3dd8a52fbb94f2c7ee633dc356ef0dcf6ef99c4eac14a28ad8b35af32a16b0146b97dc0a82d6d0abcc6af1fd6d8adfd4d0c96b79b17c"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (89, '{"o":["12121212595a5a0ce28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6084e7da8dca1ce0853968e53a17204aa77682bbef7102e3e7c4025d4a4ba30631507ed126c5dcfa332260f1b8d8f3e3dcc78b4f7bff77dfb416138cf5b65921410352f90c7025788de49e99f5860d12fd2cc6f13ade4b74aef2415649dba2d03494ca2bbe068eeff1bd600207514488738bd85ea71c99ddfa6597ea8d5a536d14c15dbef2b360351ed1861ef580586c641db954eb0e8fd82d67e8df9002aeee57392d86933171f8ef985619816a0c85d68173ab0a22ef283b48e9cd7e84dea1f0060bc6074a4475a79e9449ec7e4c1561f196b06f24cf0f3b1ada48d26b4de96806c2f5de03e3b21036b1e955b0ee87f1a23fa8673bcae7493c2c4bcea9848008fa1535a27baf0d61ce19d323276b8b958d4da7aa11ab2cd0a9e3b73cc51e472ef"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (90, '{"o":["12121212595a5a7be28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608a149868528a36ee0302dae18114dc39c75918d991ec33c2076adb98cc7894bb1002345ea78eda9993ab3f3337ccdace09b28e44d039b0e291296b506c906912022e965d8e300c71f8c77b187acbe831e2f03cdb2461670633840b4e372c2beea1bf9c4172b7ee4a5db8b5eb2a9c6e27ba85e2ebd6b059a74e31044d01d0644a13c622cb2a8e5c2c1c5e0b9e4eb021e1096353ec31a1b62f60b5cd3882b0c4d7e399da65003c37e4bba20ae66092acbc2892feb302eae36a7b90a987b6c1424d0d70ecabc695d96730cc7d987c5aeae05cf8b491fc304325f5c48b6ee8391deaf7d00766e08e278472a016c32b644d1936d2555ea326d566780533ca62646284391daa46d043fa60fe74fd6220dfb538f64b387fbc78f72d80ab3b78aca0eeeb4"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (91, '{"o":["12121212595a5aefe28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac60886b2c3c61a2e74ffb5bb76b09f33afb7524b4c0ac44edb49170e93518aa9722f52ab2c88429386b750cb8b49d1e9d903f82e4c0970e55c783f04c113e3eaf48340e0f852737f7f9197253f8fc0e5c39c73faf9ded1dc6f9511c48b18747f664b70c22c05abe11651d64560608c9a0f2e72d6565603756e7deff9535680d2b4c46f9dfe441611d5dc7364950f41f642ee32f419101b145ccea7744b527cd68f686cd5cc19832120d9c56f10617bbb7f348adb7565174d8864c254e7a930049023dddc238c796f1b57cba34cab331138a0e7e11a2a0b570d606fd2c3b16f6baf28bf5260394e2555828393cd30e60a6acc1b38f9291d4c27cef72a0fd5cff98b26f5e58c97b60200ae2342ed9f9aeb0a8f81a2055e382456445bf39d7625c1e484"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (92, '{"o":["12121212595a5acce28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6088960919d48782e826e4255ac1df616426a0c6b5864c424cf72d07e5da512c198e5f42e069a2b2c0b0d1290a489491b9960de20bff5a37a0c6154b79b189f2ecc6580e431db2e14708171635952d8a439c921dffc9e27de86c5ffa215b7b850860e4774e00efcd62469d9e0b4f0c79e20781f6e065fc969afe87b80c1410a80f09ec265e731d4f5b36dd1fd00eda989ee63462d5bf8a1d0cb521d6c66d6185df524030878bd1a6cfeab3da73452dfec7994a2f41348e226628e572d0afe84b6ff51a417b4a977aed1c4ec50b06061a0636b9d8c55df96adfc65eb6c4dad3f0d60580004bd697bfd6022d8553caba51ead2f138a8a1e8db64fd31132878bbe228d86e0c4716dd045d20f2b11324fc75ae384f56e795b04bbe25ff1cd1522b0caee"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (93, '{"o":["12121212595a5aeee28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac60807ec25267cad1be60ae347ef4993376fbcedcedb5de016af5e1be7ae01f999368580b689bb1939872cc7e3f533a13f9082c852e08efb22a8a9a3aa73db2328652d1c662c11e1eff078e53f7ebeb895ba81c0c92fb2d511a0a246cc6b168126b82620eb578b9687d5135674e499d660b130bb13a1ad238d31c5ef456af9f09bef9edb8d390219dde798f0484a7739beb60d91eaba0c0208a404e5eae2377e498032f7b3283fc90410cc42313c1c18ea0b44d008ee9f4612854333d002c572c2eda91a37b8d1ccdf49ec3d626ce2321d3c0c0b166aad98511219c22f1ae853372585137e4b6f38be326ec35a81438669b86f85c8c4e0e2d30510f03b6864a6787778f500403119b343cabbad64b38ccb58cc6dc9a0d2bf90a3c734566a7b926670"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (94, '{"o":["12121212595a5a41e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6086c345a74b2000754c23912834b321440682a805feb025927862af0e422b5b4ddfd30de4177626e043c060c0fe20aa04f0099eb5ab3e3ea53ab7eb5df376955f9467c02628935b93c00c065b51f300d1db238e1163bf26587685cde19228016acf7ab66124384e05bee20f2968429bdf213fb636daea986f8290de85ef23e539dd04fcc68d0a1e2bac32cd02ce158caac754e06c384404260cedf753aa25bd721e98ec7525dbba78459b166c1f20ce665fa46310f0058d6bce02af7a2fed9aaba6761f48da1b34a5a1ab769645886495820ed4aacedd4ea61948fff1a74313b0b7ca0d570f2fd8d58254051894dee9cb7f234d78a3b431dbcdf0c54d1ef37d7cfa699d0c2ac5324614b391825d2b646923460ef6ccb0440569d271dcbb366e49f"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (95, '{"o":["12121212595a5a34e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608b4b262f9497bb5450165c6b6ca4608f7c1f70d78b6d6cec2f309b9496ac004cd9de2b155dd9ba4c340e937368de28af4018525619bde0ea5a7517cdaec1e0490437622866ffd4e05e60632a3ee6405a8e717d0d57735ce54bf20eae3c27dd363ca54e3897c8e15cbeb84ed544bae232d39da5e5b963af489e269d20f5582b241059f50b0d946f8ad2f030105e364b5208539486ea8c8856d6b45d22d8cc52485969a21a75d161232e4ab6f5804c6cf29a18bbc788368ce67a2ca63c22dde5ec5d0ae795d6112214331f2a560193ca471c4717e10d5a5c2f2ce9eec4bceda32bd6ac1ad4d1ee18d6094b6e034b88fbc5764e33e914df7f6a1a50bcf7f3fc7b2e89bf1b2092a7badd1316e842f25a4cad88773b02657bcf2e36aa72c7acbd8d2cc"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (96, '{"o":["12121212595a5a15e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608746a0d0990943c161c78ce1489965a9add8247ae8372723c53e37e9e60e748b3d170a34d86351b49573e665044dfd164c4cd3f2bf12a4935c066024d09afd7d500912c6121ba5f0f070b1075b7e938a95211e36c55b09a14d75a9d66f3745496ca690b4b7fcc09d5bfd1dd4d92f32e77ebcdd98d15a4ecf3fb272f9d0715da17531158b3afbc25de842a8c80d21eaaec48945c7f002422f9ca3512137c31ed6a6efe80d9901c2a9d917874d7497e313d6d19f59db7d34415936d06d59d117716a65c3ba05f49873463e29092f01dae6cfd5d97375d82c31a39a7b57599b83235b6d69e72380242bf4f990ec1bd4174bde130bd26181c86e57080dbc4f9388fb13070cf6f75f6ef76e48e3e1d79ea0381a55e4fe564b00732e06d4c52441087ba"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (97, '{"o":["12121212595a5a7ee28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608c245f21ee9e7d27442e441e75bfa7069ddfbbd71a7fc236457a0a876091f9a0df330c224c068e30afc7d11ed7c85089ec890222e647460ac71e788f83527a3b884c100b25e6ab07b0b84b7e6ff73f841648ed0964d6d8d3dd017ae43e66be64290f896e32f8904b4846a7c9db200879049a1fac2fd474a10171dd200fe921d1647ccfd7ced5e784f1acb4ffac120c3d206f1d32da1a3505b82eef8ae878ce46c3edc9872401b47ceaefacc44d0fcf1f62dbdaf65c49893438f432dce4a048350e36d87fc2dfd534bc9e670d88f52d03371a9ad069cf6f839510efe2d3931cb96b165c815482775a8a7812dcc5495ce6790c85329740d7dfe889208128cac8cf994334c83ebba9c942ef57ae163837307b0d7d5e54fcd4a5e5c9df5a4e3827e8e"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (98, '{"o":["12121212595a5a95e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac608e58f8bd54b79a9cba19af71ae183ce47ae3162bf3b1b6a05fca35646443ecbf81f99616d92c55c6fce4d3fb87a9e47c82093f2eaa13af1c40275e8a50857948f91489c3a11f49049bc7cf161f1da1f686ddef78a00febb6607e8e93574be50e58a0d5071a1e7bf5f4067876388619fa1275a799e80b373f9c8e0c760dd86ec247374275f7fd47583f7262cb6e1e9040c165aad658f0c67cdd0b7b5705522df15a65caf6300f242f477ed69534f1bee50804fe98f0045df42c07c01533b440eac20ac07d599bacbce6d94075254e82f228f9cd8448cd7b0666d73376637137283fad2ad02c96e35d96815be62c9a064de9e21c493dc966a5f255ca9d4b6a42d85da472aecca898db91281bf188bd7496e15ba868c96fd85f6313c517d15dca6c4"]}'::jsonb::eql_v2_encrypted); +INSERT INTO ore(id, e) VALUES (99, '{"o":["12121212595a5a60e28282d03415e7714fccd69eb7eb476c70743e485e20331f59cbc1c848dcdeda716f351eb20588c406a7df5fb8917ebf816739aa1414ac3b8498e493bf0badea7dd9f328aa5fd564f2ab7361251f8e54023f71aa1423dc282fcf826e9895c4949ff284567c9991632dac6e1773aac6084a20e584329d1e08ca87ac3c3ace52f8d33ef895428290a524463d430fda7d6751625c11183b746ed548de7ed97a9d858133ffef040f5c9f3322a3f67d8393462a78caca0879f8e5405f378757bcba47672d6092f30ca90438db960bee435af9e32947083ae490ca8ea18703aab1c84c0f228e76b61aec4446a5ed5f63254d102d46e2a491036cdcf4517614beca2ca56f8e6abef382f4258f8ec726ffa62016dc9d91571e7ce68cfed619a57d161a6c3525cf9f733a93e77aad636fd117456e6435af23109e2b81405e1a2a9647c47a16ab28620e8c49bf0b1d0183cee1129b8292e1114da677ed872b251d12ee3c151eb4b900a46432dd6c9802e809145bd51e007b2176f918fc9cd7591aeb78bfd0da45adad325609fb8d0fa0198f745107"]}'::jsonb::eql_v2_encrypted); diff --git a/tests/test_helpers.sql b/tests/test_helpers.sql new file mode 100644 index 00000000..99a3322b --- /dev/null +++ b/tests/test_helpers.sql @@ -0,0 +1,618 @@ +\set ON_ERROR_STOP on + +-- +-- Various Helper functions +-- + + + +-- +-- Creates a table with an encrypted column for testing +-- +DROP FUNCTION IF EXISTS create_table_with_encrypted(); +CREATE FUNCTION create_table_with_encrypted() + RETURNS void +AS $$ + BEGIN + DROP TABLE IF EXISTS encrypted; + CREATE TABLE encrypted + ( + id bigint GENERATED ALWAYS AS IDENTITY, + -- name_encrypted eql_v2_encrypted, + e eql_v2_encrypted, + PRIMARY KEY(id) + ); +END; +$$ LANGUAGE plpgsql; + +-- +-- Creates a table with an encrypted column for testing +-- +DROP FUNCTION IF EXISTS truncate_table_with_encrypted(); +CREATE FUNCTION truncate_table_with_encrypted() + RETURNS void +AS $$ + BEGIN + TRUNCATE encrypted; + END; +$$ LANGUAGE plpgsql; + + + +DROP FUNCTION IF EXISTS get_numeric_ste_vec_10(); +CREATE FUNCTION get_numeric_ste_vec_10() + RETURNS jsonb +AS $$ + BEGIN + RETURN '{ + "sv": [ + { + "b": "7b4ffe5d60e4e4300dc3e28d9c300c87", + "c": "mBbLGB9xHAGzLvUj-`@Wmf=IhD87n7r3ir3n!Sk6AKir_YawR=0c>pk(OydB;ntIEXK~c>V&4>)rNkfF eql_v2_encrypted[] +-- a [ +-- 1 +-- ] +-- + +-- ORIGINAL $.a encoding +-- { +-- "b": "8258356162d2415d55244abf49e40da3", +-- "c": "mBbL9j9(QoRD)R+z?=Fvn#=FR9iI)K4Nzk-ea`~#Lx@wBSDPSmkp-h+tNEHoo@T@#vwh?Ejvk%78G}b+je+xufQA5mSwHSid)iEOkg@>mpuh", +-- "s": "f510853730e1c3dbd31b86963f029dd5" +-- }, + +DROP FUNCTION IF EXISTS get_array_ste_vec(); +CREATE FUNCTION get_array_ste_vec() + RETURNS jsonb +AS $$ + BEGIN + RETURN '{ + "sv": [ + { + "b": "7b4ffe5d60e4e4300dc3e28d9c300c87", + "c": "mBbL9j9(QoRD)R+z?=Fvn#=FRIg79JJM`MCq+nE0*U^ca-cViL884d-TInfY&E9HW@X>!U&lkYne2!EecKG8xwLYb0X#y7|05rrPvwh?Ejvk%78G}b+je+xufQA5mSwHSid)iEOkg@>mpuh", + "s": "bca213de9ccce676fa849ff9c4807963" + }, + { + "c": "mBbL9j9(QoRD)R+z?=Fvn#=FR6Z{(4c^$CD^7q>z{xl^%5S4=m#2~YMW7y15TC<^_oBO-6ni$TotY#2~YMz{xl^%5S4=m#2~YMW7y15TC<^_oBO-6ni$TotY#2~YMoG#B*Y-IedG9!9-X`ygGXYGf%A%hh5&w9KkiR^+DvtjvHG(8jjwtt=6Wr{!%WJ?vt(v&0~?edG9!9-X`ygGXYGf%A%hh5&w9KkiR^+DvtjvH'$.n' -> '2517068c0d1f9d4d41d2c666211f785e' + -- e->'2517068c0d1f9d4d41d2c666211f785e' + -- e->>'2517068c0d1f9d4d41d2c666211f785e' ciphertext/c + + RETURN '{ + "sv": [ + { + "b": "7b4ffe5d60e4e4300dc3e28d9c300c87", + "c": "mBbM0#UZON2jQ3@LiWcvns2YfD7#?5ZXlp8Wk1R*iA%o6cD0VZWqPY%l%_z!JC9wAR4?XKSouV_AjBXFod39C7TF-SiCD-NgkG)l%Vw=l!tX>H*Pjq@SFR7iRajU#?{(K%x=#2^Zs|F~fm*&w!wSjZQIUaj-XX01=c??f8cq8*Vf?zEu5", + "ocv": "af96e1dabbec581f36d71e3a48ffb427f54832851b4fefa6989887ccaf7e038f66f8cb40e6959458", + "s": "a7cea93975ed8c01f861ccb6bd082784" + }, + { + "c": "mBbM0#UZON2jQ3@LiWcvns2Yf6y3L;hykEh`}*fX#aF;n*=>+*o5Uarod39C7TF-SiCD-NgkG)l%Vw=l!tX>H*PkiFH&De7C+d}sDugsc?JuI*>$AsG83nsXvrND0-(S", + "b": "7b4ffe5d60e4e4300dc3e28d9c300c87", + "s": "bca213de9ccce676fa849ff9c4807963" + }, + { + "c": "mBbK0Cob5dQ5Jki69vRd75f9k8Rn)lVSgZ9Q3jQYu)}sv8};==6AExb8MwqC=TCnxQeQ_FKiJQxgbDw71`CJTb)@Vv6Q`bN$sH2{puh", + "ocv": "af96e1dabbec5913707844664eb160923982fdec75bda4bcd063e26b4254a9f334ce7ebc2612713c", + "s": "a7cea93975ed8c01f861ccb6bd082784" + }, + { + "c": "mBbK0Cob5dQ5Jki69vRd75f9k6yc;BV`COqamPOX6P`g5TMr)AeZ(N=Pk%%2`Uq=={*w3hh3IBNp3y0Ztr0g;ir=DoZ9TNhezy", + "ocf": "b0c13d4a4a9ffcb2ef8629d60d5e32db453fad8792b2450d02f37ec5fe207b42da30093fd14c4975c9b192ecbf939b2d5a56a7ae2db1254e6532aa7569971462", + "s": "2517068c0d1f9d4d41d2c666211f785e" + } + ] + }'::jsonb; + END; +$$ LANGUAGE plpgsql; + + +-- -- +-- -- +-- +-- Creates a table with an encrypted column for testing +-- +-- JSON -- '{"hello": "world", "n": 42}' +-- +-- Paths +-- $ -> bca213de9ccce676fa849ff9c4807963 +-- $.hello -> a7cea93975ed8c01f861ccb6bd082784 +-- $.n -> 2517068c0d1f9d4d41d2c666211f785e +-- +-- -- +-- -- +DROP FUNCTION IF EXISTS create_encrypted_json(integer); +CREATE FUNCTION create_encrypted_json(id integer) + RETURNS eql_v2_encrypted +AS $$ + DECLARE + s text; + m jsonb; + start integer; + stop integer; + random_key text; + random_val text; + sv jsonb; + ore_term jsonb; + BEGIN + + start := (10 * id); + stop := (10 * id) + 5; + m := array_to_json(array(SELECT generate_series(start, stop))); + + select substr(md5(random()::text), 1, 25) INTO random_key; + select substr(md5(random()::text), 1, 25) INTO random_val; + + CASE id + WHEN 1 THEN + sv := get_numeric_ste_vec_10(); + WHEN 2 THEN + sv := get_numeric_ste_vec_20(); + WHEN 3 THEN + sv := get_numeric_ste_vec_30(); + ELSE + sv := get_numeric_ste_vec_42(); + END CASE; + + + SELECT ore.e FROM ore WHERE ore.id = start INTO ore_term; + + -- PERFORM eql_v2.log('ore_term: ', ore_term::text); + + s := format( + '{ + "%s": "%s", + "c": "ciphertext", + "i": { + "t": "encrypted", + "c": "e" + }, + "u": "unique.%s", + "b": "blake3.%s", + "m": %s + }', + random_key, + random_val, + id, id, m); + + s := s::jsonb || sv || ore_term; + + -- PERFORM eql_v2.log('json: %', s); + + RETURN s::eql_v2_encrypted; + END; +$$ LANGUAGE plpgsql; + + +DROP FUNCTION IF EXISTS create_encrypted_json(integer, VARIADIC indexes text[]); +CREATE FUNCTION create_encrypted_json(id integer, VARIADIC indexes text[]) + RETURNS eql_v2_encrypted +AS $$ + DECLARE + j jsonb; + BEGIN + j := create_encrypted_json(id); + + j := ( + SELECT jsonb_object_agg(key, value) + FROM jsonb_each(j) + WHERE key = ANY(indexes) + ); + + RETURN j::eql_v2_encrypted; + + END; +$$ LANGUAGE plpgsql; + + +DROP FUNCTION IF EXISTS create_encrypted_json(VARIADIC indexes text[]); +CREATE FUNCTION create_encrypted_json(VARIADIC indexes text[]) + RETURNS eql_v2_encrypted +AS $$ + DECLARE + default_indexes text[]; + j jsonb; + BEGIN + + default_indexes := ARRAY['c', 'i', 'v']; + + j := create_encrypted_json(1); + + j := ( + SELECT jsonb_object_agg(key, value) + FROM jsonb_each(j) + WHERE key = ANY(indexes || default_indexes) + ); + + RETURN j::eql_v2_encrypted; + + END; +$$ LANGUAGE plpgsql; + + + +DROP FUNCTION IF EXISTS create_encrypted_ore_json(val integer); +CREATE FUNCTION create_encrypted_ore_json(val integer) + RETURNS eql_v2_encrypted +AS $$ + DECLARE + e eql_v2_encrypted; + ore_term jsonb; + BEGIN + EXECUTE format('SELECT ore.e FROM ore WHERE id = %s', val) INTO ore_term; + e := create_encrypted_json('o')::jsonb || ore_term; + RETURN e::eql_v2_encrypted; + END; +$$ LANGUAGE plpgsql; + + +DROP FUNCTION IF EXISTS create_encrypted_json(); +CREATE FUNCTION create_encrypted_json() + RETURNS eql_v2_encrypted +AS $$ + DECLARE + id integer; + j jsonb; + BEGIN + id := trunc(random() * 1000 + 1); + j := create_encrypted_json(id); + RETURN j::eql_v2_encrypted; + END; +$$ LANGUAGE plpgsql; + + +DROP FUNCTION IF EXISTS seed_encrypted(eql_v2_encrypted); +CREATE FUNCTION seed_encrypted(e eql_v2_encrypted) + RETURNS void +AS $$ + BEGIN + INSERT INTO encrypted (e) VALUES (e); + END; +$$ LANGUAGE plpgsql; + + +-- +-- Truncates and creates base test data +-- +DROP FUNCTION IF EXISTS seed_encrypted_json(); +CREATE FUNCTION seed_encrypted_json() + RETURNS void +AS $$ + BEGIN + PERFORM truncate_table_with_encrypted(); + PERFORM seed_encrypted(create_encrypted_json(1)); + PERFORM seed_encrypted(create_encrypted_json(2)); + PERFORM seed_encrypted(create_encrypted_json(3)); + END; +$$ LANGUAGE plpgsql; + + +-- +-- Creates a table with an encrypted column for testing +-- +DROP FUNCTION IF EXISTS drop_table_with_encrypted(); +CREATE FUNCTION drop_table_with_encrypted() + RETURNS void +AS $$ + BEGIN + DROP TABLE IF EXISTS encrypted; +END; +$$ LANGUAGE plpgsql; + + +-- +-- Convenience function to describe a test +-- +DROP FUNCTION IF EXISTS describe(text); +CREATE FUNCTION describe(s text) + RETURNS void +AS $$ + BEGIN + RAISE NOTICE '%', s; +END; +$$ LANGUAGE plpgsql; + + +-- +-- Assert the the provided SQL statement returns a non-null result +-- +DROP FUNCTION IF EXISTS assert_result(describe text, sql text); + +CREATE FUNCTION assert_result(describe text, sql text) + RETURNS void +AS $$ + DECLARE + result record; + BEGIN + RAISE NOTICE '%', describe; + EXECUTE sql into result; + + if result IS NULL THEN + RAISE NOTICE 'ASSERT RESULT FAILED'; + RAISE NOTICE '%', regexp_replace(sql, '^\s+|\s*$', '', 'g'); + ASSERT false; + END IF; + + END; +$$ LANGUAGE plpgsql; + + +-- +-- Assert the the provided SQL statement returns a non-null result +-- +DROP FUNCTION IF EXISTS assert_result(describe text, sql text, result text); + +CREATE FUNCTION assert_result(describe text, sql text, expected text) + RETURNS void +AS $$ + DECLARE + result text; + BEGIN + RAISE NOTICE '%', describe; + EXECUTE sql into result; + + if result <> expected THEN + RAISE NOTICE 'ASSERT EXPECTED RESULT FAILED'; + RAISE NOTICE 'Expected: %', expected; + RAISE NOTICE 'Result: %', result; + RAISE NOTICE '%', regexp_replace(sql, '^\s+|\s*$', '', 'g'); + ASSERT false; + END IF; + + END; +$$ LANGUAGE plpgsql; + +-- +-- Assert the the provided SQL statement returns a non-null result +-- +DROP FUNCTION IF EXISTS assert_id(describe text, sql text, id integer); + +CREATE FUNCTION assert_id(describe text, sql text, id integer) + RETURNS void +AS $$ + DECLARE + result_id integer; + BEGIN + RAISE NOTICE '%', describe; + EXECUTE sql into result_id; + + IF result_id <> id THEN + RAISE NOTICE 'ASSERT ID FAILED'; + RAISE NOTICE 'Expected row with id % but returned %', id, result_id; + RAISE NOTICE '%', regexp_replace(sql, '^\s+|\s*$', '', 'g'); + ASSERT false; + END IF; + + END; +$$ LANGUAGE plpgsql; + + +-- +-- Assert the the provided SQL statement returns a non-null result +-- +DROP FUNCTION IF EXISTS assert_no_result(describe text, sql text); + +CREATE FUNCTION assert_no_result(describe text, sql text) + RETURNS void +AS $$ + DECLARE + result record; + BEGIN + RAISE NOTICE '%', describe; + EXECUTE sql into result; + + IF result IS NOT NULL THEN + RAISE NOTICE 'ASSERT NO RESULT FAILED'; + RAISE NOTICE '%', regexp_replace(sql, '^\s+|\s*$', '', 'g'); + ASSERT false; + END IF; + + END; +$$ LANGUAGE plpgsql; + + + +-- +-- Assert the the provided SQL statement returns a non-null result +-- +DROP FUNCTION IF EXISTS assert_count(describe text, sql text, expected integer); + +CREATE FUNCTION assert_count(describe text, sql text, expected integer) + RETURNS void +AS $$ + DECLARE + result integer; + BEGIN + RAISE NOTICE '%', describe; + + -- Remove any trailing ; so that the query can be wrapped with count(*) below + sql := TRIM(TRAILING ';' FROM sql); + + EXECUTE format('SELECT COUNT(*) FROM (%s) as q', sql) INTO result; + + if result <> expected THEN + RAISE NOTICE 'ASSERT COUNT FAILED'; + RAISE NOTICE 'Expected % rows and returned %', expected, result; + RAISE NOTICE '%', regexp_replace(sql, '^\s+|\s*$', '', 'g'); + ASSERT false; + END IF; + + END; +$$ LANGUAGE plpgsql; + + + +-- +-- Assert the the provided SQL statement raises an exception +-- +DROP FUNCTION IF EXISTS assert_exception(describe text, sql text); + +CREATE FUNCTION assert_exception(describe text, sql text) + RETURNS void +AS $$ + BEGIN + RAISE NOTICE '%', describe; + + BEGIN + EXECUTE sql; + RAISE NOTICE 'ASSERT EXCEPTION FAILED'; + RAISE NOTICE 'EXPECTED STATEMENT TO RAISE EXCEPTION'; + RAISE NOTICE '%', regexp_replace(sql, '^\s+|\s*$', '', 'g'); + ASSERT false; + EXCEPTION + WHEN OTHERS THEN + ASSERT true; + END; + + END; +$$ LANGUAGE plpgsql; diff --git a/tests/version.sql b/tests/version.sql deleted file mode 100644 index 5330ad10..00000000 --- a/tests/version.sql +++ /dev/null @@ -1,8 +0,0 @@ -\set ON_ERROR_STOP on - -DO $$ - BEGIN - ASSERT (SELECT true WHERE cs_eql_version() = 'DEV'); - - END; -$$ LANGUAGE plpgsql; diff --git a/utils/uninstall.sql b/utils/uninstall.sql index 578af1ca..17d5ad2b 100644 --- a/utils/uninstall.sql +++ b/utils/uninstall.sql @@ -1,23 +1,23 @@ -- Drop constraints on domains -ALTER DOMAIN cs_encrypted_v1 DROP CONSTRAINT IF EXISTS cs_encrypted_v1_check; -ALTER DOMAIN cs_configuration_data_v1 DROP CONSTRAINT IF EXISTS cs_configuration_data_v1_check; +ALTER DOMAIN cs_encrypted_v2 DROP CONSTRAINT IF EXISTS cs_encrypted_v2_check; +ALTER DOMAIN cs_configuration_data_v2 DROP CONSTRAINT IF EXISTS cs_configuration_data_v2_check; -- Drop functions -DROP FUNCTION IF EXISTS cs_count_encrypted_with_active_config_v1(text, text); -DROP FUNCTION IF EXISTS cs_rename_encrypted_columns_v1(); -DROP FUNCTION IF EXISTS cs_create_encrypted_columns_v1(); -DROP FUNCTION IF EXISTS cs_ready_for_encryption_v1(); -DROP FUNCTION IF EXISTS cs_select_target_columns_v1(); -DROP FUNCTION IF EXISTS cs_select_pending_columns_v1(); -DROP FUNCTION IF EXISTS _cs_diff_config_v1(jsonb, jsonb); -DROP FUNCTION IF EXISTS cs_add_column_v1(text, text); -DROP FUNCTION IF EXISTS cs_remove_column_v1(text, text); -DROP FUNCTION IF EXISTS cs_add_index_v1(text, text, text, text, jsonb); -DROP FUNCTION IF EXISTS cs_remove_index_v1(text, text, text); -DROP FUNCTION IF EXISTS cs_modify_index_v1(text, text, text, text, jsonb); -DROP FUNCTION IF EXISTS cs_encrypt_v1(); -DROP FUNCTION IF EXISTS cs_activate_v1(); -DROP FUNCTION IF EXISTS cs_discard_v1(); +DROP FUNCTION IF EXISTS cs_count_encrypted_with_active_config_v2(text, text); +DROP FUNCTION IF EXISTS cs_rename_encrypted_columns_v2(); +DROP FUNCTION IF EXISTS cs_create_encrypted_columns_v2(); +DROP FUNCTION IF EXISTS cs_ready_for_encryption_v2(); +DROP FUNCTION IF EXISTS cs_select_target_columns_v2(); +DROP FUNCTION IF EXISTS cs_select_pending_columns_v2(); +DROP FUNCTION IF EXISTS _cs_diff_config_v2(jsonb, jsonb); +DROP FUNCTION IF EXISTS cs_add_column_v2(text, text); +DROP FUNCTION IF EXISTS cs_remove_column_v2(text, text); +DROP FUNCTION IF EXISTS cs_add_index_v2(text, text, text, text, jsonb); +DROP FUNCTION IF EXISTS cs_remove_index_v2(text, text, text); +DROP FUNCTION IF EXISTS cs_modify_index_v2(text, text, text, text, jsonb); +DROP FUNCTION IF EXISTS cs_encrypt_v2(); +DROP FUNCTION IF EXISTS cs_activate_v2(); +DROP FUNCTION IF EXISTS cs_discard_v2(); DROP FUNCTION IF EXISTS cs_refresh_encrypt_config(); DROP FUNCTION IF EXISTS _cs_config_default(jsonb); DROP FUNCTION IF EXISTS _cs_config_match_default(); @@ -25,44 +25,44 @@ DROP FUNCTION IF EXISTS _cs_config_add_table(text, jsonb); DROP FUNCTION IF EXISTS _cs_config_add_column(text, text, jsonb); DROP FUNCTION IF EXISTS _cs_config_add_cast(text, text, text, jsonb); DROP FUNCTION IF EXISTS _cs_config_add_index(text, text, text, jsonb, jsonb); -DROP FUNCTION IF EXISTS cs_ciphertext_v1(jsonb); -DROP FUNCTION IF EXISTS cs_ciphertext_v1_v0(jsonb); -DROP FUNCTION IF EXISTS cs_ciphertext_v1_v0_0(jsonb); -DROP FUNCTION IF EXISTS cs_match_v1(jsonb); -DROP FUNCTION IF EXISTS cs_match_v1_v0(jsonb); -DROP FUNCTION IF EXISTS cs_match_v1_v0_0(jsonb); -DROP FUNCTION IF EXISTS cs_unique_v1(jsonb); -DROP FUNCTION IF EXISTS cs_unique_v1_v0(jsonb); -DROP FUNCTION IF EXISTS cs_unique_v1_v0_0(jsonb); -DROP FUNCTION IF EXISTS cs_ste_vec_v1(jsonb); -DROP FUNCTION IF EXISTS cs_ste_vec_v1_v0(jsonb); -DROP FUNCTION IF EXISTS cs_ste_vec_v1_v0_0(jsonb); -DROP FUNCTION IF EXISTS cs_ore_64_8_v1(jsonb); -DROP FUNCTION IF EXISTS cs_ore_64_8_v1_v0(jsonb); -DROP FUNCTION IF EXISTS cs_ore_64_8_v1_v0_0(jsonb); -DROP FUNCTION IF EXISTS _cs_text_to_ore_64_8_v1_term_v1_0(text) CASCADE; -DROP FUNCTION IF EXISTS cs_check_encrypted_v1(jsonb); +DROP FUNCTION IF EXISTS cs_ciphertext_v2(jsonb); +DROP FUNCTION IF EXISTS cs_ciphertext_v2_v0(jsonb); +DROP FUNCTION IF EXISTS cs_ciphertext_v2_v0_0(jsonb); +DROP FUNCTION IF EXISTS cs_match_v2(jsonb); +DROP FUNCTION IF EXISTS cs_match_v2_v0(jsonb); +DROP FUNCTION IF EXISTS cs_match_v2_v0_0(jsonb); +DROP FUNCTION IF EXISTS cs_unique_v2(jsonb); +DROP FUNCTION IF EXISTS cs_unique_v2_v0(jsonb); +DROP FUNCTION IF EXISTS cs_unique_v2_v0_0(jsonb); +DROP FUNCTION IF EXISTS cs_ste_vec_v2(jsonb); +DROP FUNCTION IF EXISTS cs_ste_vec_v2_v0(jsonb); +DROP FUNCTION IF EXISTS cs_ste_vec_v2_v0_0(jsonb); +DROP FUNCTION IF EXISTS cs_ore_64_8_v2(jsonb); +DROP FUNCTION IF EXISTS cs_ore_64_8_v2_v0(jsonb); +DROP FUNCTION IF EXISTS cs_ore_64_8_v2_v0_0(jsonb); +DROP FUNCTION IF EXISTS _cs_text_to_ore_64_8_v2_term_v2_0(text) CASCADE; +DROP FUNCTION IF EXISTS cs_check_encrypted_v2(jsonb); DROP FUNCTION IF EXISTS _cs_encrypted_check_kind(jsonb); DROP FUNCTION IF EXISTS _cs_config_check_indexes(jsonb); DROP FUNCTION IF EXISTS _cs_config_check_cast(jsonb); -- Drop cast -DROP CAST IF EXISTS (text AS ore_64_8_v1_term); +DROP CAST IF EXISTS (text AS ore_64_8_v2_term); -- Drop indexes -DROP INDEX IF EXISTS cs_configuration_v1_index_active; -DROP INDEX IF EXISTS cs_configuration_v1_index_pending; -DROP INDEX IF EXISTS cs_configuration_v1_index_encrypting; +DROP INDEX IF EXISTS cs_configuration_v2_index_active; +DROP INDEX IF EXISTS cs_configuration_v2_index_pending; +DROP INDEX IF EXISTS cs_configuration_v2_index_encrypting; -- Drop table -DROP TABLE IF EXISTS cs_configuration_v1; +DROP TABLE IF EXISTS cs_configuration_v2; -- Drop domains -DROP DOMAIN IF EXISTS cs_match_index_v1; -DROP DOMAIN IF EXISTS cs_unique_index_v1; -DROP DOMAIN IF EXISTS cs_ste_vec_index_v1; -DROP DOMAIN IF EXISTS cs_encrypted_v1; -- Note: This domain cannot be dropped if it's in use -DROP DOMAIN IF EXISTS cs_configuration_data_v1; +DROP DOMAIN IF EXISTS cs_match_index_v2; +DROP DOMAIN IF EXISTS cs_unique_index_v2; +DROP DOMAIN IF EXISTS cs_ste_vec_index_v2; +DROP DOMAIN IF EXISTS cs_encrypted_v2; -- Note: This domain cannot be dropped if it's in use +DROP DOMAIN IF EXISTS cs_configuration_data_v2; -- Drop type -DROP TYPE IF EXISTS cs_configuration_state_v1; +DROP TYPE IF EXISTS cs_configuration_state_v2;