From 1f13381bf9b20106ed36f4f46098abaa3aeac0cd Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 17 Oct 2024 09:51:28 +0200 Subject: [PATCH 1/8] Add json_keys function. --- packages/sync-rules/src/sql_functions.ts | 27 +++++++++++++++++++ .../sync-rules/test/src/sql_functions.test.ts | 24 +++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/packages/sync-rules/src/sql_functions.ts b/packages/sync-rules/src/sql_functions.ts index 715f1494f..a6832ae5d 100644 --- a/packages/sync-rules/src/sql_functions.ts +++ b/packages/sync-rules/src/sql_functions.ts @@ -223,6 +223,32 @@ const json_valid: DocumentedSqlFunction = { documentation: 'Returns 1 if valid, 0 if invalid' }; +const json_keys: DocumentedSqlFunction = { + debugName: 'json_keys', + call(json: SqliteValue) { + const jsonString = castAsText(json); + if (jsonString == null) { + return null; + } + + const jsonParsed = JSONBig.parse(jsonString); + if (typeof jsonParsed != 'object') { + throw new Error(`Cannot call json_keys on a scalar`); + } else if (Array.isArray(jsonParsed)) { + throw new Error(`Cannot call json_keys on an array`); + } + const keys = Object.keys(jsonParsed as {}); + // Keys are always strings, safe to use plain JSON. + return JSON.stringify(keys); + }, + parameters: [{ name: 'json', type: ExpressionType.ANY, optional: false }], + getReturnType(args) { + // TODO: proper nullable types + return ExpressionType.TEXT; + }, + detail: 'Returns the keys of a JSON object as a JSON array' +}; + const unixepoch: DocumentedSqlFunction = { debugName: 'unixepoch', call(value?: SqliteValue, specifier?: SqliteValue, specifier2?: SqliteValue) { @@ -397,6 +423,7 @@ export const SQL_FUNCTIONS_NAMED = { json_extract, json_array_length, json_valid, + json_keys, unixepoch, datetime, st_asgeojson, diff --git a/packages/sync-rules/test/src/sql_functions.test.ts b/packages/sync-rules/test/src/sql_functions.test.ts index 525133c3b..9c11b9352 100644 --- a/packages/sync-rules/test/src/sql_functions.test.ts +++ b/packages/sync-rules/test/src/sql_functions.test.ts @@ -44,6 +44,30 @@ describe('SQL functions', () => { expect(fn.json_array_length(`{"a":[1,2,3,4]}`)).toEqual(0n); }); + test('json_keys', () => { + expect(fn.json_keys(`{"a": 1, "b": "2", "0": "test", "c": {"d": "e"}}`)).toEqual(`["0","a","b","c"]`); + expect(fn.json_keys(`{}`)).toEqual(`[]`); + expect(fn.json_keys(null)).toEqual(null); + expect(fn.json_keys()).toEqual(null); + expect(() => fn.json_keys(`{"a": 1, "a": 2}`)).toThrow(); + expect(() => fn.json_keys(`[1,2,3]`)).toThrow(); + expect(() => fn.json_keys(3)).toThrow(); + }); + + test('json_valid', () => { + expect(fn.json_valid(`{"a": 1, "b": "2", "0": "test", "c": {"d": "e"}}`)).toEqual(1n); + expect(fn.json_valid(`{}`)).toEqual(1n); + expect(fn.json_valid(null)).toEqual(0n); + expect(fn.json_valid()).toEqual(0n); + expect(fn.json_valid(`{"a": 1, "a": 2}`)).toEqual(0n); + expect(fn.json_valid(`[1,2,3]`)).toEqual(1n); + expect(fn.json_valid(3)).toEqual(1n); + expect(fn.json_valid('test')).toEqual(0n); + expect(fn.json_valid('"test"')).toEqual(1n); + expect(fn.json_valid('true')).toEqual(1n); + expect(fn.json_valid('TRUE')).toEqual(0n); + }); + test('typeof', () => { expect(fn.typeof(null)).toEqual('null'); expect(fn.typeof('test')).toEqual('text'); From 093e2b29808b3e30fdc718dab72f4873dfac7ca5 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 17 Oct 2024 10:15:23 +0200 Subject: [PATCH 2/8] Add substring function. --- packages/sync-rules/src/sql_functions.ts | 56 +++++++++++++++++++ .../sync-rules/test/src/sql_functions.test.ts | 16 ++++++ 2 files changed, 72 insertions(+) diff --git a/packages/sync-rules/src/sql_functions.ts b/packages/sync-rules/src/sql_functions.ts index a6832ae5d..de74b0e5f 100644 --- a/packages/sync-rules/src/sql_functions.ts +++ b/packages/sync-rules/src/sql_functions.ts @@ -70,6 +70,61 @@ const lower: DocumentedSqlFunction = { detail: 'Convert text to lower case' }; +const substring: DocumentedSqlFunction = { + debugName: 'substring', + call(value: SqliteValue, start: SqliteValue, length?: SqliteValue) { + const text = castAsText(value); + if (text == null) { + return null; + } + const startIndex = cast(start, 'integer') as bigint | null; + if (startIndex == null) { + return null; + } + if (length === null) { + // Different from undefined in this case, to match SQLite behavior + return null; + } + const castLength = cast(length ?? null, 'integer') as bigint | null; + let realLength: number; + if (castLength == null) { + // undefined (not specified) + realLength = text.length + 1; // +1 to account for the start = 0 special case + } else { + realLength = Number(castLength); + } + + let realStart = 0; + if (startIndex < 0n) { + realStart = Math.max(0, text.length + Number(startIndex)); + } else if (startIndex == 0n) { + // Weird special case + realStart = 0; + realLength -= 1; + } else { + realStart = Number(startIndex) - 1; + } + + if (realLength < 0) { + // Negative length means we return that many characters _before_ + // the start index. + return text.substring(realStart + realLength, realStart); + } + + return text.substring(realStart, realStart + realLength); + }, + parameters: [ + { name: 'value', type: ExpressionType.TEXT, optional: false }, + { name: 'start', type: ExpressionType.INTEGER, optional: false }, + { name: 'length', type: ExpressionType.INTEGER, optional: true } + ], + getReturnType(args) { + return ExpressionType.TEXT; + }, + detail: 'Compute a substring', + documentation: 'The start index starts at 1. If no length is specified, the remainder of the string is returned.' +}; + const hex: DocumentedSqlFunction = { debugName: 'hex', call(value: SqliteValue) { @@ -415,6 +470,7 @@ const st_y: DocumentedSqlFunction = { export const SQL_FUNCTIONS_NAMED = { upper, lower, + substring, hex, length, base64, diff --git a/packages/sync-rules/test/src/sql_functions.test.ts b/packages/sync-rules/test/src/sql_functions.test.ts index 9c11b9352..1a155d6af 100644 --- a/packages/sync-rules/test/src/sql_functions.test.ts +++ b/packages/sync-rules/test/src/sql_functions.test.ts @@ -125,6 +125,22 @@ describe('SQL functions', () => { expect(fn.lower(Uint8Array.of(0x61, 0x62, 0x43))).toEqual('abc'); }); + test('substring', () => { + expect(fn.substring(null)).toEqual(null); + expect(fn.substring('abc')).toEqual(null); + expect(fn.substring('abcde', 2, 3)).toEqual('bcd'); + expect(fn.substring('abcde', 2)).toEqual('bcde'); + expect(fn.substring('abcde', 2, null)).toEqual(null); + expect(fn.substring('abcde', 0, 1)).toEqual(''); + expect(fn.substring('abcde', 0, 2)).toEqual('a'); + expect(fn.substring('abcde', 1, 2)).toEqual('ab'); + expect(fn.substring('abcde', -2)).toEqual('de'); + expect(fn.substring('abcde', -2, 1)).toEqual('d'); + expect(fn.substring('abcde', 6, -5)).toEqual('abcde'); + expect(fn.substring('abcde', 5, -2)).toEqual('cd'); + expect(fn.substring('2023-06-28 14:12:00.999Z', 1, 10)).toEqual('2023-06-28'); + }); + test('cast', () => { expect(cast(null, 'text')).toEqual(null); expect(cast(null, 'integer')).toEqual(null); From 0f90b028b57c50dbcfe7e19e122043a5be969a00 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 17 Oct 2024 10:30:46 +0200 Subject: [PATCH 3/8] Add changeset. --- .changeset/thirty-llamas-carry.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/thirty-llamas-carry.md diff --git a/.changeset/thirty-llamas-carry.md b/.changeset/thirty-llamas-carry.md new file mode 100644 index 000000000..c789a7f26 --- /dev/null +++ b/.changeset/thirty-llamas-carry.md @@ -0,0 +1,5 @@ +--- +'@powersync/service-sync-rules': minor +--- + +Support substring and json_keys functions in sync rules From 0689dcfcb8766ce7a9655a27ce3cb717205286c4 Mon Sep 17 00:00:00 2001 From: Conrad Hofmeyr Date: Tue, 29 Oct 2024 05:07:43 -0600 Subject: [PATCH 4/8] README MySQL updates --- .github/workflows/test.yml | 2 +- DEVELOP.md | 4 ++-- README.md | 2 +- service/README.md | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9c2b757c2..a1a8d4899 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ on: jobs: test-service-container-build: - name: Build and Test Powersync Service + name: Build and Test PowerSync Service runs-on: ubuntu-latest steps: - name: Checkout diff --git a/DEVELOP.md b/DEVELOP.md index 23eb51d4d..d757be503 100644 --- a/DEVELOP.md +++ b/DEVELOP.md @@ -18,7 +18,7 @@ pnpm build ## Dependent Services -The PowerSync service requires Postgres and MongoDB server connections. These configuration details can be specified in a `powersync.yaml` (or JSON) configuration file. +The PowerSync Service requires Postgres and MongoDB server connections. These configuration details can be specified in a `powersync.yaml` (or JSON) configuration file. See the [self-hosting demo](https://github.com/powersync-ja/self-host-demo) for demos of starting these services. @@ -36,7 +36,7 @@ One method to obtain access is to add the following to `/etc/hosts` (on Unix-lik 127.0.0.1 mongo ``` -This will start all the services defined in the Self hosting demo except for the PowerSync service - which will be started from this repository. +This will start all the services defined in the Self hosting demo except for the PowerSync Service - which will be started from this repository. ## Local Configuration diff --git a/README.md b/README.md index dd5f13b31..06a11cf0a 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@

-*[PowerSync](https://www.powersync.com) is a sync engine for building local-first apps with instantly-responsive UI/UX and simplified state transfer. Syncs between SQLite on the client-side and Postgres or MongoDB on the server-side (MySQL coming soon).* +*[PowerSync](https://www.powersync.com) is a sync engine for building local-first apps with instantly-responsive UI/UX and simplified state transfer. Syncs between SQLite on the client-side and Postgres, MongoDB or MySQL on the server-side.* # PowerSync Service diff --git a/service/README.md b/service/README.md index b5af8acf2..f884271c3 100644 --- a/service/README.md +++ b/service/README.md @@ -2,7 +2,7 @@

-*[PowerSync](https://www.powersync.com) is a sync engine for building local-first apps with instantly-responsive UI/UX and simplified state transfer. Syncs between SQLite on the client-side and Postgres or MongoDB on the server-side (MySQL coming soon).* +*[PowerSync](https://www.powersync.com) is a sync engine for building local-first apps with instantly-responsive UI/UX and simplified state transfer. Syncs between SQLite on the client-side and Postgres, MongoDB or MySQL on the server-side.* # Quick reference From 06250954e5ec2f5db67369afaee9bb3a0555c382 Mon Sep 17 00:00:00 2001 From: Conrad Hofmeyr Date: Tue, 29 Oct 2024 05:09:15 -0600 Subject: [PATCH 5/8] Minor formatting: --- DEVELOP.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEVELOP.md b/DEVELOP.md index d757be503..3b88419e6 100644 --- a/DEVELOP.md +++ b/DEVELOP.md @@ -36,7 +36,7 @@ One method to obtain access is to add the following to `/etc/hosts` (on Unix-lik 127.0.0.1 mongo ``` -This will start all the services defined in the Self hosting demo except for the PowerSync Service - which will be started from this repository. +This will start all the services defined in the self-hosting demo except for the PowerSync Service - which will be started from this repository. ## Local Configuration From 6e812a55a2aaeb8a304a7664e2548f779ee4801a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 29 Oct 2024 11:33:50 +0000 Subject: [PATCH 6/8] Version Packages --- .changeset/funny-pianos-allow.md | 5 ----- .changeset/green-books-shout.md | 5 ----- .changeset/thirty-llamas-carry.md | 5 ----- packages/service-core/CHANGELOG.md | 9 +++++++++ packages/service-core/package.json | 2 +- packages/sync-rules/CHANGELOG.md | 6 ++++++ packages/sync-rules/package.json | 2 +- service/CHANGELOG.md | 10 ++++++++++ service/package.json | 2 +- test-client/CHANGELOG.md | 8 ++++++++ test-client/package.json | 2 +- 11 files changed, 37 insertions(+), 19 deletions(-) delete mode 100644 .changeset/funny-pianos-allow.md delete mode 100644 .changeset/green-books-shout.md delete mode 100644 .changeset/thirty-llamas-carry.md diff --git a/.changeset/funny-pianos-allow.md b/.changeset/funny-pianos-allow.md deleted file mode 100644 index 3df90e3bc..000000000 --- a/.changeset/funny-pianos-allow.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@powersync/service-core': patch ---- - -Fix "operation exceeded time limit" error diff --git a/.changeset/green-books-shout.md b/.changeset/green-books-shout.md deleted file mode 100644 index 1b2ef8139..000000000 --- a/.changeset/green-books-shout.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@powersync/service-core': patch ---- - -Fix storageStats error in metrics endpoint when collections don't exist. diff --git a/.changeset/thirty-llamas-carry.md b/.changeset/thirty-llamas-carry.md deleted file mode 100644 index c789a7f26..000000000 --- a/.changeset/thirty-llamas-carry.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@powersync/service-sync-rules': minor ---- - -Support substring and json_keys functions in sync rules diff --git a/packages/service-core/CHANGELOG.md b/packages/service-core/CHANGELOG.md index e76e019d4..eb39c2e21 100644 --- a/packages/service-core/CHANGELOG.md +++ b/packages/service-core/CHANGELOG.md @@ -1,5 +1,14 @@ # @powersync/service-core +## 0.8.6 + +### Patch Changes + +- 2d3bb6a: Fix "operation exceeded time limit" error +- 17a6db0: Fix storageStats error in metrics endpoint when collections don't exist. +- Updated dependencies [0f90b02] + - @powersync/service-sync-rules@0.20.0 + ## 0.8.5 ### Patch Changes diff --git a/packages/service-core/package.json b/packages/service-core/package.json index 686f01541..1e7129664 100644 --- a/packages/service-core/package.json +++ b/packages/service-core/package.json @@ -5,7 +5,7 @@ "publishConfig": { "access": "public" }, - "version": "0.8.5", + "version": "0.8.6", "main": "dist/index.js", "license": "FSL-1.1-Apache-2.0", "type": "module", diff --git a/packages/sync-rules/CHANGELOG.md b/packages/sync-rules/CHANGELOG.md index 099bf11db..d74892def 100644 --- a/packages/sync-rules/CHANGELOG.md +++ b/packages/sync-rules/CHANGELOG.md @@ -1,5 +1,11 @@ # @powersync/service-sync-rules +## 0.20.0 + +### Minor Changes + +- 0f90b02: Support substring and json_keys functions in sync rules + ## 0.19.0 ### Minor Changes diff --git a/packages/sync-rules/package.json b/packages/sync-rules/package.json index 4108f6bf7..580d103d5 100644 --- a/packages/sync-rules/package.json +++ b/packages/sync-rules/package.json @@ -1,7 +1,7 @@ { "name": "@powersync/service-sync-rules", "repository": "https://github.com/powersync-ja/powersync-service", - "version": "0.19.0", + "version": "0.20.0", "main": "dist/index.js", "types": "dist/index.d.ts", "license": "FSL-1.1-Apache-2.0", diff --git a/service/CHANGELOG.md b/service/CHANGELOG.md index 6c12e7529..74172eeeb 100644 --- a/service/CHANGELOG.md +++ b/service/CHANGELOG.md @@ -1,5 +1,15 @@ # @powersync/service-image +## 0.5.6 + +### Patch Changes + +- Updated dependencies [2d3bb6a] +- Updated dependencies [17a6db0] +- Updated dependencies [0f90b02] + - @powersync/service-core@0.8.6 + - @powersync/service-sync-rules@0.20.0 + ## 0.5.5 ### Patch Changes diff --git a/service/package.json b/service/package.json index 68cc28c10..48e84d793 100644 --- a/service/package.json +++ b/service/package.json @@ -1,6 +1,6 @@ { "name": "@powersync/service-image", - "version": "0.5.5", + "version": "0.5.6", "private": true, "license": "FSL-1.1-Apache-2.0", "type": "module", diff --git a/test-client/CHANGELOG.md b/test-client/CHANGELOG.md index 9c90f775c..1056f5d19 100644 --- a/test-client/CHANGELOG.md +++ b/test-client/CHANGELOG.md @@ -1,5 +1,13 @@ # test-client +## 0.1.8 + +### Patch Changes + +- Updated dependencies [2d3bb6a] +- Updated dependencies [17a6db0] + - @powersync/service-core@0.8.6 + ## 0.1.7 ### Patch Changes diff --git a/test-client/package.json b/test-client/package.json index f66eb0dec..e50f8462f 100644 --- a/test-client/package.json +++ b/test-client/package.json @@ -2,7 +2,7 @@ "name": "test-client", "repository": "https://github.com/powersync-ja/powersync-service", "private": true, - "version": "0.1.7", + "version": "0.1.8", "main": "dist/index.js", "bin": "dist/bin.js", "license": "Apache-2.0", From 6b72e6c17a8633594286309173cadeb88f6c5423 Mon Sep 17 00:00:00 2001 From: stevensJourney <51082125+stevensJourney@users.noreply.github.com> Date: Mon, 4 Nov 2024 10:07:39 +0200 Subject: [PATCH 7/8] feat: postgres connection port restrictions (#116) --- .changeset/eighty-coins-jog.md | 5 +++++ packages/service-core/package.json | 2 +- packages/types/src/config/normalize.ts | 7 +++---- 3 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 .changeset/eighty-coins-jog.md diff --git a/.changeset/eighty-coins-jog.md b/.changeset/eighty-coins-jog.md new file mode 100644 index 000000000..5ff40ee4f --- /dev/null +++ b/.changeset/eighty-coins-jog.md @@ -0,0 +1,5 @@ +--- +'@powersync/service-core': patch +--- + +Improved Postgres connection port restrictions. Connections are now supported on ports >= 1024. diff --git a/packages/service-core/package.json b/packages/service-core/package.json index 1e7129664..130cd777b 100644 --- a/packages/service-core/package.json +++ b/packages/service-core/package.json @@ -13,7 +13,7 @@ "build": "tsc -b", "build:tests": "tsc -b test/tsconfig.json", "test": "vitest --no-threads", - "clean": "rm -rf ./lib && tsc -b --clean" + "clean": "rm -rf ./dist && tsc -b --clean" }, "dependencies": { "@js-sdsl/ordered-set": "^4.4.2", diff --git a/packages/types/src/config/normalize.ts b/packages/types/src/config/normalize.ts index 76e7bf49a..284b3077c 100644 --- a/packages/types/src/config/normalize.ts +++ b/packages/types/src/config/normalize.ts @@ -1,5 +1,5 @@ -import { PostgresConnection } from './PowerSyncConfig.js'; import * as urijs from 'uri-js'; +import { PostgresConnection } from './PowerSyncConfig.js'; /** * Validate and normalize connection options. @@ -97,11 +97,10 @@ export function validatePort(port: string | number): number { if (typeof port == 'string') { port = parseInt(port); } - if (port >= 1024 && port <= 49151) { - return port; - } else { + if (port < 1024) { throw new Error(`Port ${port} not supported`); } + return port; } /** From 575ce3be9f046d5e50a1edc59ddb81d924e447c1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 4 Nov 2024 10:13:18 +0200 Subject: [PATCH 8/8] Version Packages (#117) Co-authored-by: github-actions[bot] --- .changeset/eighty-coins-jog.md | 5 ----- packages/service-core/CHANGELOG.md | 6 ++++++ packages/service-core/package.json | 2 +- service/CHANGELOG.md | 7 +++++++ service/package.json | 2 +- test-client/CHANGELOG.md | 7 +++++++ test-client/package.json | 2 +- 7 files changed, 23 insertions(+), 8 deletions(-) delete mode 100644 .changeset/eighty-coins-jog.md diff --git a/.changeset/eighty-coins-jog.md b/.changeset/eighty-coins-jog.md deleted file mode 100644 index 5ff40ee4f..000000000 --- a/.changeset/eighty-coins-jog.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@powersync/service-core': patch ---- - -Improved Postgres connection port restrictions. Connections are now supported on ports >= 1024. diff --git a/packages/service-core/CHANGELOG.md b/packages/service-core/CHANGELOG.md index eb39c2e21..4a718373f 100644 --- a/packages/service-core/CHANGELOG.md +++ b/packages/service-core/CHANGELOG.md @@ -1,5 +1,11 @@ # @powersync/service-core +## 0.8.7 + +### Patch Changes + +- 6b72e6c: Improved Postgres connection port restrictions. Connections are now supported on ports >= 1024. + ## 0.8.6 ### Patch Changes diff --git a/packages/service-core/package.json b/packages/service-core/package.json index 130cd777b..ad1fc1af3 100644 --- a/packages/service-core/package.json +++ b/packages/service-core/package.json @@ -5,7 +5,7 @@ "publishConfig": { "access": "public" }, - "version": "0.8.6", + "version": "0.8.7", "main": "dist/index.js", "license": "FSL-1.1-Apache-2.0", "type": "module", diff --git a/service/CHANGELOG.md b/service/CHANGELOG.md index 74172eeeb..fc767a8cf 100644 --- a/service/CHANGELOG.md +++ b/service/CHANGELOG.md @@ -1,5 +1,12 @@ # @powersync/service-image +## 0.5.7 + +### Patch Changes + +- Updated dependencies [6b72e6c] + - @powersync/service-core@0.8.7 + ## 0.5.6 ### Patch Changes diff --git a/service/package.json b/service/package.json index 48e84d793..1dfefaee0 100644 --- a/service/package.json +++ b/service/package.json @@ -1,6 +1,6 @@ { "name": "@powersync/service-image", - "version": "0.5.6", + "version": "0.5.7", "private": true, "license": "FSL-1.1-Apache-2.0", "type": "module", diff --git a/test-client/CHANGELOG.md b/test-client/CHANGELOG.md index 1056f5d19..21411c998 100644 --- a/test-client/CHANGELOG.md +++ b/test-client/CHANGELOG.md @@ -1,5 +1,12 @@ # test-client +## 0.1.9 + +### Patch Changes + +- Updated dependencies [6b72e6c] + - @powersync/service-core@0.8.7 + ## 0.1.8 ### Patch Changes diff --git a/test-client/package.json b/test-client/package.json index e50f8462f..10bc1b69a 100644 --- a/test-client/package.json +++ b/test-client/package.json @@ -2,7 +2,7 @@ "name": "test-client", "repository": "https://github.com/powersync-ja/powersync-service", "private": true, - "version": "0.1.8", + "version": "0.1.9", "main": "dist/index.js", "bin": "dist/bin.js", "license": "Apache-2.0",