New sync rule functions: json_keys and substring

.changeset/thirty-llamas-carry.md

@@ -0,0 +1,5 @@
+---
+'@powersync/service-sync-rules': minor
+---
+
+Support substring and json_keys functions in sync rules

packages/sync-rules/src/sql_functions.ts

@@ -70,6 +70,61 @@ const lower: DocumentedSqlFunction = {
   detail: 'Convert text to lower case'
 };
 
+const substring: DocumentedSqlFunction = {
+  debugName: 'substring',
+  call(value: SqliteValue, start: SqliteValue, length?: SqliteValue) {
+    const text = castAsText(value);
+    if (text == null) {
+      return null;
+    }
+    const startIndex = cast(start, 'integer') as bigint | null;
+    if (startIndex == null) {
+      return null;
+    }
+    if (length === null) {
+      // Different from undefined in this case, to match SQLite behavior
+      return null;
+    }
+    const castLength = cast(length ?? null, 'integer') as bigint | null;
+    let realLength: number;
+    if (castLength == null) {
+      // undefined (not specified)
+      realLength = text.length + 1; // +1 to account for the start = 0 special case
+    } else {
+      realLength = Number(castLength);
+    }
+
+    let realStart = 0;
+    if (startIndex < 0n) {
+      realStart = Math.max(0, text.length + Number(startIndex));
+    } else if (startIndex == 0n) {
+      // Weird special case
+      realStart = 0;
+      realLength -= 1;
+    } else {
+      realStart = Number(startIndex) - 1;
+    }
+
+    if (realLength < 0) {
+      // Negative length means we return that many characters _before_
+      // the start index.
+      return text.substring(realStart + realLength, realStart);
+    }
+
+    return text.substring(realStart, realStart + realLength);
+  },
+  parameters: [
+    { name: 'value', type: ExpressionType.TEXT, optional: false },
+    { name: 'start', type: ExpressionType.INTEGER, optional: false },
+    { name: 'length', type: ExpressionType.INTEGER, optional: true }
+  ],
+  getReturnType(args) {
+    return ExpressionType.TEXT;
+  },
+  detail: 'Compute a substring',
+  documentation: 'The start index starts at 1. If no length is specified, the remainder of the string is returned.'
+};
+
 const hex: DocumentedSqlFunction = {
   debugName: 'hex',
   call(value: SqliteValue) {
@@ -223,6 +278,32 @@ const json_valid: DocumentedSqlFunction = {
   documentation: 'Returns 1 if valid, 0 if invalid'
 };
 
+const json_keys: DocumentedSqlFunction = {
+  debugName: 'json_keys',
+  call(json: SqliteValue) {
+    const jsonString = castAsText(json);
+    if (jsonString == null) {
+      return null;
+    }
+
+    const jsonParsed = JSONBig.parse(jsonString);
+    if (typeof jsonParsed != 'object') {
+      throw new Error(`Cannot call json_keys on a scalar`);
+    } else if (Array.isArray(jsonParsed)) {
+      throw new Error(`Cannot call json_keys on an array`);
+    }
+    const keys = Object.keys(jsonParsed as {});
+    // Keys are always strings, safe to use plain JSON.
+    return JSON.stringify(keys);
+  },
+  parameters: [{ name: 'json', type: ExpressionType.ANY, optional: false }],
+  getReturnType(args) {
+    // TODO: proper nullable types
+    return ExpressionType.TEXT;
+  },
+  detail: 'Returns the keys of a JSON object as a JSON array'
+};
+
 const unixepoch: DocumentedSqlFunction = {
   debugName: 'unixepoch',
   call(value?: SqliteValue, specifier?: SqliteValue, specifier2?: SqliteValue) {
@@ -389,6 +470,7 @@ const st_y: DocumentedSqlFunction = {
 export const SQL_FUNCTIONS_NAMED = {
   upper,
   lower,
+  substring,
   hex,
   length,
   base64,
@@ -397,6 +479,7 @@ export const SQL_FUNCTIONS_NAMED = {
   json_extract,
   json_array_length,
   json_valid,
+  json_keys,
   unixepoch,
   datetime,
   st_asgeojson,

packages/sync-rules/test/src/sql_functions.test.ts

@@ -44,6 +44,30 @@ describe('SQL functions', () => {
     expect(fn.json_array_length(`{"a":[1,2,3,4]}`)).toEqual(0n);
   });
 
+  test('json_keys', () => {
+    expect(fn.json_keys(`{"a": 1, "b": "2", "0": "test", "c": {"d": "e"}}`)).toEqual(`["0","a","b","c"]`);
+    expect(fn.json_keys(`{}`)).toEqual(`[]`);
+    expect(fn.json_keys(null)).toEqual(null);
+    expect(fn.json_keys()).toEqual(null);
+    expect(() => fn.json_keys(`{"a": 1, "a": 2}`)).toThrow();
+    expect(() => fn.json_keys(`[1,2,3]`)).toThrow();
+    expect(() => fn.json_keys(3)).toThrow();
+  });
+
+  test('json_valid', () => {
+    expect(fn.json_valid(`{"a": 1, "b": "2", "0": "test", "c": {"d": "e"}}`)).toEqual(1n);
+    expect(fn.json_valid(`{}`)).toEqual(1n);
+    expect(fn.json_valid(null)).toEqual(0n);
+    expect(fn.json_valid()).toEqual(0n);
+    expect(fn.json_valid(`{"a": 1, "a": 2}`)).toEqual(0n);
+    expect(fn.json_valid(`[1,2,3]`)).toEqual(1n);
+    expect(fn.json_valid(3)).toEqual(1n);
+    expect(fn.json_valid('test')).toEqual(0n);
+    expect(fn.json_valid('"test"')).toEqual(1n);
+    expect(fn.json_valid('true')).toEqual(1n);
+    expect(fn.json_valid('TRUE')).toEqual(0n);
+  });
+
   test('typeof', () => {
     expect(fn.typeof(null)).toEqual('null');
     expect(fn.typeof('test')).toEqual('text');
@@ -101,6 +125,22 @@ describe('SQL functions', () => {
     expect(fn.lower(Uint8Array.of(0x61, 0x62, 0x43))).toEqual('abc');
   });
 
+  test('substring', () => {
+    expect(fn.substring(null)).toEqual(null);
+    expect(fn.substring('abc')).toEqual(null);
+    expect(fn.substring('abcde', 2, 3)).toEqual('bcd');
+    expect(fn.substring('abcde', 2)).toEqual('bcde');
+    expect(fn.substring('abcde', 2, null)).toEqual(null);
+    expect(fn.substring('abcde', 0, 1)).toEqual('');
+    expect(fn.substring('abcde', 0, 2)).toEqual('a');
+    expect(fn.substring('abcde', 1, 2)).toEqual('ab');
+    expect(fn.substring('abcde', -2)).toEqual('de');
+    expect(fn.substring('abcde', -2, 1)).toEqual('d');
+    expect(fn.substring('abcde', 6, -5)).toEqual('abcde');
+    expect(fn.substring('abcde', 5, -2)).toEqual('cd');
+    expect(fn.substring('2023-06-28 14:12:00.999Z', 1, 10)).toEqual('2023-06-28');
+  });
+
   test('cast', () => {
     expect(cast(null, 'text')).toEqual(null);
     expect(cast(null, 'integer')).toEqual(null);