Compatibility for times

Author: simolus3

libs/lib-services/src/codec/codecs.ts

@@ -1,6 +1,6 @@
 import * as t from 'ts-codec';
 import * as bson from 'bson';
-import { TimeValue } from '@powersync/service-sync-rules';
+import { DateTimeValue } from '@powersync/service-sync-rules';
 
 export const buffer = t.codec<Buffer, string>(
   'Buffer',
@@ -13,7 +13,7 @@ export const buffer = t.codec<Buffer, string>(
   (buffer) => Buffer.from(buffer, 'base64')
 );
 
-export const date = t.codec<Date, string | TimeValue>(
+export const date = t.codec<Date, string | DateTimeValue>(
   'Date',
   (date) => {
     if (!(date instanceof Date)) {
@@ -24,7 +24,7 @@ export const date = t.codec<Date, string | TimeValue>(
   (date) => {
     // In our jpgwire wrapper, we patch the row decoding logic to map timestamps into TimeValue instances, so we need to
     // support those here.
-    const parsed = new Date(date instanceof TimeValue ? date.iso8601Representation : date);
+    const parsed = new Date(date instanceof DateTimeValue ? date.iso8601Representation : date);
     if (isNaN(parsed.getTime())) {
       throw new t.TransformError([`Invalid date`]);
     }

modules/module-mongodb/src/replication/MongoRelation.ts

@@ -11,7 +11,7 @@ import {
   SqliteInputValue,
   SqliteRow,
   SqliteValue,
-  TimeValue
+  DateTimeValue
 } from '@powersync/service-sync-rules';
 
 import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
@@ -72,7 +72,7 @@ export function toMongoSyncRulesValue(data: any): SqliteInputValue {
     return data.toHexString();
   } else if (data instanceof Date) {
     const isoString = data.toISOString();
-    return new TimeValue(isoString);
+    return new DateTimeValue(isoString);
   } else if (data instanceof mongo.Binary) {
     return new Uint8Array(data.buffer);
   } else if (data instanceof mongo.Long) {
@@ -125,7 +125,7 @@ function filterJsonData(data: any, context: CompatibilityContext, depth = 0): an
     return data;
   } else if (data instanceof Date) {
     const isoString = data.toISOString();
-    return new TimeValue(isoString).toSqliteValue(context);
+    return new DateTimeValue(isoString).toSqliteValue(context);
   } else if (data instanceof mongo.ObjectId) {
     return data.toHexString();
   } else if (data instanceof mongo.UUID) {

modules/module-postgres/test/src/pg_test.test.ts

@@ -1,6 +1,12 @@
 import { constructAfterRecord } from '@module/utils/pgwire_utils.js';
 import * as pgwire from '@powersync/service-jpgwire';
-import { applyRowContext, CompatibilityContext, SqliteInputRow, TimeValue } from '@powersync/service-sync-rules';
+import {
+  applyRowContext,
+  CompatibilityContext,
+  SqliteInputRow,
+  DateTimeValue,
+  TimeValue
+} from '@powersync/service-sync-rules';
 import { describe, expect, test } from 'vitest';
 import { clearTestDb, connectPgPool, connectPgWire, TEST_URI } from './util.js';
 import { WalStream } from '@module/replication/WalStream.js';
@@ -158,9 +164,9 @@ VALUES(10, ARRAY['null']::TEXT[]);
     expect(transformed[2]).toMatchObject({
       id: 3n,
       date: '2023-03-06',
-      time: '15:47:00',
-      timestamp: new TimeValue('2023-03-06T15:47:00'),
-      timestamptz: new TimeValue('2023-03-06T13:47:00Z')
+      time: new TimeValue('15:47:00'),
+      timestamp: new DateTimeValue('2023-03-06T15:47:00.000000', '2023-03-06 15:47:00'),
+      timestamptz: new DateTimeValue('2023-03-06T13:47:00.000000Z', '2023-03-06 13:47:00Z')
     });
 
     expect(transformed[3]).toMatchObject({
@@ -175,26 +181,26 @@ VALUES(10, ARRAY['null']::TEXT[]);
     expect(transformed[4]).toMatchObject({
       id: 5n,
       date: '0000-01-01',
-      time: '00:00:00',
-      timestamp: new TimeValue('0000-01-01T00:00:00'),
-      timestamptz: new TimeValue('0000-01-01T00:00:00Z')
+      time: new TimeValue('00:00:00'),
+      timestamp: new DateTimeValue('0000-01-01T00:00:00'),
+      timestamptz: new DateTimeValue('0000-01-01T00:00:00Z')
     });
 
     expect(transformed[5]).toMatchObject({
       id: 6n,
-      timestamp: new TimeValue('1970-01-01T00:00:00'),
-      timestamptz: new TimeValue('1970-01-01T00:00:00Z')
+      timestamp: new DateTimeValue('1970-01-01T00:00:00.000000', '1970-01-01 00:00:00'),
+      timestamptz: new DateTimeValue('1970-01-01T00:00:00.000000Z', '1970-01-01 00:00:00Z')
     });
 
     expect(transformed[6]).toMatchObject({
       id: 7n,
-      timestamp: new TimeValue('9999-12-31T23:59:59'),
-      timestamptz: new TimeValue('9999-12-31T23:59:59Z')
+      timestamp: new DateTimeValue('9999-12-31T23:59:59'),
+      timestamptz: new DateTimeValue('9999-12-31T23:59:59Z')
     });
 
     expect(transformed[7]).toMatchObject({
       id: 8n,
-      timestamptz: new TimeValue('0022-02-03T09:13:14Z')
+      timestamptz: new DateTimeValue('0022-02-03T09:13:14.000000Z', '0022-02-03 09:13:14Z')
     });
 
     expect(transformed[8]).toMatchObject({

packages/jpgwire/src/pgwire_types.ts

@@ -1,7 +1,7 @@
 // Adapted from https://github.com/kagis/pgwire/blob/0dc927f9f8990a903f238737326e53ba1c8d094f/mod.js#L2218
 
 import { JsonContainer } from '@powersync/service-jsonbig';
-import { type DatabaseInputValue } from '@powersync/service-sync-rules';
+import { TimeValue, type DatabaseInputValue } from '@powersync/service-sync-rules';
 import { dateToSqlite, lsnMakeComparable, timestampToSqlite, timestamptzToSqlite } from './util.js';
 
 export enum PgTypeOid {
@@ -19,6 +19,7 @@ export enum PgTypeOid {
   DATE = 1082,
   TIMESTAMP = 1114,
   TIMESTAMPTZ = 1184,
+  TIME = 1083,
   JSON = 114,
   JSONB = 3802,
   PG_LSN = 3220
@@ -131,6 +132,8 @@ export class PgType {
         return timestampToSqlite(text);
       case PgTypeOid.TIMESTAMPTZ:
         return timestamptzToSqlite(text);
+      case PgTypeOid.TIME:
+        return TimeValue.parse(text);
       case PgTypeOid.JSON:
       case PgTypeOid.JSONB:
         // Don't parse the contents

packages/jpgwire/src/util.ts

@@ -5,7 +5,7 @@ import { DEFAULT_CERTS } from './certs.js';
 import * as pgwire from './pgwire.js';
 import { PgType } from './pgwire_types.js';
 import { ConnectOptions } from './socket_adapter.js';
-import { DatabaseInputValue, TimeValue } from '@powersync/service-sync-rules';
+import { DatabaseInputValue, DateTimeValue } from '@powersync/service-sync-rules';
 
 // TODO this is duplicated, but maybe that is ok
 export interface NormalizedConnectionConfig {
@@ -222,6 +222,8 @@ export function lsnMakeComparable(text: string) {
   return h.padStart(8, '0') + '/' + l.padStart(8, '0');
 }
 
+const timeRegex = /^([\d\-]+) ([\d:]+)(\.\d+)?([+-][\d:]+)?$/;
+
 /**
  * Convert a postgres timestamptz to a SQLite-compatible/normalized timestamp.
  *
@@ -233,17 +235,17 @@ export function lsnMakeComparable(text: string) {
  *
  * We have specific exceptions for -infinity and infinity.
  */
-export function timestamptzToSqlite(source?: string): TimeValue | null {
+export function timestamptzToSqlite(source?: string): DateTimeValue | null {
   if (source == null) {
     return null;
   }
   // Make compatible with SQLite
-  const match = /^([\d\-]+) ([\d:]+)(\.\d+)?([+-][\d:]+)$/.exec(source);
+  const match = timeRegex.exec(source);
   if (match == null) {
     if (source == 'infinity') {
-      return new TimeValue('9999-12-31T23:59:59Z');
+      return new DateTimeValue('9999-12-31T23:59:59Z');
     } else if (source == '-infinity') {
-      return new TimeValue('0000-01-01T00:00:00Z');
+      return new DateTimeValue('0000-01-01T00:00:00Z');
     } else {
       return null;
     }
@@ -258,10 +260,15 @@ export function timestamptzToSqlite(source?: string): TimeValue | null {
     return null;
   }
 
-  const baseValue = parsed.toISOString().replace('Z', '');
-  const baseText = `${baseValue}Z`;
+  const baseValue = parsed.toISOString().replace('.000', '').replace('Z', '');
 
-  return new TimeValue(baseText);
+  // In the new format, we always use ISO 8601. Since Postgres drops zeroes from the fractional seconds, we also pad
+  // that back to the highest theoretical precision (microseconds). This ensures that sorting returned values as text
+  // returns them in order of the time value they represent.
+  //
+  // In the old format, we keep the sub-second precision only if it's not `.000`.
+  const missingPrecision = precision?.padEnd(7, '0') ?? '.000000';
+  return new DateTimeValue(`${baseValue}${missingPrecision}Z`, `${baseValue.replace('T', ' ')}${precision ?? ''}Z`);
 }
 
 /**
@@ -271,17 +278,26 @@ export function timestamptzToSqlite(source?: string): TimeValue | null {
  *
  * https://www.postgresql.org/docs/current/datatype-datetime.html#DATATYPE-DATETIME-SPECIAL-VALUES
  */
-export function timestampToSqlite(source?: string): TimeValue | null {
+export function timestampToSqlite(source?: string): DateTimeValue | null {
   if (source == null) {
     return null;
   }
-  if (source == 'infinity') {
-    return new TimeValue('9999-12-31T23:59:59');
-  } else if (source == '-infinity') {
-    return new TimeValue('0000-01-01T00:00:00');
-  } else {
-    return new TimeValue(source.replace(' ', 'T'));
+
+  const match = timeRegex.exec(source);
+  if (match == null) {
+    if (source == 'infinity') {
+      return new DateTimeValue('9999-12-31T23:59:59');
+    } else if (source == '-infinity') {
+      return new DateTimeValue('0000-01-01T00:00:00');
+    } else {
+      return null;
+    }
   }
+
+  const [_, date, time, precision, __] = match as any;
+  const missingPrecision = precision?.padEnd(7, '0') ?? '.000000';
+
+  return new DateTimeValue(`${date}T${time}${missingPrecision}`, source);
 }
 /**
  * For date, we keep it mostly as-is.

packages/sync-rules/src/quirks.ts

@@ -50,7 +50,7 @@ export class CompatibilityContext {
    */
   readonly explicitlyFixed: Quirk[];
 
-  constructor(level: CompatibilityLevel, explicitlyFixed: Quirk[]) {
+  constructor(level: CompatibilityLevel, explicitlyFixed: Quirk[] = []) {
     this.level = level;
     this.explicitlyFixed = explicitlyFixed;
   }

packages/sync-rules/src/types.ts

@@ -6,7 +6,7 @@ import { TablePattern } from './TablePattern.js';
 import { toSyncRulesParameters } from './utils.js';
 import { BucketPriority } from './BucketDescription.js';
 import { ParameterLookup } from './BucketParameterQuerier.js';
-import { TimeValue } from './types/time.js';
+import { DateTimeValue } from './types/time.js';
 import { CustomSqliteValue } from './types/custom_sqlite_value.js';
 import { CompatibilityContext } from './quirks.js';
 

packages/sync-rules/src/types/time.ts

@@ -1,5 +1,6 @@
 import { SqliteValueType } from '../ExpressionType.js';
 import { CompatibilityContext, Quirk } from '../quirks.js';
+import { SqliteValue } from '../types.js';
 import { CustomSqliteValue } from './custom_sqlite_value.js';
 
 /**
@@ -9,18 +10,19 @@ import { CustomSqliteValue } from './custom_sqlite_value.js';
  * This is not ISO 6801 compatible, but changing it would be breaking existing users. So, this option is opt-in and
  * disabled by default until a major upgrade.
  */
-export class TimeValue extends CustomSqliteValue {
+export class DateTimeValue extends CustomSqliteValue {
   // YYYY-MM-DDThh:mm:ss.sss / YYYY-MM-DDThh:mm:ss.sssZ
-  readonly iso8601Representation: string;
 
-  constructor(iso8601Representation: string) {
+  constructor(
+    readonly iso8601Representation: string,
+    private readonly fixedLegacyRepresentation: string | undefined = undefined
+  ) {
     super();
-    this.iso8601Representation = iso8601Representation;
   }
 
   // YYYY-MM-DD hh:mm:ss.sss / YYYY-MM-DD hh:mm:ss.sssZ
   public get legacyRepresentation(): string {
-    return this.iso8601Representation.replace('T', ' ');
+    return this.fixedLegacyRepresentation ?? this.iso8601Representation.replace('T', ' ');
   }
 
   get sqliteType(): SqliteValueType {
@@ -31,3 +33,41 @@ export class TimeValue extends CustomSqliteValue {
     return context.isFixed(Quirk.nonIso8601Timestampts) ? this.iso8601Representation : this.legacyRepresentation;
   }
 }
+
+/**
+ * In old versions of the sync service, time values didn't consistently contain a sub-second interval.
+ *
+ * A value like `12:13:14.156789` would be represented as-is, but `12:13:14.000000` would be synced as `12:13:14`. This
+ * is undesirable because it means that sorting values alphabetically doesn't preserve their value.
+ */
+export class TimeValue extends CustomSqliteValue {
+  constructor(
+    readonly timeSeconds: string,
+    readonly fraction: string | undefined = undefined
+  ) {
+    super();
+  }
+
+  static parse(value: string): TimeValue | null {
+    const match = /^([\d:]+)(\.\d+)?$/.exec(value);
+    if (match == null) {
+      return null;
+    }
+
+    const [_, timeSeconds, fraction] = match as any;
+    return new TimeValue(timeSeconds, fraction);
+  }
+
+  toSqliteValue(context: CompatibilityContext): SqliteValue {
+    if (context.isFixed(Quirk.nonIso8601Timestampts)) {
+      const fraction = this.fraction?.padEnd(7, '0') ?? '.000000';
+      return `${this.timeSeconds}${fraction}`;
+    } else {
+      return `${this.timeSeconds}${this.fraction ?? ''}`;
+    }
+  }
+
+  get sqliteType(): SqliteValueType {
+    return 'text';
+  }
+}

packages/sync-rules/test/src/quirks.test.ts

@@ -1,11 +1,11 @@
 import { describe, expect, test } from 'vitest';
-import { CustomSqliteValue, SqlSyncRules, TimeValue, toSyncRulesValue } from '../../src/index.js';
+import { CustomSqliteValue, SqlSyncRules, DateTimeValue, toSyncRulesValue } from '../../src/index.js';
 
 import { ASSETS, PARSE_OPTIONS } from './util.js';
 
 describe('handling historical quirks', () => {
   describe('timestamps', () => {
-    const value = new TimeValue('2025-08-19T09:21:00Z');
+    const value = new DateTimeValue('2025-08-19T09:21:00Z');
 
     test('uses old format by default', () => {
       const rules = SqlSyncRules.fromYaml(
@@ -77,7 +77,7 @@ fixed_quirks:
   });
 
   test('arrays', () => {
-    const data = toSyncRulesValue(['static value', new TimeValue('2025-08-19T09:21:00Z')]);
+    const data = toSyncRulesValue(['static value', new DateTimeValue('2025-08-19T09:21:00Z')]);
 
     for (const withFixedQuirk of [false, true]) {
       let syncRules = `

packages/sync-rules/test/src/utils.test.ts

@@ -2,25 +2,43 @@ import {
   applyValueContext,
   CompatibilityContext,
   evaluateOperator,
+  DateTimeValue,
+  toSyncRulesValue,
   TimeValue,
-  toSyncRulesValue
+  CompatibilityLevel
 } from '../../src/index.js';
 import { describe, expect, test } from 'vitest';
 
 describe('toSyncRulesValue', () => {
   test('custom value', () => {
     expect(
       applyValueContext(
-        toSyncRulesValue([1n, 'two', [new TimeValue('2025-08-19T00:00:00')]]),
+        toSyncRulesValue([1n, 'two', [new DateTimeValue('2025-08-19T00:00:00')]]),
         CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY
       )
     ).toStrictEqual('[1,"two",["2025-08-19 00:00:00"]]');
 
     expect(
       applyValueContext(
-        toSyncRulesValue({ foo: { bar: new TimeValue('2025-08-19T00:00:00') } }),
+        toSyncRulesValue({ foo: { bar: new DateTimeValue('2025-08-19T00:00:00') } }),
         CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY
       )
     ).toStrictEqual('{"foo":{"bar":"2025-08-19 00:00:00"}}');
   });
+
+  test('time value', () => {
+    expect(
+      TimeValue.parse('12:13:14')?.toSqliteValue(new CompatibilityContext(CompatibilityLevel.SYNC_STREAMS))
+    ).toStrictEqual('12:13:14.000000');
+    expect(
+      TimeValue.parse('12:13:14')?.toSqliteValue(new CompatibilityContext(CompatibilityLevel.LEGACY))
+    ).toStrictEqual('12:13:14');
+
+    expect(
+      TimeValue.parse('12:13:14.15')?.toSqliteValue(new CompatibilityContext(CompatibilityLevel.SYNC_STREAMS))
+    ).toStrictEqual('12:13:14.150000');
+    expect(
+      TimeValue.parse('12:13:14.15')?.toSqliteValue(new CompatibilityContext(CompatibilityLevel.LEGACY))
+    ).toStrictEqual('12:13:14.15');
+  });
 });