Story - Related data paging (#8)

Author: samhuk

related-data-paging.sql

@@ -0,0 +1,127 @@
+select * from "user"
+
+select * from "user_group"
+
+select * from user_to_user_group
+
+select * from "image"
+
+select * from "image" where creator_user_id = 2
+
+------------------------------------------------
+-- non-many-to-many, non-range-constrained
+------------------------------------------------
+select
+-- Root data node columns
+"1".uuid "1.uuid", "1".date_created "1.dateCreated", "1".file_name "1.fileName",
+-- Root data node linked field column
+"1".creator_user_id "1.creatorUserId",
+-- To-one related data columns
+"2".id "2.id", "2".name "2.name"
+-- Root from
+from "image" "1"
+-- To-one related data left joins
+left join "user" "2" on "2".id = "1"."creator_user_id"
+-- Linked field value where clause
+where "1".creator_user_id = any (values (1),(2),(3))
+-- 	Root data node query SQL
+and "1".date_deleted is null
+order by "1".id
+
+------------------------------------------------
+-- non-many-to-many, range-constrained
+------------------------------------------------
+select
+"_0".id "_0.id", "_0".name "_0.name",
+-- Root data node columns
+"1"."uuid" "1.uuid", "1"."file_name" "1.fileName",
+-- Root data node linked field column
+"1"."creator_user_id" "1.creatorUserId",
+-- To-one related data columns
+"2".id "2.id", "2".name "2.name"
+-- Root from
+from "user" "_0"
+-- Range-constraint part of root "from"
+join lateral (
+	select
+	"uuid", "date_created", "file_name", "creator_user_id"
+	from "image" "1"
+	where "1".creator_user_id = "_0"."id"
+	-- 	Root data node query SQL
+	and "1"."date_deleted" is null
+	order by "1".id
+	limit 2
+	offset 2
+) as "1" on "1"."creator_user_id" = "_0"."id"
+-- To-one related data left joins
+left join "user" "2" on "2".id = "1"."creator_user_id"
+-- Linked field value where clause
+where "_0"."id" = any (values (1),(2),(3))
+
+------------------------------------------------
+-- many-to-many, non-range-constrained
+------------------------------------------------
+select
+-- Root data node columns
+"1".id "1.id", "1".uuid "1.uuid", "1".name "1.name", "1".date_created "1.dateCreated",
+-- Root data node join table columns
+"u2ug".user_id "u2ug.user_id", "u2ug".user_group_id "u2ug.user_group_id",
+-- To-one related data columns
+"2".id "2.id", "2".name "2.name"
+-- Root from
+from "user_to_user_group" "u2ug"
+join "user_group" "1" on "1".id = "u2ug".user_group_id
+-- To-one related data left joins
+left join "user" "2" on "2".id = "1"."id"
+-- Linked field value where clause
+where "u2ug".user_id = any (values (1),(2),(3))
+-- 	Root data node query SQL
+and "1".date_deleted is null
+order by "1".id
+
+------------------------------------------------
+-- many-to-many, range-constrained
+------------------------------------------------
+select
+-- Root data node columns
+"1"."id" "1.id", "1"."uuid" "1.uuid", "1"."name" "1.name", "1"."date_created" "1.dateCreated",
+-- Root data node join table columns
+"1"."u2ug.user_id" "u2ug.user_id", "1"."u2ug.user_group_id" "u2ug.user_group_id",
+-- To-one related data columns
+"2".id "2.id", "2".name "2.name"
+-- Root from
+from "user" "_0"
+-- Range-constraint part of root "from"
+join lateral (
+	select
+	"1".id "id", "1".uuid "uuid", "1".name "name", "1".date_created "date_created",
+	"u2ug".user_id "u2ug.user_id", "u2ug".user_group_id "u2ug.user_group_id"
+	from "user_to_user_group" "u2ug"
+	join "user_group" "1" on "1".id = "u2ug"."user_group_id"
+	where "u2ug"."user_id" = "_0"."id"
+	-- Root data node query SQL
+	and "1".date_deleted is null
+	order by "1".id
+	limit 2
+	offset 2
+) as "1" on "1"."u2ug.user_id" = "_0"."id"
+-- To-one related data left joins
+left join "user" "2" on "2".id = "1"."id"
+-- Linked field value where clause
+where "_0"."id" = any (values (1),(2),(3))
+
+
+select
+"1".id "1.id", "1".uuid "1.uuid", "1".date_created "1.dateCreated", "1".date_deleted "1.dateDeleted", "1".name "1.name", "1".description "1.description", "1".image_id "1.imageId",
+"1"."user_to_user_group.user_id" "user_to_user_group.user_id", "1"."user_to_user_group.user_group_id" "user_to_user_group.user_group_id"
+from "user" "_0"
+join lateral (
+select
+"1"."id" "id", "1"."uuid" "uuid", "1"."date_created" "date_created", "1"."date_deleted" "date_deleted", "1"."name" "name", "1"."description" "description", "1"."image_id" "image_id",
+"user_to_user_group".user_id "user_to_user_group.user_id", "user_to_user_group".user_group_id "user_to_user_group.user_group_id"
+from user_to_user_group "user_to_user_group"
+join "user_group" "1" on "1".id = "user_to_user_group"."user_group_id"
+where "user_to_user_group"."user_id" = "_0"."id"
+limit 1 offset 0
+) as "1" on "1"."user_to_user_group.user_id" = "_0"."id"
+where "_0"."id" = any (values (1),(2),(3))

src/dataFormat/index.ts

@@ -114,7 +114,7 @@ export const createDataFormat = <T extends DataFormatDeclaration>(
     name: dataFormatDeclaration.name,
     capitalizedName: capitalize(dataFormatDeclaration.name) as any,
     pluralizedName,
-    capitalizedPluralizedName: capitalize(pluralizedName),
+    capitalizedPluralizedName: capitalize(pluralizedName) as any,
     declaration: dataFormatDeclaration,
     fields,
     fieldNames: fieldNamesDict,
@@ -131,9 +131,8 @@ export const createDataFormat = <T extends DataFormatDeclaration>(
         if (fname === COMMON_FIELDS.dateDeleted.name)
           return
 
-        // TODO: Although fname is always going to be a key in "fields", TS doesn't see it.
-        // @ts-ignore
-        record[fname] = createSampleData(fields[fname])
+        // Although fname is always going to be a key in "fields", TS doesn't see it.
+        (record as any)[fname] = createSampleData(fields[fname])
       })
       return record
     },

src/dataFormat/types.ts

@@ -6,78 +6,78 @@ export enum DataType {
   /**
    * Any numeric value, i.e. integer, serial, real.
    */
-  NUMBER,
+  NUMBER = 'num',
   /**
    * Any string value, i.e. varying length, fixed length, etc.
    */
-  STRING,
+  STRING = 'str',
   /**
    * Any boolean value
    */
-  BOOLEAN,
+  BOOLEAN = 'bool',
   /**
    * Any epoch value, i.e. time, date, date-time, date-time with timezone.
    */
-  DATE,
+  DATE = 'date',
   /**
    * Any object or array value.
    */
-  JSON
+  JSON = 'json'
 }
 
 export enum NumberDataSubType {
-  INTEGER,
+  INTEGER = 'int',
   /**
    * A serial integer value. This is likely to be for the unique primary key field of the data format.
    */
-  SERIAL,
-  REAL
+  SERIAL = 'ser',
+  REAL = 'real'
 }
 
 export enum StringDataSubType {
   /**
    * A string value that is a UUID V4, i.e. a 36 character randomly-generated string.
    */
-  UUID_V4,
+  UUID_V4 = 'uuidv4',
   /**
    * A fixed-length string.
    */
-  FIXED_LENGTH,
+  FIXED_LENGTH = 'fixed',
   /**
    * A varying-length string.
    */
-  VARYING_LENGTH,
+  VARYING_LENGTH = 'var',
   /**
    * A string that, underlying, is some kind of enumeration.
    */
-  STRING_ENUM,
+  STRING_ENUM = 'enum',
 }
 
 export enum BooleanDataSubType {
-  TRUE_FALSE,
+  TRUE_FALSE = 'truefalse',
 }
 
 export enum DateDataSubType {
-  DATE,
-  TIME,
-  DATE_TIME,
-  DATE_TIME_WITH_TIMEZONE
+  DATE = 'date',
+  TIME = 'time',
+  DATE_TIME = 'datetime',
+  DATE_TIME_WITH_TIMEZONE = 'datetime_timezone'
 }
 
 export enum ThreeStepNumberSize {
-  SMALL,
-  REGULAR,
-  LARGE,
+  SMALL = 'small',
+  REGULAR = 'reg',
+  LARGE = 'large',
 }
 
 export enum TwoStepNumberSize {
-  REGULAR,
-  LARGE,
+  REGULAR = 'reg',
+  LARGE = 'large',
 }
 
 export enum JsonDataSubType {
-  ARRAY,
-  OBJECT
+  ARRAY = 'arr',
+  OBJECT = 'obj'
 }
 
 export type DataTypeToSubType = {

src/examples/realDbTest/index.ts

@@ -247,6 +247,19 @@ const init = async () => {
   }), 'query')
   fs.writeFileSync(path.resolve(outputDir, 'user-user-groups-query.json'), JSON.stringify(result2.result, null, 2))
 
+  const result3 = await timedFn(() => stores.user.getMultiple({
+    fields: [],
+    relations: {
+      userGroups: {
+        query: {
+          page: 1,
+          pageSize: 1,
+        },
+      },
+    },
+  }), 'query')
+  fs.writeFileSync(path.resolve(outputDir, 'user-user-groups-query-range-constraint.json'), JSON.stringify(result3.result, null, 2))
+
   const dtList: number[] = []
   await repeatTimedFn(() => getResult(stores), 'query', 10, dtList)
   const avgDt = dtList.reduce((acc, dt) => acc + dt, 0) / dtList.length

src/helpers/array.ts

@@ -9,3 +9,13 @@ export const removeDuplicates = <T>(array: T[]): T[] => {
     acc.indexOf(item) === -1 ? acc.concat(item) : acc
   ), [])
 }
+
+export const removeNullAndUndefinedValues = <T>(array: T[]): T[] => {
+  if (array == null)
+    return null
+
+  if (array.length === 0)
+    return []
+
+  return array.filter(item => item != null)
+}

src/helpers/string.ts

@@ -53,14 +53,24 @@ export const capitalize = <T extends string>(s: T): Capitalize<T> => (
   `${s.charAt(0).toUpperCase()}${s.slice(1)}` as Capitalize<T>
 )
 
+/**
+ * Removes null, undefined, and empty strings from `arr`.
+ */
 export const filterForNotNullAndEmpty = (arr: string[]) => (
   arr.filter(s => s != null && s.length > 0)
 )
 
+/**
+ * Concatenates the list of strings - `arr`, if and only if `arr` is defined and has
+ * at least one entry.
+ */
 export const joinIfhasEntries = (arr: string[], joinStr: string) => (
   arr != null && arr.length > 0 ? arr.join(joinStr) : null
 )
 
+/**
+ * Concatenates `prefix` and `suffix` together if and only if both of them are defined.
+ */
 export const concatIfNotNullAndEmpty = (prefix: string, suffix: string) => {
   if (prefix != null && suffix != null)
     return prefix.concat(suffix)

src/relations/types.ts

@@ -13,7 +13,7 @@ export enum RelationType {
    *
    * Foreign field: unique
    */
-  ONE_TO_ONE,
+  ONE_TO_ONE = 'one_to_one',
   /**
    * One item of this format relates to multiple items on another format.
    *
@@ -25,7 +25,7 @@ export enum RelationType {
    *
    * Foreign field: not unique
    */
-  ONE_TO_MANY,
+  ONE_TO_MANY = 'one_to_many',
   /**
    * Multiple items of this format relates to multiple items of another format.
    *
@@ -37,7 +37,7 @@ export enum RelationType {
    * Use `createJoinTables` to create them once the relations have been loaded into
    * the TsPgOrm instance.
    */
-  MANY_TO_MANY,
+  MANY_TO_MANY = 'many_to_many',
 }
 
 type ExtractAvailableFieldRefs<T extends DataFormatDeclarations> = ExpandRecursively<ValuesUnionFromDict<{

src/store/get/dataNodes.ts

@@ -3,7 +3,7 @@ import { removeDuplicates } from '../../helpers/array'
 import { toDict } from '../../helpers/dict'
 import { RelationsDict, RelationType, Relation, RelationDeclarations } from '../../relations/types'
 import { AnyGetFunctionOptions, GetFunctionOptions } from '../types/get'
-import { RelatedDataInfoDict, DataNodes, UnresolvedDataNodes, FieldsInfo, DataNode } from './types'
+import { RelatedDataInfoDict, DataNodes, UnresolvedDataNodes, FieldsInfo, DataNode, PluralDataNode, NonPluralDataNode } from './types'
 
 /**
  * Determines all of the related data properties of the given `dataFormat`
@@ -176,12 +176,12 @@ const createFieldsInfo = (dataNode: DataNode): FieldsInfo => {
     const parentJoinTableColumnName = isFieldRefFieldRef1
       ? dataNode.relation.sql.joinTableFieldRef2ColumnName
       : dataNode.relation.sql.joinTableFieldRef1ColumnName
-    joinTableParentFullyQualifiedColumnName = `"${joinTableAlias}".${parentJoinTableColumnName}`
+    joinTableParentFullyQualifiedColumnName = `"${joinTableAlias}"."${parentJoinTableColumnName}"`
     joinTableParentColumnNameAlias = `${joinTableAlias}.${parentJoinTableColumnName}`
     const joinTableColumnName = isFieldRefFieldRef1
       ? dataNode.relation.sql.joinTableFieldRef1ColumnName
       : dataNode.relation.sql.joinTableFieldRef2ColumnName
-    joinTableFullyQualifiedColumnName = `"${joinTableAlias}".${joinTableColumnName}`
+    joinTableFullyQualifiedColumnName = `"${joinTableAlias}"."${joinTableColumnName}"`
   }
 
   return {
@@ -211,6 +211,7 @@ const resolveDataNodes = (unresolvedDataNodes: UnresolvedDataNodes): DataNodes =
       relatedDataPropName: node.relatedDataPropName,
       relation: node.relation,
       tableAlias: `"${node.id}"`,
+      unquotedTableAlias: node.id.toString(),
       createColumnsSqlSegments: () => dataNode.fieldsInfo.fieldsToSelectFor.map(fName => (
         `${dataNode.fieldsInfo.fieldToFullyQualifiedColumnName[fName]} "${dataNode.fieldsInfo.fieldToColumnNameAlias[fName]}"`
       )),
@@ -260,12 +261,14 @@ export const toDataNodes = <
     unresolvedDataNodes,
     { id: 0 },
   )
-  /* Resolve each data node in the dict, which means, amongst some other changes, means
-   * to convert all of the child and parent data node id links into references to other
+  /* Resolve each data node in the dict, which means, amongst some other changes,
+   * converting all of the child and parent data node id links into references to other
    * data nodes. This will make things much easier later on in the query plan framework,
    * such as making it so that we don't have to carry around the whole data nodes dict
    * all the time (because each resovled data node will have direct references to
    * surrounding data nodes).
    */
   return resolveDataNodes(unresolvedDataNodes)
 }
+
+export const isDataNodePlural = (dataNode: PluralDataNode | NonPluralDataNode): dataNode is PluralDataNode => dataNode.isPlural