Merge pull request #975 from jboolean/pagination

Pagination for stories and outtakes

Author: jboolean

backend/cloneStories.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+PGPASSWORD=$(aws ssm get-parameter --name fourtiesnyc-production-db-password --query Parameter.Value --output text --with-decryption) \
+pg_dump \
+--clean --if-exists --table stories \
+  --host $(aws ssm get-parameter --name fourtiesnyc-production-db-host --query Parameter.Value --output text --with-decryption) \
+  --port $(aws ssm get-parameter --name fourtiesnyc-production-db-port --query Parameter.Value --output text --with-decryption) \
+  --username $(aws ssm get-parameter --name fourtiesnyc-production-db-username --query Parameter.Value --output text --with-decryption) \
+  --dbname $(aws ssm get-parameter --name fourtiesnyc-production-db-database --query Parameter.Value --output text --with-decryption) \
+  | \
+PGPASSWORD=$(aws ssm get-parameter --name fourtiesnyc-staging-db-password --query Parameter.Value --output text --with-decryption) \
+psql --host $(aws ssm get-parameter --name fourtiesnyc-staging-db-host --query Parameter.Value --output text --with-decryption) \
+  --port $(aws ssm get-parameter --name fourtiesnyc-staging-db-port --query Parameter.Value --output text --with-decryption) \
+  --username $(aws ssm get-parameter --name fourtiesnyc-staging-db-username --query Parameter.Value --output text --with-decryption) \
+  --dbname $(aws ssm get-parameter --name fourtiesnyc-staging-db-database --query Parameter.Value --output text --with-decryption)

backend/src/api/photos/PhotosController.ts

@@ -11,9 +11,12 @@ import {
   SuccessResponse,
 } from 'tsoa';
 import { getRepository, In } from 'typeorm';
+import Paginated from '../../business/pagination/Paginated';
+import mapPaginated from '../../business/utils/mapPaginated';
 import Photo from '../../entities/Photo';
 import Collection from '../../enum/Collection';
 import getLngLatForIdentifier from '../../repositories/getLngLatForIdentifier';
+import { getPaginated } from '../../repositories/paginationUtils';
 import { PhotoApiModel } from './PhotoApiModel';
 import photoToApi from './photoToApi';
 
@@ -100,18 +103,36 @@ export class PhotosController extends Controller {
 
   @Get('/outtake-summaries')
   public async getOuttakeSummaries(
-    @Query() collection: Collection = Collection.FOURTIES
-  ): Promise<{ identifier: string }[]> {
+    @Query() collection: Collection = Collection.FOURTIES,
+
+    // pagination
+    @Query('pageToken') pageToken?: string,
+    @Query('pageSize') pageSize = 100
+  ): Promise<Paginated<{ identifier: string }>> {
     const photoRepo = getRepository(Photo);
 
-    const photos = await photoRepo.find({
-      where: { isOuttake: true, collection: collection },
-      select: ['identifier'],
-      order: {
-        identifier: 'ASC',
+    const qb = photoRepo
+      .createQueryBuilder('photo')
+      .select(['photo.identifier'])
+      .where({ isOuttake: true, collection: collection });
+
+    const page = await getPaginated(
+      qb,
+      {
+        key: 'identifier',
+        sortDirection: 'ASC',
+        getSerializedToken: (photo) => photo.identifier,
+        deserializeToken: (token) => token,
       },
-    });
-    return photos.map(({ identifier }) => ({ identifier }));
+      {
+        pageToken,
+        pageSize,
+      }
+    );
+
+    return mapPaginated(page, ({ identifier }: Pick<Photo, 'identifier'>) => ({
+      identifier,
+    }));
   }
 
   @Get('/{identifier}')

backend/src/api/stories/StoriesController.ts

@@ -16,7 +16,10 @@ import {
   Security,
 } from 'tsoa';
 
+import { UserData as NetlifyUserData } from 'gotrue-js';
 import { BadRequest, Forbidden, NotFound } from 'http-errors';
+import EmailCampaignService from '../../business/email/EmailCampaignService';
+import Paginated from '../../business/pagination/Paginated';
 import {
   backfillUserStoryEmails,
   onStateTransition,
@@ -26,6 +29,9 @@ import {
   verifyStoryToken,
 } from '../../business/stories/StoryTokenService';
 import { validateRecaptchaToken } from '../../business/utils/grecaptcha';
+import mapPaginated from '../../business/utils/mapPaginated';
+import normalizeEmail from '../../business/utils/normalizeEmail';
+import required from '../../business/utils/required';
 import Story from '../../entities/Story';
 import StoryState from '../../enum/StoryState';
 import StoryType from '../../enum/StoryType';
@@ -42,10 +48,6 @@ import {
   toDraftStoryResponse,
   toPublicStoryResponse,
 } from './storyToApi';
-import EmailCampaignService from '../../business/email/EmailCampaignService';
-import normalizeEmail from '../../business/utils/normalizeEmail';
-import { UserData as NetlifyUserData } from 'gotrue-js';
-import required from '../../business/utils/required';
 
 function updateModelFromRequest(
   story: Story,
@@ -213,19 +215,29 @@ export class StoriesController extends Controller {
 
   @Get('/')
   public async getStories(
-    @Query('forPhotoIdentifier') identifier?: string
-  ): Promise<PublicStoryResponse[]> {
-    let stories: Story[];
+    @Query('forPhotoIdentifier') identifier?: string,
+
+    // pagination
+    @Query('pageToken') pageToken?: string,
+    @Query('pageSize') pageSize = 100
+  ): Promise<Paginated<PublicStoryResponse>> {
+    let stories: Paginated<Story>;
+
+    const paginationInput = {
+      pageToken,
+      pageSize,
+    };
 
     if (identifier) {
       stories = await StoryRepository().findPublishedForPhotoIdentifier(
-        identifier
+        identifier,
+        paginationInput
       );
     } else {
-      stories = await StoryRepository().findPublished();
+      stories = await StoryRepository().findPublished(paginationInput);
     }
 
-    return map(stories, toPublicStoryResponse);
+    return mapPaginated(stories, toPublicStoryResponse);
   }
 
   @Security('netlify', ['moderator'])

backend/src/business/pagination/Paginated.ts

@@ -0,0 +1,8 @@
+interface Paginated<T> {
+  items: T[];
+  total: number;
+  hasNextPage: boolean;
+  nextToken?: string;
+}
+
+export default Paginated;

backend/src/business/pagination/PaginationInput.ts

@@ -0,0 +1,6 @@
+interface PaginationInput {
+  pageToken?: string;
+  pageSize: number;
+}
+
+export default PaginationInput;

backend/src/business/pagination/PaginationOptions.ts

@@ -0,0 +1,10 @@
+import { ObjectLiteral } from 'typeorm';
+
+interface PaginationOptions<E extends ObjectLiteral> {
+  key: string;
+  sortDirection?: 'ASC' | 'DESC';
+  getSerializedToken: (entity: E) => string;
+  deserializeToken: (token: string) => unknown;
+}
+
+export default PaginationOptions;

backend/src/business/utils/mapPaginated.ts

@@ -0,0 +1,12 @@
+import map from 'lodash/map';
+import Paginated from '../pagination/Paginated';
+
+export default function mapPaginated<I, O>(
+  paginated: Paginated<I>,
+  fn: (I) => O
+): Paginated<O> {
+  return {
+    ...paginated,
+    items: map<I, O>(paginated.items, fn),
+  };
+}

backend/src/repositories/StoryRepository.ts

@@ -1,20 +1,24 @@
 import { Brackets, getRepository, In, IsNull, Repository } from 'typeorm';
+import Paginated from '../business/pagination/Paginated';
+import PaginationInput from '../business/pagination/PaginationInput';
 import Story from '../entities/Story';
 import StoryState from '../enum/StoryState';
 import getLngLatForIdentifier from './getLngLatForIdentifier';
+import { getPaginated } from './paginationUtils';
 
 // eslint-disable-next-line @typescript-eslint/explicit-function-return-type -- better type is inferred
 const StoryRepository = () =>
   getRepository(Story).extend({
     async findPublishedForPhotoIdentifier(
       this: Repository<Story>,
-      identifier: string
+      identifier: string,
+      pagination: PaginationInput
     ) {
       // Get the lng,lat for this photo, so we can return stories
       // for this photo and stories for other photos in the same location
       const maybeLngLat = await getLngLatForIdentifier(identifier);
 
-      return this.createQueryBuilder('story')
+      const qb = this.createQueryBuilder('story')
         .where({ state: StoryState.PUBLISHED })
         .andWhere(
           new Brackets((qb) => {
@@ -31,19 +35,39 @@ const StoryRepository = () =>
         )
         .leftJoinAndSelect('story.photo', 'photo')
         .leftJoinAndSelect('photo.effectiveAddress', 'effectiveAddress')
-        .leftJoinAndSelect('photo.effectiveGeocode', 'effectiveGeocode')
-        .orderBy('story.created_at', 'DESC')
-        .getMany();
+        .leftJoinAndSelect('photo.effectiveGeocode', 'effectiveGeocode');
+      return getPaginated(
+        qb,
+        {
+          key: 'createdAt',
+          sortDirection: 'DESC',
+          getSerializedToken: (story) => story.createdAt.toISOString(),
+          deserializeToken: (token) => Date.parse(token),
+        },
+        pagination
+      );
     },
 
-    async findPublished(this: Repository<Story>) {
-      return this.createQueryBuilder('story')
+    async findPublished(
+      this: Repository<Story>,
+      pagination: PaginationInput
+    ): Promise<Paginated<Story>> {
+      const qb = this.createQueryBuilder('story')
         .where({ state: StoryState.PUBLISHED })
-        .orderBy('story.created_at', 'DESC')
         .leftJoinAndSelect('story.photo', 'photo')
         .leftJoinAndSelect('photo.effectiveAddress', 'effectiveAddress')
-        .leftJoinAndSelect('photo.effectiveGeocode', 'effectiveGeocode')
-        .getMany();
+        .leftJoinAndSelect('photo.effectiveGeocode', 'effectiveGeocode');
+
+      return getPaginated(
+        qb,
+        {
+          key: 'createdAt',
+          sortDirection: 'DESC',
+          getSerializedToken: (story) => story.createdAt.toISOString(),
+          deserializeToken: (token) => new Date(token),
+        },
+        pagination
+      );
     },
 
     /**

backend/src/repositories/paginationUtils.ts

@@ -0,0 +1,60 @@
+import last from 'lodash/last';
+import {
+  FindOperator,
+  IsNull,
+  LessThan,
+  MoreThan,
+  Not,
+  ObjectLiteral,
+  SelectQueryBuilder,
+} from 'typeorm';
+import Paginated from '../business/pagination/Paginated';
+import PaginationInput from '../business/pagination/PaginationInput';
+import PaginationOptions from '../business/pagination/PaginationOptions';
+import required from '../business/utils/required';
+
+export async function getPaginated<E extends ObjectLiteral>(
+  qb: SelectQueryBuilder<E>,
+  {
+    key,
+    sortDirection,
+    getSerializedToken,
+    deserializeToken,
+  }: PaginationOptions<E>,
+  { pageToken: nextToken, pageSize }: PaginationInput
+): Promise<Paginated<E>> {
+  let filterOp: FindOperator<unknown> = Not(IsNull());
+  if (nextToken) {
+    const tokenDeserialized = deserializeToken(nextToken);
+    filterOp =
+      sortDirection === 'ASC'
+        ? MoreThan(tokenDeserialized)
+        : LessThan(tokenDeserialized);
+  }
+
+  const hasWhere = qb.expressionMap.wheres.length > 0;
+
+  // Note clone doesn't actually seem to do anything, so it's important to do count before adding where clauses
+  const count = await qb.clone().getCount();
+
+  const results = await qb
+    .clone()
+    .orderBy(`${qb.alias}.${key}`, sortDirection)
+    [hasWhere ? 'andWhere' : 'where']({
+      [key]: filterOp,
+    })
+    .take(pageSize + 1)
+    .getMany();
+
+  const items = results.slice(0, pageSize);
+  const hasNextPage = results.length > pageSize;
+  const nextNextToken = hasNextPage
+    ? getSerializedToken(required(last(items), 'last'))
+    : undefined;
+  return {
+    items,
+    total: count,
+    hasNextPage,
+    nextToken: nextNextToken,
+  };
+}