Merge branch 'beta' into #13-more-robust-uuid-implementation

Author: mumenthalers

src/config/update-config.function.spec.ts

@@ -0,0 +1,31 @@
+// tslint:disable:no-empty
+
+import { DateToNumberMapper } from '../mapper/custom'
+import { dynamoEasyConfig } from './dynamo-easy-config'
+import { updateDynamoEasyConfig } from './update-config.function'
+
+describe('updateDynamoEasyConfig', () => {
+
+  it('should throw when providing invalid stuff', () => {
+    expect(() => updateDynamoEasyConfig({ logReceiver: <any>null })).toThrow()
+    expect(() => updateDynamoEasyConfig({ dateMapper: <any>null })).toThrow()
+  })
+
+  it('should have defaults', () => {
+    expect(dynamoEasyConfig.logReceiver).toBeDefined()
+    expect(dynamoEasyConfig.dateMapper).toBeDefined()
+  })
+
+  it('should work when providing valid stuff', () => {
+    const myLogReceiver = () => {}
+    const myDateMapper = { ...DateToNumberMapper }
+    updateDynamoEasyConfig({
+      logReceiver: myLogReceiver,
+      dateMapper: myDateMapper,
+    })
+    expect(dynamoEasyConfig.logReceiver).toBe(myLogReceiver)
+    expect(dynamoEasyConfig.dateMapper).toBe(myDateMapper)
+
+  })
+
+})

src/config/update-config.function.ts

@@ -5,5 +5,8 @@ export function updateDynamoEasyConfig(config: Partial<Config>): void {
   if (config.logReceiver !== undefined && typeof config.logReceiver !== 'function') {
     throw new Error('Config.logReceiver has to be a function')
   }
+  if (config.dateMapper !== undefined && (config.dateMapper === null || typeof config.dateMapper.toDb !== 'function' || typeof config.dateMapper.fromDb !== 'function')) {
+    throw new Error('Config.dateMapper must be an object of type MapperForType')
+  }
   Object.assign(dynamoEasyConfig, config)
 }

src/decorator/metadata/metadata.spec.ts

@@ -0,0 +1,85 @@
+// tslint:disable:no-non-null-assertion
+import {
+  ModelWithABunchOfIndexes, ModelWithAutogeneratedId,
+  ModelWithGSI,
+  ModelWithLSI,
+  SimpleWithCompositePartitionKeyModel,
+  SimpleWithPartitionKeyModel,
+} from '../../../test/models'
+import { INDEX_ACTIVE, INDEX_ACTIVE_CREATED_AT, INDEX_COUNT } from '../../../test/models/model-with-indexes.model'
+import { Metadata } from './metadata'
+
+describe('metadata', () => {
+  let metaDataPartitionKey: Metadata<SimpleWithPartitionKeyModel>
+  let metaDataComposite: Metadata<SimpleWithCompositePartitionKeyModel>
+  let metaDataLsi: Metadata<ModelWithLSI>
+  let metaDataGsi: Metadata<ModelWithGSI>
+  let metaDataIndexes: Metadata<ModelWithABunchOfIndexes>
+  let metaDataUuid: Metadata<ModelWithAutogeneratedId>
+
+  beforeEach(() => {
+    metaDataPartitionKey = new Metadata(SimpleWithPartitionKeyModel)
+    metaDataComposite = new Metadata(SimpleWithCompositePartitionKeyModel)
+    metaDataLsi = new Metadata(ModelWithLSI)
+    metaDataGsi = new Metadata(ModelWithGSI)
+    metaDataIndexes = new Metadata(ModelWithABunchOfIndexes)
+    metaDataUuid = new Metadata(ModelWithAutogeneratedId)
+  })
+
+  it('forProperty', () => {
+    const forId = metaDataPartitionKey.forProperty('id')
+    expect(forId).toBeDefined()
+    expect(forId!.key).toBeDefined()
+    expect(forId!.name).toBe('id')
+    expect(forId!.typeInfo).toBeDefined()
+    expect(forId!.typeInfo!.isCustom).toBeFalsy()
+  })
+
+  it('getKeysWithUUID', () => {
+    const uuid = metaDataUuid.getKeysWithUUID()
+    expect(uuid.length).toBe(1)
+    expect(uuid[0].key).toBeDefined()
+    expect(uuid[0].key!.uuid).toBeTruthy()
+    expect(uuid[0].name).toBe('id')
+
+  })
+
+  it('getPartitionKey', () => {
+    expect(metaDataPartitionKey.getPartitionKey()).toEqual('id')
+    expect(metaDataGsi.getPartitionKey(INDEX_ACTIVE)).toEqual('active')
+    expect(metaDataIndexes.getPartitionKey(INDEX_COUNT)).toEqual('myId')
+    expect(metaDataIndexes.getPartitionKey(INDEX_ACTIVE_CREATED_AT)).toEqual('active')
+
+  })
+
+  it('getSortKey', () => {
+    expect(metaDataPartitionKey.getSortKey()).toBe(null)
+    expect(metaDataComposite.getSortKey()).toBe('creationDate')
+    expect(metaDataLsi.getSortKey(INDEX_ACTIVE)).toBe('active')
+    expect(() => metaDataGsi.getSortKey(INDEX_ACTIVE)).toThrow()
+    expect(metaDataIndexes.getSortKey(INDEX_ACTIVE_CREATED_AT)).toBe('createdAt')
+  })
+
+  it('getIndexes', () => {
+    expect(metaDataLsi.getIndexes()).toEqual([
+      { partitionKey: 'id', sortKey: 'active' },
+    ])
+    expect(metaDataGsi.getIndexes()).toEqual([
+      { partitionKey: 'active' },
+    ])
+    expect(metaDataIndexes.getIndexes()).toEqual([
+      { partitionKey: 'active', sortKey: 'createdAt' },
+      { partitionKey: 'myId', sortKey: 'count' },
+    ])
+  })
+
+  it('getIndex', () => {
+    expect(metaDataPartitionKey.getIndexes().length).toBe(0)
+    expect(metaDataPartitionKey.getIndex('blub')).toBe(null)
+    expect(metaDataIndexes.getIndex(INDEX_ACTIVE_CREATED_AT)).toEqual({
+      partitionKey: 'active',
+      sortKey: 'createdAt',
+    })
+  })
+
+})

src/dynamo/batchget/batch-get-full.response.ts

@@ -6,7 +6,7 @@ export interface BatchGetFullResponse {
   /**
    * A map of table name to a list of items. Each object in Responses consists of a table name, along with a map of attribute data consisting of the data type and attribute value.
    */
-  Responses?: BatchGetResponse
+  Responses: BatchGetResponse
   /**
    * A map of tables and their respective keys that were not processed with the current response. The UnprocessedKeys value is in the same form as RequestItems, so the value can be provided directly to a subsequent BatchGetItem operation. For more information, see RequestItems in the Request Parameters section. Each element consists of:    Keys - An array of primary key attribute values that define specific items in the table.    ProjectionExpression - One or more attributes to be retrieved from the table or index. By default, all attributes are returned. If a requested attribute is not found, it does not appear in the result.    ConsistentRead - The consistency of a read operation. If set to true, then a strongly consistent read is used; otherwise, an eventually consistent read is used.   If there are no unprocessed keys remaining, the response contains an empty UnprocessedKeys map.
    */

src/dynamo/batchget/batch-get-utils.spec.ts

@@ -0,0 +1,118 @@
+import { DynamoDB } from 'aws-sdk'
+import { of } from 'rxjs'
+import { DynamoRx } from '../dynamo-rx'
+import { batchGetItemsFetchAll, combineBatchGetResponses, hasUnprocessedKeys } from './batch-get-utils'
+
+describe('batch-get utils', () => {
+
+  describe('hasUnprocessedKeys', () => {
+    it('should return bool according to given object', () => {
+      expect(hasUnprocessedKeys({})).toBeFalsy()
+      expect(hasUnprocessedKeys({ Responses: {} })).toBeFalsy()
+      expect(hasUnprocessedKeys({ UnprocessedKeys: {} })).toBeFalsy()
+      expect(hasUnprocessedKeys({ UnprocessedKeys: { 'aTableName': { Keys: [] } } })).toBeFalsy()
+      expect(hasUnprocessedKeys({ UnprocessedKeys: { 'aTableName': { Keys: [{ id: { S: 'id' } }] } } })).toBeTruthy()
+    })
+  })
+
+  describe('combineBatchGetResponses', () => {
+    const resp1: DynamoDB.BatchGetItemOutput = {
+      Responses: {
+        'tableA': [
+          { id: { S: 'id-a1' } },
+        ],
+        'tableB': [
+          { id: { S: 'id-b' } },
+        ],
+      },
+      UnprocessedKeys: {
+        'tableA:': { Keys: [{ id: { S: 'id-a2' } }] },
+        'tableC:': { Keys: [{ id: { S: 'id-c' } }] },
+        'tableD:': { Keys: [{ id: { S: 'id-d' } }] },
+      },
+    }
+    const resp2: DynamoDB.BatchGetItemOutput = {
+      Responses: {
+        'tableA': [
+          { id: { S: 'id-a2' } },
+        ],
+        'tableC': [
+          { id: { S: 'id-c' } },
+        ],
+      },
+      UnprocessedKeys: {
+        'tableD:': { Keys: [{ id: { S: 'id-d' } }] },
+      },
+    }
+    const expectedOutput: DynamoDB.BatchGetItemOutput = {
+      Responses: {
+        'tableA': [
+          { id: { S: 'id-a1' } },
+          { id: { S: 'id-a2' } },
+        ],
+        'tableB': [
+          { id: { S: 'id-b' } },
+        ],
+        'tableC': [
+          { id: { S: 'id-c' } },
+        ],
+      },
+      UnprocessedKeys: {
+        'tableD:': { Keys: [{ id: { S: 'id-d' } }] },
+      },
+    }
+    it('should combine correctly', () => {
+      expect(combineBatchGetResponses(resp1)(resp2)).toEqual(expectedOutput)
+    })
+
+  })
+
+  describe('batchGetItemsFetchAll', () => {
+    let batchGetItemsSpy: jasmine.Spy
+    let dynamoRx: DynamoRx
+    let backoffTimerMock: { next: jasmine.Spy }
+
+    const output1: DynamoDB.BatchGetItemOutput = {
+      Responses: {
+        'tableA': [{ id: { S: 'id-A' } }],
+      },
+      UnprocessedKeys: {
+        'tableA': {
+          Keys: [{ id: { S: 'id-A' } }],
+        },
+      },
+    }
+    const output2: DynamoDB.BatchGetItemOutput = {
+      Responses: {
+        'tableA': [{ id: { S: 'id-A' } }],
+      },
+    }
+
+    beforeEach(async () => {
+      batchGetItemsSpy = jasmine.createSpy().and.returnValues(of(output1), of(output2))
+      dynamoRx = <any>{ batchGetItems: batchGetItemsSpy }
+      backoffTimerMock = { next: jasmine.createSpy().and.returnValue({ value: 0 }) }
+
+      await batchGetItemsFetchAll(
+        dynamoRx,
+        <any>{},
+        <IterableIterator<number>><any>backoffTimerMock,
+        0,
+      ).toPromise()
+    })
+
+
+    it('should use UnprocessedKeys for next request', () => {
+      expect(batchGetItemsSpy).toHaveBeenCalledTimes(2)
+      expect(batchGetItemsSpy.calls.mostRecent().args[0]).toBeDefined()
+      expect(batchGetItemsSpy.calls.mostRecent().args[0].RequestItems).toBeDefined()
+      expect(batchGetItemsSpy.calls.mostRecent().args[0].RequestItems).toEqual(output1.UnprocessedKeys)
+    })
+
+    it('should backoff when UnprocessedItems', () => {
+      expect(backoffTimerMock.next).toHaveBeenCalledTimes(1)
+    })
+
+  })
+
+})

src/dynamo/batchget/batch-get-utils.ts

@@ -0,0 +1,67 @@
+import { DynamoDB } from 'aws-sdk'
+import { BatchGetRequestMap } from 'aws-sdk/clients/dynamodb'
+import { Observable, of } from 'rxjs'
+import { delay, map, mergeMap } from 'rxjs/operators'
+import { DynamoRx } from '../dynamo-rx'
+
+export function batchGetItemsFetchAll(
+  dynamoRx: DynamoRx,
+  params: DynamoDB.BatchGetItemInput,
+  backoffTimer: IterableIterator<number>,
+  throttleTimeSlot: number,
+): Observable<DynamoDB.BatchGetItemOutput> {
+  return dynamoRx.batchGetItems(params)
+    .pipe(
+      mergeMap(response => {
+        if (hasUnprocessedKeys(response)) {
+          return of(response.UnprocessedKeys)
+            .pipe(
+              delay(backoffTimer.next().value * throttleTimeSlot),
+              mergeMap((UnprocessedKeys: DynamoDB.BatchGetRequestMap) => {
+                const nextParams = { ...params, RequestItems: UnprocessedKeys }
+                return batchGetItemsFetchAll(dynamoRx, nextParams, backoffTimer, throttleTimeSlot)
+              }),
+              map(combineBatchGetResponses(response)),
+            )
+        }
+        return of(response)
+      }),
+    )
+}
+
+export type ResponseWithUnprocessedKeys = DynamoDB.BatchGetItemOutput & { UnprocessedKeys: BatchGetRequestMap }
+
+export function hasUnprocessedKeys(response: DynamoDB.BatchGetItemOutput): response is ResponseWithUnprocessedKeys {
+  if (!response.UnprocessedKeys) {
+    return false
+  }
+  return Object.values(response.UnprocessedKeys)
+    .some(t => !!t && t.Keys && t.Keys.length > 0)
+}
+
+/**
+ * combines a first with a second response. ConsumedCapacity is always from the latter.
+ * @param response1
+ */
+export function combineBatchGetResponses(response1: DynamoDB.BatchGetItemOutput) {
+  return (response2: DynamoDB.BatchGetItemOutput): DynamoDB.BatchGetItemOutput => {
+    const tableNames: string[] = Object.keys(response1.Responses || {})
+
+    Object.keys(response2.Responses || {})
+      .filter(tn => !tableNames.includes(tn))
+      .forEach(tn => tableNames.push(tn))
+
+    const Responses = tableNames
+      .reduce((u, tableName) => ({
+        ...u,
+        [tableName]: [
+          ...(response1.Responses && response1.Responses[tableName] || []),
+          ...(response2.Responses && response2.Responses[tableName] || []),
+        ],
+      }), {})
+    return {
+      ...response2,
+      Responses,
+    }
+  }
+}