Merge branch 'part_five_elastic' into part_five

vilasmaciel · vilasmaciel · commit 3e26a9557276 · 2020-11-25T10:40:20.000+01:00
diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml
@@ -10,6 +10,7 @@ jobs:
       matrix:
         node-version: [12.x, 14.x]
         mongodb-version: [4.0, 4.2]
+        elasticsearch-version: ['7.9.3']
 
     steps:
       - uses: actions/checkout@v1
@@ -21,6 +22,21 @@ jobs:
         uses: wbari/start-mongoDB@v0.2
         with:
           mongoDBVersion: ${{ matrix.mongodb-version }}
+      - name: Configure sysctl limits
+        run: |
+          sudo swapoff -a
+          sudo sysctl -w vm.swappiness=1
+          sudo sysctl -w fs.file-max=262144
+          sudo sysctl -w vm.max_map_count=262144
+      - name: Launch elasticsearch
+        uses: getong/elasticsearch-action@v1.2
+        with:
+          elasticsearch version: ${{ matrix.elasticsearch-version }}
+          host port: 9200
+          container port: 9200
+          host node port: 9300
+          node port: 9300
+          discovery type: 'single-node'
       - name: npm install, build, and test
         run: |
           npm install
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -7,6 +7,8 @@ x-app-service: &default-app
     - MONGO_URL=mongodb://mongo:27017/dev
   depends_on:
     - mongo
+    - kibana
+    - elasticsearch
   volumes:
     - .:/code:delegated
     - node_modules:/code/node_modules:delegated
@@ -34,5 +36,33 @@ services:
     ports:
       - 27017:27017
 
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.9.3
+    container_name: codely-elasticsearch
+    environment:
+      - node.name=codely-elasticsearch
+      - discovery.type=single-node #Elasticsearch forms a single-node cluster
+      - bootstrap.memory_lock=true # might cause the JVM or shell session to exit if it tries to allocate more memory than is available!
+      - 'ES_JAVA_OPTS=-Xms512m -Xmx512m'
+    ulimits:
+      memlock:
+        soft: -1 # The memlock soft and hard values configures the range of memory that ElasticSearch will use. Setting this to –1 means unlimited.
+        hard: -1
+    volumes:
+      - esdata:/usr/share/elasticsearch/data
+    ports:
+      - '9200:9200'
+
+  kibana:
+    image: docker.elastic.co/kibana/kibana:7.8.1
+    container_name: codely-kibana
+    environment:
+      ELASTICSEARCH_URL: http://codely-elasticsearch:9200
+      ELASTICSEARCH_HOSTS: http://codely-elasticsearch:9200
+    ports:
+      - 5601:5601
+
 volumes:
   node_modules:
+  esdata:
+    driver: local
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -31,6 +31,7 @@
     "cypress:run": "NODE_ENV=test ts-node tests/utils/cypress/run"
   },
   "dependencies": {
+    "@elastic/elasticsearch": "^7.9.1",
     "@types/bson": "^4.0.2",
     "@types/compression": "^1.7.0",
     "@types/convict": "^5.2.1",
diff --git a/src/Contexts/Backoffice/infrastructure/ElasticBackofficeCourseRepository.ts b/src/Contexts/Backoffice/infrastructure/ElasticBackofficeCourseRepository.ts
@@ -0,0 +1,34 @@
+import { ElasticRepository } from '../../Shared/infrastructure/persistence/elasticsearch/ElasticRepository';
+import { BackofficeCourse } from '../domain/BackofficeCourse';
+import { BackofficeCourseRepository } from '../domain/BackofficeCourseRepository';
+
+type ElasticBackofficeCourseDocument = { _source: { id: string; duration: string; name: string } };
+
+export class ElasticBackofficeCourseRepository
+  extends ElasticRepository<BackofficeCourse>
+  implements BackofficeCourseRepository {
+  protected moduleName(): string {
+    return 'backofficecourses';
+  }
+
+  async searchAll(): Promise<BackofficeCourse[]> {
+    const client = await this.client();
+
+    const response = await client.search({
+      index: this.moduleName(),
+      body: {
+        query: {
+          match_all: {}
+        }
+      }
+    });
+
+    return response.body.hits.hits.map((hit: ElasticBackofficeCourseDocument) =>
+      BackofficeCourse.fromPrimitives({ ...hit._source })
+    );
+  }
+
+  async save(course: BackofficeCourse): Promise<void> {
+    return this.persist(this.moduleName(), course);
+  }
+}
diff --git a/src/Contexts/Shared/infrastructure/persistence/elasticsearch/ElasticClientFactory.ts b/src/Contexts/Shared/infrastructure/persistence/elasticsearch/ElasticClientFactory.ts
@@ -0,0 +1,33 @@
+import { Client as ElasticClient } from '@elastic/elasticsearch';
+import config from '../../../../../apps/backoffice/backend/config/config';
+import { Nullable } from '../../../domain/Nullable';
+
+export class ElasticClientFactory {
+  private static clients: { [key: string]: ElasticClient } = {};
+
+  static async createClient(contextName: string): Promise<ElasticClient> {
+    let client = ElasticClientFactory.getClient(contextName);
+
+    if (!client) {
+      client = await ElasticClientFactory.createAndConnectClient();
+
+      ElasticClientFactory.registerClient(client, contextName);
+    }
+
+    return client;
+  }
+
+  private static getClient(contextName: string): Nullable<ElasticClient> {
+    return ElasticClientFactory.clients[contextName];
+  }
+
+  private static async createAndConnectClient(): Promise<ElasticClient> {
+    const client = new ElasticClient({ node: config.get('elastic.url') });
+
+    return client;
+  }
+
+  private static registerClient(client: ElasticClient, contextName: string): void {
+    ElasticClientFactory.clients[contextName] = client;
+  }
+}
diff --git a/src/Contexts/Shared/infrastructure/persistence/elasticsearch/ElasticRepository.ts b/src/Contexts/Shared/infrastructure/persistence/elasticsearch/ElasticRepository.ts
@@ -0,0 +1,19 @@
+import { Client as ElasticClient } from '@elastic/elasticsearch';
+import { AggregateRoot } from '../../../../Mooc/Courses/domain/AggregateRoot';
+
+export abstract class ElasticRepository<T extends AggregateRoot> {
+  constructor(private _client: Promise<ElasticClient>) {}
+
+  protected abstract moduleName(): string;
+
+  protected client(): Promise<ElasticClient> {
+    return this._client;
+  }
+
+  protected async persist(index: string, aggregateRoot: T): Promise<void> {
+    const document = { ...aggregateRoot.toPrimitives() };
+    const client = await this.client();
+
+    await client.index({ index: index, body: document, refresh: 'wait_for' }); // wait_for wait for a refresh to make this operation visible to search
+  }
+}
diff --git a/src/apps/backoffice/backend/config/config.ts b/src/apps/backoffice/backend/config/config.ts
@@ -14,6 +14,14 @@ const convictConfig = convict({
       env: 'MONGO_URL',
       default: 'mongodb://localhost:27017/backoffice-backend-dev'
     }
+  },
+  elastic: {
+    url: {
+      doc: 'The Elastic connection URL',
+      format: String,
+      env: 'ELASTIC_URL',
+      default: 'http://localhost:9200'
+    }
   }
 });
 
diff --git a/src/apps/backoffice/backend/config/dependency-injection/Courses/application.yaml b/src/apps/backoffice/backend/config/dependency-injection/Courses/application.yaml
@@ -1,25 +1,28 @@
 services:
-    Backoffice.Backend.courses.BackofficeCourseRepository:
-        class: ../../../../../../Contexts/Backoffice/Shared/infrastructure/MongoBackofficeCourseRepository
-        arguments: ['@Shared.ConnectionManager']
+  Backoffice.Backend.courses.BackofficeCourseRepository:
+    class: ../../../../../../Contexts/Backoffice/Shared/infrastructure/MongoBackofficeCourseRepository
+    arguments: ['@Shared.ConnectionManager']
 
-    Backoffice.Backend.courses.CoursesFinder:
-        class: ../../../../../../Contexts/Backoffice/SearchAll/application//CoursesFinder
-        arguments: ["@Backoffice.Backend.courses.BackofficeCourseRepository"]
+  Backoffice.Backend.courses.BackofficeCourseRepositoryElastic:
+    class: ../../../../../../Contexts/Backoffice/infrastructure/ElasticBackofficeCourseRepository
+    arguments: ['@Shared.ConnectionManagerElastic']
 
-    Backoffice.Backend.courses.SearchAllCoursesQueryHandler:
-        class: ../../../../../../Contexts/Backoffice/SearchAll/application//SearchAllCoursesQueryHandler
-        arguments: ["@Backoffice.Backend.courses.CoursesFinder"]
-        tags:
-            - { name: 'queryHandler' }
+  Backoffice.Backend.courses.CoursesFinder:
+    class: ../../../../../../Contexts/Backoffice/SearchAll/application//CoursesFinder
+    arguments: ['@Backoffice.Backend.courses.BackofficeCourseRepository']
 
-    Backoffice.Backend.courses.BackofficeCourseCreator:
-        class: ../../../../../../Contexts/Backoffice/Courses/application/BackofficeCourseCreator
-        arguments: ["@Backoffice.Backend.courses.BackofficeCourseRepository"]
+  Backoffice.Backend.courses.SearchAllCoursesQueryHandler:
+    class: ../../../../../../Contexts/Backoffice/SearchAll/application//SearchAllCoursesQueryHandler
+    arguments: ['@Backoffice.Backend.courses.CoursesFinder']
+    tags:
+      - { name: 'queryHandler' }
 
-    Backoffice.Backend.courses.CreateBackofficeCourseOnCourseCreated:
-        class: ../../../../../../Contexts/Backoffice/Courses/application/CreateBackofficeCourseOnCourseCreated
-        arguments: ['@Backoffice.Backend.courses.BackofficeCourseCreator']
-        tags:
-            - { name: 'domainEventSubscriber' }
+  Backoffice.Backend.courses.BackofficeCourseCreator:
+    class: ../../../../../../Contexts/Backoffice/Courses/application/BackofficeCourseCreator
+    arguments: ['@Backoffice.Backend.courses.BackofficeCourseRepository']
 
+  Backoffice.Backend.courses.CreateBackofficeCourseOnCourseCreated:
+    class: ../../../../../../Contexts/Backoffice/Courses/application/CreateBackofficeCourseOnCourseCreated
+    arguments: ['@Backoffice.Backend.courses.BackofficeCourseCreator']
+    tags:
+      - { name: 'domainEventSubscriber' }
diff --git a/src/apps/backoffice/backend/config/dependency-injection/Shared/application.yaml b/src/apps/backoffice/backend/config/dependency-injection/Shared/application.yaml
@@ -1,11 +1,17 @@
 services:
   Shared.Logger:
-    class:  ../../../../../../Contexts/Shared/infrastructure/WinstonLogger
+    class: ../../../../../../Contexts/Shared/infrastructure/WinstonLogger
     arguments: []
 
   Shared.ConnectionManager:
     factory:
-      class:  ../../../../../../Contexts/Shared/infrastructure/persistence/mongo/MongoClientFactory
+      class: ../../../../../../Contexts/Shared/infrastructure/persistence/mongo/MongoClientFactory
+      method: 'createClient'
+      arguments: ['mooc']
+
+  Shared.ConnectionManagerElastic:
+    factory:
+      class: ../../../../../../Contexts/Shared/infrastructure/persistence/elasticsearch/ElasticClientFactory
       method: 'createClient'
     arguments: ['mooc']
 
@@ -18,7 +24,7 @@ services:
     arguments: ['@Shared.QueryHandlersInformation']
 
   Shared.EventBus:
-    class:  ../../../../../../Contexts/Shared/infrastructure/EventBus/InMemorySyncEventBus
+    class: ../../../../../../Contexts/Shared/infrastructure/EventBus/InMemorySyncEventBus
     arguments: []
 
   Shared.EventBus.DomainEventMapping:
diff --git a/src/apps/backoffice/backend/config/dependency-injection/application_test.yaml b/src/apps/backoffice/backend/config/dependency-injection/application_test.yaml
@@ -4,4 +4,7 @@ imports:
 services:
   Backoffice.Backend.EnvironmentArranger:
     class: ../../../../../../tests/Contexts/Shared/infrastructure/mongo/MongoEnvironmentArranger
-    arguments: ['@Shared.ConnectionManager']
+    arguments: ['@Shared.ConnectionManager']
+  Backoffice.Backend.ElasticEnvironmentArranger:
+    class: ../../../../../../tests/Contexts/Shared/infrastructure/elastic/ElasticEnvironmentArranger
+    arguments: ['@Shared.ConnectionManagerElastic']
diff --git a/tests/Contexts/Backoffice/infrastructure/ElasticBackofficeCourseRepository.test.ts b/tests/Contexts/Backoffice/infrastructure/ElasticBackofficeCourseRepository.test.ts
@@ -0,0 +1,33 @@
+import container from '../../../../src/apps/backoffice/backend/config/dependency-injection';
+import { BackofficeCourse } from '../../../../src/Contexts/Backoffice/domain/BackofficeCourse';
+import { ElasticBackofficeCourseRepository } from '../../../../src/Contexts/Backoffice/infrastructure/ElasticBackofficeCourseRepository';
+import { EnvironmentArranger } from '../../Shared/infrastructure/arranger/EnvironmentArranger';
+import { BackofficeCourseMother } from '../application/domain/BackofficeCourseMother';
+
+const repository: ElasticBackofficeCourseRepository = container.get(
+  'Backoffice.Backend.courses.BackofficeCourseRepositoryElastic'
+);
+const environmentArranger: Promise<EnvironmentArranger> = container.get(
+  'Backoffice.Backend.ElasticEnvironmentArranger'
+);
+
+function sort(backofficeCourse1: BackofficeCourse, backofficeCourse2: BackofficeCourse): number {
+  return backofficeCourse1?.id?.value.localeCompare(backofficeCourse2?.id?.value);
+}
+
+afterEach(async () => {
+  await (await environmentArranger).arrange();
+});
+
+describe('Search all courses', () => {
+  it('should return the existing courses', async () => {
+    const courses = [BackofficeCourseMother.random(), BackofficeCourseMother.random()];
+
+    await Promise.all(courses.map(async course => repository.save(course)));
+
+    const expectedCourses = await repository.searchAll();
+
+    expect(courses.length).toEqual(expectedCourses.length);
+    expect(courses.sort(sort)).toEqual(expectedCourses.sort(sort));
+  });
+});
diff --git a/tests/Contexts/Shared/infrastructure/elastic/ElasticEnvironmentArranger.ts b/tests/Contexts/Shared/infrastructure/elastic/ElasticEnvironmentArranger.ts

Original file line number	Diff line number	Diff line change
`@@ -14,6 +14,14 @@ const convictConfig = convict({`
`14`	`14`	`env: 'MONGO_URL',`
`15`	`15`	`default: 'mongodb://localhost:27017/backoffice-backend-dev'`
`16`	`16`	`}`
	`17`	`+ },`
	`18`	`+ elastic: {`
	`19`	`+ url: {`
	`20`	`+ doc: 'The Elastic connection URL',`
	`21`	`+ format: String,`
	`22`	`+ env: 'ELASTIC_URL',`
	`23`	`+ default: 'http://localhost:9200'`
	`24`	`+ }`
`17`	`25`	`}`
`18`	`26`	`});`
`19`	`27`