Add reading support for Elasticsearch (#1999)

* Elasticsearch reading support

* Fix add the missing dependencies to the SortExtension service

* Fix tests and some minor refactoring

* Fix duplicate properties

* Fix CS

* Fix PHPStan errors

* Replace the deprecated nested_path with the new nested sort option

* Disable write operations by default

* Convert snake_case document fields to lowerCamelCase

* Add unit tests

* Add functional tests

* Update deprecations

* Fix CS

* Fix Roland's comment

* Fix er1z's comments

* Fix Kévin's comments

* Update the changelog

Author: meyerbaptiste

.circleci/config.yml

@@ -186,7 +186,7 @@ jobs:
           name: Run Behat tests
           command: |-
             mkdir -p build/logs/tmp build/cov
-            for f in $(find features -name '*.feature' -not -path 'features/main/exposed_state.feature' | circleci tests split --split-by=timings); do
+            for f in $(find features -name '*.feature' -not -path 'features/main/exposed_state.feature' -not -path 'features/elasticsearch/*' | circleci tests split --split-by=timings); do
               _f=${f//\//_}
               FEATURE="${_f}" phpdbg -qrr vendor/bin/behat --profile=coverage --suite=default --format=progress --out=std --format=junit --out=build/logs/tmp/"${_f}" "$f"
             done

.php_cs.dist

@@ -13,7 +13,6 @@ HEADER;
 
 $finder = PhpCsFixer\Finder::create()
     ->in(__DIR__)
-    ->exclude('tests/Fixtures/app/cache')
     ->exclude('tests/Fixtures/app/var')
 ;
 

.travis.yml

@@ -9,6 +9,10 @@ jobs:
   include:
     - php: '7.1'
     - php: '7.2'
+    - php: '7.2'
+      services:
+        - elasticsearch
+      env: APP_ENV=elasticsearch
     - php: '7.3'
     - php: '7.3'
       env: deps=low
@@ -31,6 +35,9 @@ jobs:
   fast_finish: true
 
 before_install:
+  - if [[ $APP_ENV = 'elasticsearch' ]]; then
+      curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.5.0.deb && sudo dpkg -i --force-confnew elasticsearch-6.5.0.deb && sudo service elasticsearch restart;
+    fi
   - phpenv config-rm xdebug.ini || echo "xdebug not available"
   - echo "memory_limit=-1" >> ~/.phpenv/versions/$(phpenv version-name)/etc/conf.d/travis.ini
   - export PATH="$PATH:$HOME/.composer/vendor/bin"
@@ -48,6 +55,8 @@ script:
   - tests/Fixtures/app/console cache:clear
   - if [[ $APP_ENV = 'postgres' ]]; then
       vendor/bin/behat --suite=postgres --format=progress;
+    elif [[ $APP_ENV = 'elasticsearch' ]]; then
+      vendor/bin/behat --suite=elasticsearch --format=progress;
     else
       vendor/bin/behat --suite=default --format=progress;
     fi

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 2.4.0
+
+* Elasticsearch: add reading support (including pagination, sort filter and term filter)
+
 ## 2.3.5
 
 * GraphQL: compatibility with `webonyx/graphql-php` 0.13

behat.yml.dist

@@ -2,32 +2,56 @@ default:
   suites:
     default:
       contexts:
-        - 'FeatureContext': { doctrine: '@doctrine' }
+        - 'DoctrineContext':
+            doctrine: '@doctrine'
+        - 'HttpHeaderContext'
         - 'GraphqlContext'
         - 'JsonContext'
         - 'HydraContext'
         - 'SwaggerContext'
         - 'HttpCacheContext'
-        - 'JsonApiContext': { doctrine: '@doctrine', jsonApiSchemaFile: 'tests/Fixtures/JsonSchema/jsonapi.json' }
-        - 'JsonHalContext': { schemaFile: 'tests/Fixtures/JsonHal/jsonhal.json' }
+        - 'JsonApiContext':
+            doctrine: '@doctrine'
+            jsonApiSchemaFile: '%paths.base%/tests/Fixtures/JsonSchema/jsonapi.json'
+        - 'JsonHalContext':
+            schemaFile: '%paths.base%/tests/Fixtures/JsonHal/jsonhal.json'
         - 'Behat\MinkExtension\Context\MinkContext'
         - 'Behatch\Context\RestContext'
       filters:
-        tags: ~@postgres
+        tags: '~@postgres&&~@elasticsearch'
     postgres:
       contexts:
-        - 'FeatureContext': { doctrine: '@doctrine' }
+        - 'DoctrineContext':
+            doctrine: '@doctrine'
+        - 'HttpHeaderContext'
         - 'GraphqlContext'
         - 'JsonContext'
         - 'HydraContext'
         - 'SwaggerContext'
         - 'HttpCacheContext'
-        - 'JsonApiContext': { doctrine: '@doctrine', jsonApiSchemaFile: 'tests/Fixtures/JsonSchema/jsonapi.json' }
-        - 'JsonHalContext': { schemaFile: 'tests/Fixtures/JsonHal/jsonhal.json' }
+        - 'JsonApiContext':
+            doctrine: '@doctrine'
+            jsonApiSchemaFile: '%paths.base%/tests/Fixtures/JsonSchema/jsonapi.json'
+        - 'JsonHalContext':
+            schemaFile: '%paths.base%/tests/Fixtures/JsonHal/jsonhal.json'
         - 'Behat\MinkExtension\Context\MinkContext'
         - 'Behatch\Context\RestContext'
       filters:
-        tags: ~@sqlite
+        tags: '~@sqlite&&~@elasticsearch'
+    elasticsearch:
+      paths:
+        - '%paths.base%/features/elasticsearch'
+      contexts:
+        - 'ElasticsearchContext':
+            client: '@test.api_platform.elasticsearch.client'
+            elasticsearchMappingsPath: '%paths.base%/tests/Fixtures/Elasticsearch/Mappings/'
+            elasticsearchFixturesPath: '%paths.base%/tests/Fixtures/Elasticsearch/Fixtures/'
+        - 'HttpHeaderContext'
+        - 'JsonContext'
+        - 'Behatch\Context\RestContext'
+        - 'Behat\MinkExtension\Context\MinkContext'
+      filters:
+        tags: '@elasticsearch'
   extensions:
     'Behat\Symfony2Extension':
       kernel:
@@ -36,7 +60,7 @@ default:
         path: 'tests/Fixtures/app/AppKernel.php'
         bootstrap: 'tests/Fixtures/app/bootstrap.php'
     'Behat\MinkExtension':
-      base_url: "http://example.com/"
+      base_url: 'http://example.com/'
       sessions:
         default:
           symfony2: ~
@@ -46,16 +70,21 @@ coverage:
   suites:
     default:
       contexts:
-        - 'FeatureContext': { doctrine: '@doctrine' }
+        - 'DoctrineContext':
+            doctrine: '@doctrine'
+        - 'HttpHeaderContext'
         - 'GraphqlContext'
         - 'JsonContext'
         - 'HydraContext'
         - 'SwaggerContext'
         - 'HttpCacheContext'
-        - 'JsonApiContext': { doctrine: '@doctrine', jsonApiSchemaFile: 'tests/Fixtures/JsonSchema/jsonapi.json' }
-        - 'JsonHalContext': { schemaFile: 'tests/Fixtures/JsonHal/jsonhal.json' }
+        - 'JsonApiContext':
+            doctrine: '@doctrine'
+            jsonApiSchemaFile: '%paths.base%/tests/Fixtures/JsonSchema/jsonapi.json'
+        - 'JsonHalContext':
+            schemaFile: '%paths.base%/tests/Fixtures/JsonHal/jsonhal.json'
         - 'CoverageContext'
         - 'Behat\MinkExtension\Context\MinkContext'
         - 'Behatch\Context\RestContext'
       filters:
-        tags: ~@postgres
+        tags: '~@postgres&&~@elasticsearch'

composer.json

@@ -36,6 +36,7 @@
         "doctrine/doctrine-cache-bundle": "^1.3.5",
         "doctrine/doctrine-bundle": "^1.8",
         "doctrine/orm": "^2.6.3",
+        "elasticsearch/elasticsearch": "^6.0",
         "friendsofsymfony/user-bundle": "^2.1",
         "guzzlehttp/guzzle": "^6.0",
         "justinrainbow/json-schema": "^5.0",
@@ -76,6 +77,7 @@
         "doctrine/common": "<2.7"
     },
     "suggest": {
+        "elasticsearch/elasticsearch": "To support Elasticsearch.",
         "friendsofsymfony/user-bundle": "To use the FOSUserBundle bridge.",
         "guzzlehttp/guzzle": "To use the HTTP cache invalidation system.",
         "phpdocumentor/reflection-docblock": "To support extracting metadata from PHPDoc.",

features/bootstrap/FeatureContext.php renamed to features/bootstrap/DoctrineContext.php

@@ -54,17 +54,14 @@
 use ApiPlatform\Core\Tests\Fixtures\TestBundle\Entity\User;
 use ApiPlatform\Core\Tests\Fixtures\TestBundle\Entity\UuidIdentifierDummy;
 use Behat\Behat\Context\Context;
-use Behat\Behat\Context\SnippetAcceptingContext;
-use Behat\Behat\Hook\Scope\AfterStepScope;
-use Behatch\HttpCall\Request;
 use Doctrine\Common\Persistence\ManagerRegistry;
 use Doctrine\ORM\EntityManagerInterface;
 use Doctrine\ORM\Tools\SchemaTool;
 
 /**
  * Defines application features from the specific context.
  */
-final class FeatureContext implements Context, SnippetAcceptingContext
+final class DoctrineContext implements Context
 {
     /**
      * @var EntityManagerInterface
@@ -73,7 +70,6 @@ final class FeatureContext implements Context, SnippetAcceptingContext
     private $doctrine;
     private $schemaTool;
     private $classes;
-    private $request;
 
     /**
      * Initializes context.
@@ -82,35 +78,12 @@ final class FeatureContext implements Context, SnippetAcceptingContext
      * You can also pass arbitrary arguments to the
      * context constructor through behat.yml.
      */
-    public function __construct(ManagerRegistry $doctrine, Request $request)
+    public function __construct(ManagerRegistry $doctrine)
     {
         $this->doctrine = $doctrine;
         $this->manager = $doctrine->getManager();
         $this->schemaTool = new SchemaTool($this->manager);
         $this->classes = $this->manager->getMetadataFactory()->getAllMetadata();
-        $this->request = $request;
-    }
-
-    /**
-     * Sets the default Accept HTTP header to null (workaround to artificially remove it).
-     *
-     * @AfterStep
-     */
-    public function removeAcceptHeaderAfterRequest(AfterStepScope $event)
-    {
-        if (preg_match('/^I send a "[A-Z]+" request to ".+"/', $event->getStep()->getText())) {
-            $this->request->setHttpHeader('Accept', null);
-        }
-    }
-
-    /**
-     * Sets the default Accept HTTP header to null (workaround to artificially remove it).
-     *
-     * @BeforeScenario
-     */
-    public function removeAcceptHeaderBeforeScenario()
-    {
-        $this->request->setHttpHeader('Accept', null);
     }
 
     /**

features/bootstrap/ElasticsearchContext.php

@@ -0,0 +1,142 @@
+<?php
+
+/*
+ * This file is part of the API Platform project.
+ *
+ * (c) Kévin Dunglas <[email protected]>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+declare(strict_types=1);
+
+use ApiPlatform\Core\Bridge\Elasticsearch\Metadata\Document\DocumentMetadata;
+use Behat\Behat\Context\Context;
+use Elasticsearch\Client;
+use Symfony\Component\Finder\Finder;
+
+/**
+ * @experimental
+ *
+ * @author Baptiste Meyer <[email protected]>
+ */
+final class ElasticsearchContext implements Context
+{
+    private $client;
+    private $elasticsearchMappingsPath;
+    private $elasticsearchFixturesPath;
+
+    public function __construct(Client $client, string $elasticsearchMappingsPath, string $elasticsearchFixturesPath)
+    {
+        $this->client = $client;
+        $this->elasticsearchMappingsPath = $elasticsearchMappingsPath;
+        $this->elasticsearchFixturesPath = $elasticsearchFixturesPath;
+    }
+
+    /**
+     * @BeforeScenario
+     */
+    public function initializeElasticsearch(): void
+    {
+        static $initialized = false;
+
+        if ($initialized) {
+            return;
+        }
+
+        $this->deleteIndexes();
+        $this->createIndexesAndMappings();
+        $this->loadFixtures();
+
+        $initialized = true;
+    }
+
+    /**
+     * @Given indexes and their mappings are created
+     */
+    public function thereAreIndexes(): void
+    {
+        $this->createIndexesAndMappings();
+    }
+
+    /**
+     * @Given indexes are deleted
+     */
+    public function thereAreNoIndexes(): void
+    {
+        $this->deleteIndexes();
+    }
+
+    /**
+     * @Given fixtures files are loaded
+     */
+    public function thereAreFixtures(): void
+    {
+        $this->loadFixtures();
+    }
+
+    private function createIndexesAndMappings(): void
+    {
+        $finder = new Finder();
+        $finder->files()->in($this->elasticsearchMappingsPath);
+
+        foreach ($finder as $file) {
+            $this->client->indices()->create([
+                'index' => $file->getBasename('.json'),
+                'body' => json_decode($file->getContents(), true),
+            ]);
+        }
+    }
+
+    private function deleteIndexes(): void
+    {
+        $finder = new Finder();
+        $finder->files()->in($this->elasticsearchMappingsPath)->name('*.json');
+
+        $indexClient = $this->client->indices();
+
+        foreach ($finder as $file) {
+            $index = $file->getBasename('.json');
+            if (!$indexClient->exists(['index' => $index])) {
+                continue;
+            }
+
+            $indexClient->delete(['index' => $index]);
+        }
+    }
+
+    private function loadFixtures(): void
+    {
+        $finder = new Finder();
+        $finder->files()->in($this->elasticsearchFixturesPath)->name('*.json');
+
+        $indexClient = $this->client->indices();
+
+        foreach ($finder as $file) {
+            $index = $file->getBasename('.json');
+            $bulk = [];
+
+            foreach (json_decode($file->getContents(), true) as $document) {
+                if (null === $document['id'] ?? null) {
+                    $bulk[] = ['index' => ['_index' => $index, '_type' => DocumentMetadata::DEFAULT_TYPE]];
+                } else {
+                    $bulk[] = ['create' => ['_index' => $index, '_type' => DocumentMetadata::DEFAULT_TYPE, '_id' => (string) $document['id']]];
+                }
+
+                $bulk[] = $document;
+
+                if (0 === (count($bulk) % 50)) {
+                    $this->client->bulk(['body' => $bulk]);
+                    $bulk = [];
+                }
+            }
+
+            if ($bulk) {
+                $this->client->bulk(['body' => $bulk]);
+            }
+
+            $indexClient->refresh(['index' => $index]);
+        }
+    }
+}