[schema sync] Implemented (optional) AWS S3 sync

Author: Bertrand Dunogier

composer.json

@@ -19,7 +19,6 @@
         "ezsystems/ezplatform-admin-ui": "^2.0@dev",
         "ezsystems/ezplatform-rest": "^1.2@dev",
         "ezsystems/ezplatform-richtext": "^2.0@dev",
-        "ext-redis": "*",
         "lexik/jwt-authentication-bundle": "^2.8",
         "overblog/graphql-bundle": "^0.12",
         "erusev/parsedown": "^1.7",
@@ -44,6 +43,10 @@
         "ezsystems/ezplatform-code-style": "^0.1.0",
         "mikey179/vfsstream": "^1.6"
     },
+    "suggest": {
+        "aws/aws-sdk-php": "For compiled schema synchronization over AWS-S3",
+        "ext-redis": "*"
+    },
     "autoload": {
         "psr-4": {
             "EzSystems\\EzPlatformGraphQL\\": "src",

src/DependencyInjection/EzSystemsEzPlatformGraphQLExtension.php

@@ -6,6 +6,7 @@
  */
 namespace EzSystems\EzPlatformGraphQL\DependencyInjection;
 
+use Aws\S3\S3Client;
 use EzSystems\EzPlatformGraphQL\DependencyInjection\GraphQL\YamlSchemaProvider;
 use Symfony\Component\Config\FileLocator;
 use Symfony\Component\DependencyInjection\ContainerBuilder;
@@ -42,6 +43,13 @@ public function load(array $configs, ContainerBuilder $container)
         $loader->load('services/schema_sync.yaml');
         $loader->load('services/services.yaml');
         $loader->load('default_settings.yaml');
+
+        if (extension_loaded('redis')) {
+            $loader->load('services/schema_sync.yaml');
+            if (class_exists(S3Client::class)) {
+                $loader->load('services/schema_sync_s3.yaml');
+            }
+        }
     }
 
     /**

src/Resources/config/services/schema_sync.yaml

@@ -7,6 +7,9 @@ services:
         autoconfigure: true
         autowire: true
         public: false
+        bind:
+            EzSystems\EzPlatformGraphQL\Schema\Sync\TimestampHandler: '@EzSystems\EzPlatformGraphQL\Schema\Sync\RedisTimestampHandler'
+            Redis $graphQLSyncRedis: '@ibexa_graphql.sync.redis_client'
 
     EzSystems\EzPlatformGraphQL\Command\PublishSchemaCommand:
         arguments:
@@ -26,14 +29,11 @@ services:
         arguments:
             $definitionsDirectory: '%ibexa_graphql.definitions_directory%'
 
-    ibexa_graphql.redis:
+    ibexa_graphql.sync.redis_client:
         class: Redis
         calls:
             - connect: ['%env(REDIS_GRAPHQL_HOST)%', '%env(REDIS_GRAPHQL_PORT)%']
             - select: ['%env(REDIS_GRAPHQL_DBINDEX)%']
 
-    EzSystems\EzPlatformGraphQL\Schema\Sync\TimestampHandler: '@EzSystems\EzPlatformGraphQL\Schema\Sync\RedisTimestampHandler'
+    EzSystems\EzPlatformGraphQL\Schema\Sync\RedisTimestampHandler: ~
 
-    EzSystems\EzPlatformGraphQL\Schema\Sync\RedisTimestampHandler:
-        arguments:
-            $redis: '@ibexa_graphql.redis'

src/Resources/config/services/schema_sync_s3.yaml

@@ -0,0 +1,17 @@
+services:
+    _defaults:
+        bind:
+            Aws\S3\S3Client $graphQLSyncS3Client: '@ibexa_graphql.sync.s3_client'
+            EzSystems\EzPlatformGraphQL\Schema\Sync\TimestampHandler: '@EzSystems\EzPlatformGraphQL\Schema\Sync\RedisTimestampHandler'
+
+    ibexa_graphql.sync.s3_client:
+        class: Aws\S3\S3Client
+        arguments:
+            -   version: '2006-03-01'
+                region: '%env(GRAPHQL_SYNC_S3_REGION)%'
+
+    EzSystems\EzPlatformGraphQL\Schema\Sync\S3SharedSchema:
+        arguments:
+            $bucket: '%env(GRAPHQL_SYNC_S3_BUCKET)%'
+
+    EzSystems\EzPlatformGraphQL\Schema\Sync\SharedSchema: '@EzSystems\EzPlatformGraphQL\Schema\Sync\S3SharedSchema'

src/Schema/Sync/README.md

@@ -0,0 +1,68 @@
+# GraphQL schema sync
+
+An experimental mechanism for publishing a compiled schema so that secondary servers can pull it and install it without recompiling their own containe.
+
+## Configuration
+
+### Redis
+The feature requires Redis for publishing the latest schema timestamp (it is suggested in `composer.json`).
+
+The feature comes with a default redis client service, `ibexa_graphql.sync.redis_client`.
+It is configured using the following environment variables:
+```.dotenv
+REDIS_GRAPHQL_HOST=1.2.3.4
+REDIS_GRAPHQL_PORT=6379
+REDIS_GRAPHQL_DBINDEX=0
+```
+
+If you want to use your own client, you can redefined the service with the same name in your 
+project's services definitions.
+
+If you already have your own and want to re-use it, create an alias with that name:
+```yaml
+# config/services.yaml
+services:
+    ibexa_graphql.sync.redis_client: '@app.redis_client'
+```
+
+### AWS S3
+Amazon S3 can be used to publish the schema files. To enable it, make sure that `aws/aws-sdk-php`
+is installed on your project.
+
+It uses a default client service named `ibexa_graphql.sync.s3_client`, based on the default
+environment variables expected by the SDK:
+
+```dotenv
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+```
+
+If you already have an s3 client service, alias it to `ibexa_graphql.sync.s3_client`:
+```yaml
+# config/services.yaml
+services:
+    ibexa_graphql.sync.s3_client: '@app.s3_client'
+```
+
+The feature also requires two extra settings for the bucket and the region:
+```dotenv
+GRAPHQL_SYNC_S3_BUCKET=ibexa-graphql
+GRAPHQL_SYNC_S3_REGION=eu-west-1
+```
+
+## Usage
+One server will do the schema generation + compiling, and run a command
+(`ibexa:graphql:publish-schema`) to publish the schema. The command can be executed during a deployment process,
+or manually.
+
+Publishing will:
+
+1. push the compiled schema (`%kernel.cache_dir%/overblog/graphql-bundle/__definitions__/*`) to a SharedSchema
+2. set the published schema timestamp using a TimestampHandler.
+
+Secondary servers, when a GraphQL query is executed (`UpdateSchemaIfNeeded` subscriber), will compare the timestamp to
+theirs (modification time of `__classes.map`). If the remote schema is newer, it will be pulled and installed on shutdown.
+
+Since the graphql schema types are compiled into the container as services, types that were added to the published schema
+(new content types, etc) need to be registered on runtime. This is done by the `Schema\Sync\AddTypesSolutions` subscriber.
+It checks which of the type classes do not have a solution in the current schema, and adds them to it.

src/Schema/Sync/RedisTimestampHandler.php

@@ -20,9 +20,9 @@ class RedisTimestampHandler implements TimestampHandler
      */
     private $key;
 
-    public function __construct(Redis $redis, string $key = 'graphql_schema_timestamp')
+    public function __construct(Redis $graphQLSyncRedis, string $key = 'graphql_schema_timestamp')
     {
-        $this->redis = $redis;
+        $this->redis = $graphQLSyncRedis;
         $this->key = $key;
     }
 

src/Schema/Sync/S3SharedSchema.php

@@ -0,0 +1,97 @@
+<?php
+/**
+ * @copyright Copyright (C) eZ Systems AS. All rights reserved.
+ * @license For full copyright and license information view LICENSE file distributed with this source code.
+ */
+
+namespace EzSystems\EzPlatformGraphQL\Schema\Sync;
+
+use Aws\Result;
+use Aws\S3\Exception\S3Exception;
+use Aws\S3\S3Client;
+use EzSystems\EzPlatformGraphQL\Schema\Sync\SharedSchema;
+use EzSystems\EzPlatformGraphQL\Schema\Sync\TimestampHandler;
+use Symfony\Component\Finder\Finder;
+
+class S3SharedSchema implements SharedSchema
+{
+    /**
+     * @var array Map of filename => file contents
+     */
+    private $files = [];
+
+    /**
+     * @var \EzSystems\EzPlatformGraphQL\Schema\Sync\TimestampHandler
+     */
+    private $timestampHandler;
+    /**
+     * @var \Aws\S3\S3Client
+     */
+    private $s3;
+
+    private $bucket;
+
+    public function __construct(TimestampHandler $timestampHandler, S3Client $graphQLSyncS3Client, string $bucket)
+    {
+        $this->timestampHandler = $timestampHandler;
+        $this->s3 = $graphQLSyncS3Client;
+        $this->bucket = $bucket;
+    }
+
+    public function addFile(string $name, string $contents)
+    {
+        $this->files[$name] = $contents;
+    }
+
+    public function publish(int $timestamp)
+    {
+        foreach ($this->files as $name => $contents) {
+            $this->putFileToS3("$timestamp/$name", $contents);
+        }
+
+        $this->timestampHandler->set($timestamp);
+    }
+
+    public function getFiles(int $timestamp): array
+    {
+        if (!$this->hasFileOnS3("$timestamp/__classes.map")) {
+            throw new \Exception("Shared schema not found");
+        }
+
+        $files = [];
+
+        $prefix = "$timestamp/";
+        $listResult = $this->s3->listObjectsV2(['Bucket' => $this->bucket, 'Prefix' => $prefix]);
+        foreach ($listResult->get('Contents') as $listItem) {
+            $fileResult = $this->s3->getObject(['Bucket' => $this->bucket, 'Key' => $listItem['Key']]);
+            $fileName = str_replace($prefix, '', $listItem['Key']);
+            $files[$fileName] = $fileResult->get('Body')->getContents();
+        }
+
+        return $files;
+    }
+
+    private function putFileToS3(string $name, string $contents): void
+    {
+        try {
+            $this->s3->putObject([
+                'Bucket' => $this->bucket,
+                'Key' => $name,
+                'Body' => $contents,
+            ]);
+        } catch (S3Exception $e) {
+            throw new \Exception("Error creating file", 0, $e);
+        }
+    }
+
+    private function hasFileOnS3(string $path): bool
+    {
+        try {
+            $this->s3->getObject(['Bucket' => $this->bucket, 'Key' => $path]);
+        } catch (S3Exception $e) {
+            return false;
+        }
+
+        return true;
+    }
+}