S3 Filesystem based on Async AWS library

.github/workflows/monorepo-split.yml

@@ -74,6 +74,8 @@ jobs:
 
           - local_path: 'src/bridge/filesystem/azure'
             split_repository: 'filesystem-azure-bridge'
+          - local_path: 'src/bridge/filesystem/async-aws'
+            split_repository: 'filesystem-async-aws-bridge'
           - local_path: 'src/bridge/monolog/http'
             split_repository: 'monolog-http-bridge'
           - local_path: 'src/bridge/symfony/http-foundation'

.github/workflows/test-suite.yml

@@ -23,9 +23,8 @@ concurrency:
 jobs:
   tests:
     name: "Tests"
-
     runs-on: ${{ matrix.operating-system }}
-
+    timeout-minutes: 15
     services:
       elasticsearch:
         image: elasticsearch:7.17.2
@@ -39,7 +38,7 @@ jobs:
         ports:
           - 9200:9200
       meilisearch:
-        image: getmeili/meilisearch:latest
+        image: getmeili/meilisearch:v1.11.3
         env:
           MEILI_MASTER_KEY: masterKey
           MEILI_NO_ANALYTICS: true
@@ -67,6 +66,15 @@ jobs:
         image: redis:latest
         ports:
           - 6379:6379
+      localstack:
+        image: localstack/localstack:latest
+        env:
+          SERVICES: s3
+          AWS_ACCESS_KEY_ID: test
+          AWS_SECRET_ACCESS_KEY: test
+        ports:
+          - 4566:4566
+          - 4571:4571
 
     strategy:
       fail-fast: false
@@ -91,8 +99,8 @@ jobs:
         with:
           tools: composer:v2
           php-version: "${{ matrix.php-version }}"
-          ini-values: memory_limit=-1
-          extensions: :psr, bcmath, dom, hash, json, mbstring, xml, xmlwriter, xmlreader, zlib
+          ini-values: memory_limit=-1, post_max_size=32M, upload_max_filesize=32M
+          extensions: :psr, bcmath, dom, hash, json, mbstring, xml, xmlwriter, xmlreader, zlib, curl
 
       - name: "List PHP Extensions"
         run: php -m
@@ -178,6 +186,19 @@ jobs:
 
       - name: "Test - Bridge - Filesystem Azure"
         run: "composer test:bridge:filesystem-azure"
+        env:
+          AZURITE_HOST: "localhost"
+          AZURITE_BLOB_PORT: "10000"
+          AZURITE_ACCOUNT_NAME: "flowphpaccount01"
+          AZURITE_ACCOUNT_KEY: "flowphpkey01"
+
+      - name: "Test - Bridge - Filesystem Async AWS"
+        run: "composer test:bridge:filesystem-async-aws -vvv"
+        env:
+          S3_ENDPOINT: "http://localhost:${{ job.services.localstack.ports[4566] }}"
+          S3_REGION: "us-east-1"
+          S3_ACCESS_KEY_ID: "test"
+          S3_SECRET_ACCESS_KEY: "test"
 
       - name: "Test - Bridge - Monolog Http"
         run: "composer test:bridge:monolog-http"

bin/docs.php

@@ -48,6 +48,7 @@ public function execute(InputInterface $input, OutputInterface $output) : int
             __DIR__ . '/../src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/functions.php',
             __DIR__ . '/../src/lib/filesystem/src/Flow/Filesystem/DSL/functions.php',
             __DIR__ . '/../src/bridge/filesystem/azure/src/Flow/Filesystem/Bridge/Azure/DSL/functions.php',
+            __DIR__ . '/../src/bridge/filesystem/async-aws/src/Flow/Filesystem/Bridge/AsyncAWS/DSL/functions.php',
             __DIR__ . '/../src/lib/azure-sdk/src/Flow/Azure/SDK/DSL/functions.php',
         ];
 

compose.yml.dist

@@ -53,4 +53,16 @@ services:
         container_name: flow-php-redis
         ports:
             - "6379:6379"
+    localstack:
+        image: localstack/localstack:latest
+        container_name: flow-php-localstack
+        ports:
+            - "4566:4566"
+        environment:
+            - SERVICES=s3
+            - DEBUG=1
+            - DATA_DIR=/tmp/localstack/data
+        volumes:
+            - "./var/localstack:/tmp/localstack"
+
 

composer.json

@@ -21,6 +21,7 @@
         "ext-xmlwriter": "*",
         "ext-zlib": "*",
         "composer-runtime-api": "^2.1",
+        "async-aws/s3": "^2.6",
         "coduo/php-humanizer": "^5.0",
         "doctrine/dbal": "^3.6 || ^4.0",
         "elasticsearch/elasticsearch": "^7.6|^8.0",
@@ -49,6 +50,7 @@
         "php-http/curl-client": "^2.2",
         "php-http/mock-client": "^1.5",
         "ramsey/uuid": "^4.5",
+        "symfony/http-client": "^5.4.47 || ^6.4 || ^7.0",
         "symfony/cache": "^6.2 || ^7.0",
         "symfony/dotenv": "^6.2 || ^7.0",
         "symfony/finder": "^6.3 || ^7.0",
@@ -79,6 +81,7 @@
         "flow-php/etl-adapter-xml": "self.version",
         "flow-php/filesystem": "self.version",
         "flow-php/filesytem-azure-bridge": "self.version",
+        "flow-php/filesytem-async-aws-bridge": "self.version",
         "flow-php/monolog-http-bridge": "self.version",
         "flow-php/parquet": "self.version",
         "flow-php/parquet-viewer": "self.version",
@@ -105,6 +108,7 @@
                 "src/adapter/etl-adapter-text/src/Flow",
                 "src/adapter/etl-adapter-xml/src/Flow",
                 "src/bridge/filesystem/azure/src/Flow",
+                "src/bridge/filesystem/async-aws/src/Flow",
                 "src/bridge/monolog/http/src/Flow",
                 "src/bridge/symfony/http-foundation/src/Flow",
                 "src/cli/src/Flow",
@@ -140,6 +144,7 @@
             "src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/functions.php",
             "src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/functions.php",
             "src/bridge/filesystem/azure/src/Flow/Filesystem/Bridge/Azure/DSL/functions.php",
+            "src/bridge/filesystem/async-aws/src/Flow/Filesystem/Bridge/AsyncAWS/DSL/functions.php",
             "src/core/etl/src/Flow/ETL/DSL/functions.php",
             "src/cli/src/Flow/CLI/DSL/functions.php",
             "src/functions.php",
@@ -168,6 +173,7 @@
                 "src/adapter/etl-adapter-text/tests/Flow",
                 "src/adapter/etl-adapter-xml/tests/Flow",
                 "src/bridge/filesystem/azure/tests/Flow",
+                "src/bridge/filesystem/async-aws/tests/Flow",
                 "src/bridge/monolog/http/tests/Flow",
                 "src/bridge/symfony/http-foundation/tests/Flow",
                 "src/cli/tests/Flow",
@@ -223,6 +229,7 @@
             "@test:lib:rdsl",
             "@test:lib:snappy",
             "@test:bridge:filesystem-azure",
+            "@test:bridge:filesystem-async-aws",
             "@test:bridge:monolog-http",
             "@test:bridge:symfony-http-foundation",
             "@test:adapter:chartjs",
@@ -278,6 +285,9 @@
             "tools/phpunit/vendor/bin/phpunit --testsuite=bridge-filesystem-azure-unit",
             "tools/phpunit/vendor/bin/phpunit --testsuite=bridge-filesystem-azure-integration"
         ],
+        "test:bridge:filesystem-async-aws": [
+            "tools/phpunit/vendor/bin/phpunit --testsuite=bridge-filesystem-async-aws-integration"
+        ],
         "test:bridge:monolog-http": [
             "tools/phpunit/vendor/bin/phpunit --testsuite=bridge-monolog-http-unit"
         ],