task build completions datasets for ace autocompleters (#1968)

* feature: first version of flow dsl completer for ace editor

* feature: added scalar_function_chain:bool property to dsl.json

* fix: broken default values in plauground autocompleter

* feature: extend build:docs by dumping methods of api classes

* feature: added commands to generate ace editor completers code

* refactoring: migrated from ace editor to coddemirror

Aceeditor is great but it's lacking a good documentation and examples.
On top of that building autocompleter is a nightmare, codemirror is much
more developer friendly.

* fix: sharing playground code after migration to code mirror

* feature: added flow completer to playground

* feature: added dsl completer to playground

* feature: added dataframe completer to playground

- fixed dsl completer by making him to not suggest dsl function on
methods

* feature: added scalar function chain completer to playground

* fix: dsl autocompleter not creating functions with full namespace

* fix: tab button now put tabs in editor instead of jumping to another input on website

* feature: added playground storage to keep most recent code in browser storage

* Fixed autocompletion on fluent interface

* feature: allow to lazy load files to wasm filesystem

- moved loading flow.phar from playground-editor_controller.js to our
new dynamic loading mechanism

* feature: added status to code editor that showes from where the code is loaded

* feature: added file browser to flow playground

* feature: move all playground code to /workspace folder in wasm filesystem

this prevents displaying other files required by wasm and creates a
clean workspace

* feature: added format action to playground

format action uses php-cs-fixer under the hood to format the code and
update editor

* feature: allow users to upload their own local files to playground

* refactoring: moved away from hardcoded svg icons in playground editor

* feature: move all notification to editor output console

* feature: added help to playground editor

* feature: added more content to help section

* feature: added file preview

when user clicks on a file in files browser he will see a readonly
preview of that file below the editor

* fix: comands generating code completers for playground were missing in website build action

* fix: removed timestamp from completers templates to avoid creating empty diffs when nothing changed

Author: norberttech

.php-cs-fixer.php

@@ -26,8 +26,9 @@
         __DIR__ . '/src/lib/**/tests',
         __DIR__ . '/src/tools/**/src',
         __DIR__ . '/src/tools/**/tests',
-        __DIR__ . '/web/**/src',
-        __DIR__ . '/web/**/tests',
+        __DIR__ . '/web/landing/src',
+        __DIR__ . '/web/landing/tests',
+        __DIR__ . '/web/landing/bin',
         __DIR__ . '/examples',
         __DIR__ . '/tools/rector/src',
     ])

bin/docs.php

@@ -3,8 +3,11 @@
 
 declare(strict_types=1);
 
-use Flow\Documentation\{FunctionCollector, FunctionsExtractor};
+use Flow\Documentation\{FunctionCollector, FunctionsExtractor, MethodCollector, MethodsExtractor};
 use Flow\ETL\Attribute\Module;
+use Flow\ETL\DataFrame\GroupedDataFrame;
+use Flow\ETL\{DataFrame, Flow};
+use Flow\ETL\Function\ScalarFunctionChain;
 use Symfony\Component\Console\Application;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\{InputArgument, InputInterface};
@@ -81,4 +84,44 @@ public function execute(InputInterface $input, OutputInterface $output) : int
     }
 });
 
+$application->add(new class extends Command {
+    public function configure() : void
+    {
+        $this
+            ->setName('api:dump')
+            ->setDescription('Dump API methods from classes into json file.')
+            ->addArgument('output', InputArgument::REQUIRED, 'Where to dump methods.');
+    }
+
+    public function execute(InputInterface $input, OutputInterface $output) : int
+    {
+        $repositoryRootPath = dirname(__DIR__) . '/';
+
+        $classes = [
+            ScalarFunctionChain::class,
+            Flow::class,
+            DataFrame::class,
+            GroupedDataFrame::class,
+        ];
+
+        $normalizedMethods = [];
+
+        foreach ($classes as $className) {
+            $extractor = new MethodsExtractor(
+                $repositoryRootPath,
+                $className,
+                new MethodCollector()
+            );
+
+            foreach ($extractor->extract() as $method) {
+                $normalizedMethods[] = $method->normalize();
+            }
+        }
+
+        \file_put_contents(__DIR__ . '/../' . \ltrim((string) $input->getArgument('output'), '/'), \json_encode($normalizedMethods));
+
+        return Command::SUCCESS;
+    }
+});
+
 $application->run();

composer.json

@@ -432,15 +432,16 @@
         "build:phar": [
             "composer update --working-dir=./src/cli",
             "tools/box/vendor/bin/box compile --config ./src/cli/box.json",
-            "cp ./build/flow.phar ./web/landing/assets/wasm/flow.phar"
+            "cp ./build/flow.phar ./web/landing/assets/wasm/tools/flow.phar"
         ],
         "build:wasm": [
             "Composer\\Config::disableProcessTimeout",
             "cd wasm && ./build.sh",
             "@build:phar"
         ],
         "build:docs": [
-            "bin/docs.php dsl:dump web/landing/resources/dsl.json"
+            "bin/docs.php dsl:dump web/landing/resources/dsl.json",
+            "bin/docs.php api:dump web/landing/resources/api.json"
         ],
         "build:docs:api": [
             "./tools/phpdocumentor/vendor/bin/phpdoc --config=./phpdoc/core.xml",

rector.src.php

@@ -17,6 +17,7 @@
         __DIR__ . '/src/adapter/*/src',
         __DIR__ . '/src/bridge/*/src',
         __DIR__ . '/src/tools/*/src',
+        __DIR__ . '/web/landing/src',
     ])
     ->withSkip([
         RemoveExtraParametersRector::class,

rector.tests.php

@@ -86,6 +86,7 @@
         __DIR__ . '/src/adapter/*/*/tests',
         __DIR__ . '/src/bridge/*/*/tests',
         __DIR__ . '/src/tools/*/*/tests',
+        __DIR__ . '/web/landing/tests',
     ])
     ->withSets([
         LevelSetList::UP_TO_PHP_82,

src/tools/documentation/src/Flow/Documentation/MethodCollector.php

@@ -0,0 +1,35 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Flow\Documentation;
+
+final class MethodCollector
+{
+    /**
+     * @var array<string>
+     */
+    public array $methods = [];
+
+    /**
+     * @param class-string $className
+     */
+    public function collect(string $className) : void
+    {
+        $reflectionClass = new \ReflectionClass($className);
+        $methods = $reflectionClass->getMethods(\ReflectionMethod::IS_PUBLIC);
+
+        foreach ($methods as $method) {
+            if ($method->getDeclaringClass()->getName() !== $className) {
+                continue;
+            }
+
+            // Skip magic methods __construct and __destruct
+            if ($method->getName() === '__construct' || $method->getName() === '__destruct') {
+                continue;
+            }
+
+            $this->methods[] = $method->getName();
+        }
+    }
+}

src/tools/documentation/src/Flow/Documentation/MethodsExtractor.php

@@ -0,0 +1,40 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Flow\Documentation;
+
+use Flow\Documentation\Models\MethodModel;
+
+final readonly class MethodsExtractor
+{
+    /**
+     * @param class-string $className
+     */
+    public function __construct(
+        private string $repositoryRootPath,
+        private string $className,
+        private MethodCollector $methodCollector,
+    ) {
+    }
+
+    /**
+     * @return \Generator<MethodModel>
+     */
+    public function extract() : \Generator
+    {
+        $this->methodCollector->collect($this->className);
+
+        $reflectionClass = new \ReflectionClass($this->className);
+
+        foreach ($this->methodCollector->methods as $methodName) {
+            $reflectionMethod = $reflectionClass->getMethod($methodName);
+            $repositoryPath = \ltrim(\str_replace($this->repositoryRootPath, '', (string) $reflectionMethod->getFileName()), '/');
+
+            yield MethodModel::fromReflection(
+                $repositoryPath,
+                $reflectionMethod
+            );
+        }
+    }
+}

src/tools/documentation/src/Flow/Documentation/Models/AttributesModel.php

@@ -28,7 +28,7 @@ public static function fromArray(array $data) : self
         );
     }
 
-    public static function fromReflection(\ReflectionFunction $reflection) : self
+    public static function fromReflection(\ReflectionFunction|\ReflectionMethod $reflection) : self
     {
         return new self(
             array_map(

src/tools/documentation/src/Flow/Documentation/Models/FunctionModel.php

@@ -4,8 +4,9 @@
 
 namespace Flow\Documentation\Models;
 
-use function Flow\Types\DSL\{type_array, type_integer, type_optional, type_string, type_structure};
+use function Flow\Types\DSL\{type_array, type_boolean, type_integer, type_optional, type_string, type_structure};
 use Cocur\Slugify\Slugify;
+use Flow\ETL\Function\ScalarFunctionChain;
 
 final readonly class FunctionModel
 {
@@ -18,6 +19,7 @@ public function __construct(
         public ParametersModel $parameters,
         public TypesModel $returnType,
         public AttributesModel $attributes,
+        public bool $scalarFunctionChain,
         public ?string $docComment = null,
     ) {
 
@@ -37,6 +39,7 @@ public static function fromArray(array $data) : self
             'parameters' => type_array(),
             'return_type' => type_array(),
             'attributes' => type_array(),
+            'scalar_function_chain' => type_boolean(),
             'doc_comment' => type_optional(type_string()),
         ])->assert($data);
 
@@ -56,6 +59,7 @@ public static function fromArray(array $data) : self
             ParametersModel::fromArray($parameters),
             TypesModel::fromArray($returnType),
             AttributesModel::fromArray($attributes),
+            $data['scalar_function_chain'],
             $data['doc_comment']
         );
     }
@@ -77,6 +81,7 @@ public static function fromReflection(string $relativePath, \ReflectionFunction
             ParametersModel::fromFunctionReflection($reflectionFunction),
             TypesModel::fromReflection($returnTypeReflection),
             AttributesModel::fromReflection($reflectionFunction),
+            self::isScalarFunctionChain($returnTypeReflection),
             $reflectionFunction->getDocComment() ? \base64_encode($reflectionFunction->getDocComment()) : null,
         );
     }
@@ -95,7 +100,31 @@ public function normalize() : array
             'parameters' => $this->parameters->normalize(),
             'return_type' => $this->returnType->normalize(),
             'attributes' => $this->attributes->normalize(),
+            'scalar_function_chain' => $this->scalarFunctionChain,
             'doc_comment' => $this->docComment,
         ];
     }
+
+    private static function isScalarFunctionChain(\ReflectionType $reflectionType) : bool
+    {
+        if ($reflectionType instanceof \ReflectionNamedType) {
+            $typeName = $reflectionType->getName();
+
+            if (!\class_exists($typeName)) {
+                return false;
+            }
+
+            return \is_a($typeName, ScalarFunctionChain::class, true);
+        }
+
+        if ($reflectionType instanceof \ReflectionUnionType || $reflectionType instanceof \ReflectionIntersectionType) {
+            foreach ($reflectionType->getTypes() as $type) {
+                if (self::isScalarFunctionChain($type)) {
+                    return true;
+                }
+            }
+        }
+
+        return false;
+    }
 }