Add missing adapters to flowphar

.github/workflows/baseline.yml

@@ -39,7 +39,8 @@ jobs:
           php-version: "${{ matrix.php-version }}"
           dependencies: "locked"
           coverage: "none"
-          cache-key-suffix: "-locked"
+          extensions: ':psr, bcmath, dom, hash, json, mbstring, xml, xmlwriter, xmlreader, zlib, pg_query-https://github.com/flow-php/[email protected]'
+          cache-key-suffix: "-locked-phar"
 
       - name: "Build PHAR file"
         run: "composer build:phar"
@@ -173,11 +174,15 @@ jobs:
         with:
           php-version: "${{ matrix.php-version }}"
           dependencies: "locked"
-          cache-key-suffix: "-website"
+          extensions: ':psr, bcmath, dom, hash, json, mbstring, xml, xmlwriter, xmlreader, zlib, pg_query-https://github.com/flow-php/[email protected]'
+          cache-key-suffix: "-website-phar"
 
       - name: "Generate documentation"
         run: "composer build:docs"
 
+      - name: "Build latest version Flow Phar for playground"
+        run: "composer build:phar"
+
       - name: "Install Landing dependencies"
         run: "composer install --no-interaction --no-progress "
         working-directory: "web/landing"

.github/workflows/job-mutation-tests.yml

@@ -35,9 +35,9 @@ jobs:
         uses: "actions/cache@v4"
         with:
           path: "var/infection/cache"
-          key: "php-${{ matrix.php-version }}-cache-infection-${{ hashFiles('infection.json', 'composer.lock') }}"
+          key: "php-${{ matrix.php-version }}-infection-cache-${{ hashFiles('infection.json', 'tools/infection/phpunit.xml', 'composer.lock') }}"
           restore-keys: |
-            php-${{ matrix.php-version }}-cache-infection-
+            php-${{ matrix.php-version }}-infection-cache-
 
       - name: "Mutation Tests"
         run: |

.github/workflows/release.yml

@@ -27,6 +27,7 @@ jobs:
           tools: composer:v2
           php-version: "${{ matrix.php-version }}"
           ini-values: memory_limit=-1
+          extensions: ':psr, bcmath, dom, hash, json, mbstring, xml, xmlwriter, xmlreader, zlib, pg_query-https://github.com/flow-php/[email protected]'
 
       - name: "Get Composer Cache Directory"
         id: composer-cache

bin/docs.php

@@ -61,6 +61,7 @@ public function execute(InputInterface $input, OutputInterface $output) : int
             __DIR__ . '/../src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/functions.php',
             __DIR__ . '/../src/lib/filesystem/src/Flow/Filesystem/DSL/functions.php',
             __DIR__ . '/../src/lib/types/src/Flow/Types/DSL/functions.php',
+            __DIR__ . '/../src/lib/pg-query/src/Flow/PgQuery/DSL/functions.php',
             __DIR__ . '/../src/bridge/filesystem/azure/src/Flow/Filesystem/Bridge/Azure/DSL/functions.php',
             __DIR__ . '/../src/bridge/filesystem/async-aws/src/Flow/Filesystem/Bridge/AsyncAWS/DSL/functions.php',
             __DIR__ . '/../src/lib/azure-sdk/src/Flow/Azure/SDK/DSL/functions.php',

phpdoc/lib.pg-query.xml

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<phpdocumentor
+        configVersion="3"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xmlns="https://www.phpdoc.org"
+>
+    <title>Flow PHP</title>
+    <paths>
+        <output>./../web/landing/build/documentation/api/lib/pg-query</output>
+        <cache>./../var/phpdocumentor/cache/lib/pg-query</cache>
+    </paths>
+    <version number="1.x">
+        <api format="php">
+            <source dsn="./../">
+                <path>src/lib/pg-query/src</path>
+            </source>
+            <output>pg-query</output>
+            <default-package-name>PG Query</default-package-name>
+            <visibility>public</visibility>
+            <include-source>false</include-source>
+        </api>
+    </version>
+    <setting name="template.color" value="orange"/>
+</phpdocumentor>

src/cli/box.json

@@ -13,14 +13,13 @@
     "dotenv",
     "var-exporter"
   ],
+  "check-requirements": false,
   "compression": "GZ",
   "directories": [
     "src",
     "vendor"
   ],
   "exclude-dev-files": false,
   "git-version": "git_version",
-  "main": "flow",
-  "metadata": "FlowVersion::getVersion",
   "output": "./../../build/flow.phar"
 }

src/cli/composer.json

@@ -18,7 +18,10 @@
         "flow-php/etl-adapter-json": "self.version",
         "flow-php/etl-adapter-text": "self.version",
         "flow-php/etl-adapter-xml": "self.version",
-        "flow-php/parquet-viewer": "self.version"
+        "flow-php/etl-adapter-excel": "self.version",
+        "flow-php/etl-adapter-chartjs": "self.version",
+        "flow-php/openapi-specification-bridge": "self.version",
+        "flow-php/pg-query": "1.x-dev"
     },
     "config": {
         "optimize-autoloader": true,

src/cli/flow

@@ -1,6 +1,14 @@
 #!/usr/bin/env php
 <?php declare(strict_types=1);
 
+if (PHP_VERSION_ID < 80200) {
+    fwrite(STDERR, sprintf(
+        "Flow PHP requires PHP 8.2 or higher. You are running PHP %s.\n",
+        PHP_VERSION
+    ));
+    exit(1);
+}
+
 use Flow\CLI\Command\DatabaseTableListCommand;
 use Flow\CLI\Command\DatabaseTableSchemaCommand;
 use Flow\CLI\Command\FileAnalyzeCommand;

src/core/etl/src/Flow/ETL/Attribute/Module.php

@@ -22,6 +22,7 @@ enum Module : string
     case JSON = 'JSON';
     case MEILI_SEARCH = 'MEILI_SEARCH';
     case PARQUET = 'PARQUET';
+    case PG_QUERY = 'PG_QUERY';
     case S3_FILESYSTEM = 'S3_FILESYSTEM';
     case TEXT = 'TEXT';
     case TYPES = 'TYPES';

src/extension/pg-query-ext/ext/config.m4

@@ -127,8 +127,10 @@ if test "$PHP_PG_QUERY" != "no"; then
     AC_MSG_ERROR([libpg_query.a not found in $PG_QUERY_LIB_DIR])
   fi
 
-  dnl Add protobuf-c dependency
-  PHP_ADD_LIBRARY(protobuf-c,, PG_QUERY_SHARED_LIBADD)
+  dnl protobuf-c is bundled in libpg_query.a for static builds
+  if test "$ext_shared" = "yes"; then
+    PHP_ADD_LIBRARY(protobuf-c,, PG_QUERY_SHARED_LIBADD)
+  fi
 
   PHP_SUBST(PG_QUERY_SHARED_LIBADD)
 

src/lib/parquet/src/Flow/Parquet/Thrift/CompactProtocol.php

@@ -303,15 +303,15 @@ public function readI64(&$i64) : int
         $xorer = 0;
 
         if ($lo & 1) {
-            $xorer = 0xFFFFFFFF;
+            $xorer = (int) 0xFFFFFFFF;
         }
         $lo = ($lo >> 1) & 0x7FFFFFFF;
         $lo |= (($hi & 1) << 31);
         $hi = ($hi >> 1) ^ $xorer;
         $lo ^= $xorer;
 
         // Now put $hi and $lo back together
-        $isNeg = $hi < 0 || $hi & 0x80000000;
+        $isNeg = $hi < 0 || $hi & (int) 0x80000000;
 
         // Check for a negative
         if ($isNeg) {
@@ -330,12 +330,12 @@ public function readI64(&$i64) : int
         // explicitly below
         if ($hi & (int) 0x80000000) {
             $hi &= (int) 0x7FFFFFFF;
-            $hi += 0x80000000;
+            $hi += (int) 0x80000000;
         }
 
         if ($lo & (int) 0x80000000) {
             $lo &= (int) 0x7FFFFFFF;
-            $lo += 0x80000000;
+            $lo += (int) 0x80000000;
         }
 
         // Create as negative value first, since we can store -2^63 but not 2^63
@@ -704,13 +704,13 @@ public function writeI64(int $value) : int
             $xorer = 0;
 
             if ($neg) {
-                $xorer = 0xFFFFFFFF;
+                $xorer = (int) 0xFFFFFFFF;
             }
             $lowbit = ($lo >> 31) & 1;
             $hi = ($hi << 1) | $lowbit;
             $lo <<= 1;
-            $lo = ($lo ^ $xorer) & 0xFFFFFFFF;
-            $hi = ($hi ^ $xorer) & 0xFFFFFFFF;
+            $lo = ($lo ^ $xorer) & (int) 0xFFFFFFFF;
+            $hi = ($hi ^ $xorer) & (int) 0xFFFFFFFF;
 
             // now write out the varint, ensuring we shift both hi and lo
             $out = '';

src/lib/pg-query/src/Flow/PgQuery/DSL/functions.php

@@ -4,31 +4,48 @@
 
 namespace Flow\PgQuery\DSL;
 
+use Flow\ETL\Attribute\{DocumentationDSL, Module, Type as DSLType};
 use Flow\PgQuery\{ParsedQuery, Parser};
 
+#[DocumentationDSL(module: Module::PG_QUERY, type: DSLType::HELPER)]
 function pg_parser() : Parser
 {
     return new Parser();
 }
 
+#[DocumentationDSL(module: Module::PG_QUERY, type: DSLType::HELPER)]
 function pg_parse(string $sql) : ParsedQuery
 {
     return (new Parser())->parse($sql);
 }
 
+/**
+ * Returns a fingerprint of the given SQL query.
+ * Literal values are normalized so they won't affect the fingerprint.
+ */
+#[DocumentationDSL(module: Module::PG_QUERY, type: DSLType::HELPER)]
 function pg_fingerprint(string $sql) : ?string
 {
     return (new Parser())->fingerprint($sql);
 }
 
+/**
+ * Normalize SQL query by replacing literal values and named parameters with positional parameters.
+ * WHERE id = :id will be changed into WHERE id = $1
+ * WHERE id = 1 will be changed into WHERE id = $1.
+ */
+#[DocumentationDSL(module: Module::PG_QUERY, type: DSLType::HELPER)]
 function pg_normalize(string $sql) : ?string
 {
     return (new Parser())->normalize($sql);
 }
 
 /**
+ * Split string with multiple SQL statements into array of individual statements.
+ *
  * @return array<string>
  */
+#[DocumentationDSL(module: Module::PG_QUERY, type: DSLType::HELPER)]
 function pg_split(string $sql) : array
 {
     return (new Parser())->split($sql);

tools/infection/phpunit.xml

@@ -22,12 +22,12 @@
   <source>
     <include>
         <directory suffix=".php">../../src/core/etl/src</directory>
-        <directory suffix=".php">../../src/core/lib/array-dot</directory>
-        <directory suffix=".php">../../src/core/lib/dremel</directory>
-        <directory suffix=".php">../../src/core/lib/types</directory>
-        <directory suffix=".php">../../src/core/lib/doctrine-dbal-bulk</directory>
-        <directory suffix=".php">../../src/core/lib/filesystem</directory>
-        <directory suffix=".php">../../src/core/lib/parquet</directory>
+        <directory suffix=".php">../../src/lib/array-dot/src</directory>
+        <directory suffix=".php">../../src/lib/dremel/src</directory>
+        <directory suffix=".php">../../src/lib/types/src</directory>
+        <directory suffix=".php">../../src/lib/doctrine-dbal-bulk/src</directory>
+        <directory suffix=".php">../../src/lib/filesystem/src</directory>
+        <directory suffix=".php">../../src/lib/parquet/src</directory>
     </include>
     <exclude>
       <directory suffix=".php">../../src/lib/parquet/src/Flow/Parquet/Thrift</directory>

wasm/.gitignore

@@ -4,6 +4,8 @@
 *.phar.br
 manifest.json
 /libxml2-*
+/libpg_query
+/php-ext-snappy
 
 # Build logs
 *.log

wasm/build.sh

@@ -47,6 +47,25 @@ if [ ! -d "$LIBXML2_DIR" ]; then
     cd $PROJECT_ROOT
 fi
 
+echo "Build libpg_query for WebAssembly"
+LIBPG_QUERY_VERSION=17-6.1.0
+LIBPG_QUERY_DIR=libpg_query
+LIBPG_QUERY_INSTALL_DIR="$PROJECT_ROOT/$LIBPG_QUERY_DIR"
+
+if [ ! -d "$LIBPG_QUERY_DIR" ]; then
+    git clone --depth=1 --branch=$LIBPG_QUERY_VERSION \
+        https://github.com/pganalyze/libpg_query.git "$LIBPG_QUERY_DIR"
+    cd $LIBPG_QUERY_DIR
+
+    # Build with Emscripten - override CC and AR
+    # The Makefile does AR := $(AR) rs, but command-line AR overrides this
+    # So we must include the 'rs' flags ourselves. However, LLVM ar doesn't support 'g',
+    # and the Makefile @ suppresses the echo, so we pass 'rcs' which is compatible.
+    emmake make build -j$(nproc) CC=emcc AR="emar rcs"
+
+    cd $PROJECT_ROOT
+fi
+
 echo "Download and extract PHP if needed"
 if [ ! -d "$PHP_PATH" ]; then
     if [ ! -e $PHP_PATH.tar.xz ]; then
@@ -55,10 +74,29 @@ if [ ! -d "$PHP_PATH" ]; then
     tar xf $PHP_PATH.tar.xz
 fi
 
+echo "Add pg_query extension"
+PG_QUERY_EXT_SRC="$PROJECT_ROOT/../src/extension/pg-query-ext/ext"
+PG_QUERY_EXT_DST="$PHP_PATH/ext/pg_query"
+
+rm -rf "$PG_QUERY_EXT_DST"
+cp -r "$PG_QUERY_EXT_SRC" "$PG_QUERY_EXT_DST"
+
+echo "Add snappy extension"
+SNAPPY_EXT_DIR=php-ext-snappy
+if [ ! -d "$SNAPPY_EXT_DIR" ]; then
+    git clone --recursive --depth=1 https://github.com/kjdev/php-ext-snappy.git "$SNAPPY_EXT_DIR"
+fi
+
+SNAPPY_EXT_DST="$PHP_PATH/ext/snappy"
+rm -rf "$SNAPPY_EXT_DST"
+cp -r "$SNAPPY_EXT_DIR" "$SNAPPY_EXT_DST"
+
 echo "Configure PHP"
 
-export CFLAGS="-O3 -flto -fPIC -DZEND_MM_ERROR=0 -I$LIBXML2_INSTALL_DIR/include/libxml2 -sUSE_ZLIB=1"
-export LDFLAGS="-L$LIBXML2_INSTALL_DIR/lib -sUSE_ZLIB=1"
+# Use -Oz for size optimization instead of -O3 for speed
+export CFLAGS="-Oz -flto -fPIC -g0 -DZEND_MM_ERROR=0 -I$LIBXML2_INSTALL_DIR/include/libxml2 -I$LIBPG_QUERY_INSTALL_DIR -sUSE_ZLIB=1"
+export CXXFLAGS="-Oz -flto -fPIC -g0 -std=c++11 -sUSE_ZLIB=1"
+export LDFLAGS="-L$LIBXML2_INSTALL_DIR/lib -L$LIBPG_QUERY_INSTALL_DIR -sUSE_ZLIB=1"
 
 cd $PHP_PATH
 
@@ -102,7 +140,10 @@ emconfigure ./configure \
   --enable-xml \
   --enable-dom \
   --enable-xmlreader \
-  --enable-xmlwriter
+  --enable-xmlwriter \
+  --enable-pg-query \
+  --with-pg-query=$LIBPG_QUERY_INSTALL_DIR \
+  --enable-snappy
 
 if [ $? -ne 0 ]; then
     echo "emconfigure failed. Content of config.log:"
@@ -148,7 +189,7 @@ emcc $CFLAGS $LDFLAGS \
   -s ASYNCIFY=1 \
   -s STACK_OVERFLOW_CHECK=0 \
   -s SAFE_HEAP=0 \
-  libs/libphp.a pib_eval.o $LIBXML2_INSTALL_DIR/lib/libxml2.a -o out/php.js
+  libs/libphp.a pib_eval.o $LIBXML2_INSTALL_DIR/lib/libxml2.a $LIBPG_QUERY_INSTALL_DIR/libpg_query.a -o out/php.js
 
 echo "Copy outputs to web/landing/assets/wasm"
 OUTPUT_DIR="$PROJECT_ROOT/../web/landing/assets/wasm"