Sourcebroker update

README.md

@@ -84,14 +84,7 @@ set('shared_dirs', [
 
 ## Non-git deployment
 
-If the source host has no access to the git repository, you can replace `deploy:update_code` with the new `deploy:upload_code` task to transfer all needed files.
-
-```
-task('deploy:update_code')->disable();
-after('deploy:update_code', 'deploy:upload_code');
-```
-
-The files to upload become configured via [`upload_paths`](https://github.com/xima-media/xima-deployer-extended-typo3/blob/main/set.php#L61).
+All files become uploaded via rsync and can be configured via [`upload_paths`](https://github.com/xima-media/xima-deployer-extended-typo3/blob/main/set.php#L61).
 
 ## Reset from Gitlab Artifact
 
@@ -129,7 +122,7 @@ export-job:
 import-job:
   ...
   script:
-    - vendor/bin/dep reset:from_gitlab_artifact --options="token:$CI_VARIABLE_WITH_API_TOKEN,dumpcode:myArtifact" host-a
+    - vendor/bin/dep reset:from_gitlab_artifact --options="txToken:$CI_VARIABLE_WITH_API_TOKEN,txDumpcode:myArtifact" host-a
   when: manual
 ```
 

composer.json

@@ -11,7 +11,12 @@
     }
   ],
   "require": {
-    "sourcebroker/deployer-extended-typo3": "22.0.0"
+    "helhum/dotenv-connector": "^3.2.0",
+    "helhum/typo3-console": "^8.2.3",
+    "sourcebroker/deployer-typo3-deploy-ci": "^2.0.0",
+    "sourcebroker/deployer-typo3-database": "^1.0.0",
+    "sourcebroker/deployer-typo3-media": "^1.0.0",
+    "sourcebroker/deployer-extended": "^24.0.0"
   },
   "autoload": {
     "psr-4": {

set.php → deployer/config.php

@@ -68,6 +68,9 @@
     ],
 );
 
+// Common random that can be used between tasks. Must be in form that can be used directly in filename!
+set('random', md5(time() . mt_rand()));
+
 // disable composer version check
 set('composer_channel_autoupdate', false);
 
@@ -88,3 +91,18 @@
     'public/typo3conf/AdditionalConfiguration.php',
     'var/labels',
 ]);
+
+// Configure request buffering to avoid errors during deployments
+set('buffer_config', function () {
+    return [
+        'index.php' => [
+            'entrypoint_filename' => get('web_path') . 'index.php',
+        ],
+        'typo3/index.php' => [
+            'entrypoint_filename' => get('web_path') . 'typo3/index.php',
+        ],
+        'typo3/install.php' => [
+            'entrypoint_filename' => get('web_path') . 'typo3/install.php',
+        ]
+    ];
+});

deployer/tasks/buffer_start.php

@@ -0,0 +1,139 @@
+<?php
+
+namespace Deployer;
+
+use Deployer\Exception\GracefulShutdownException;
+
+// Read more on https://github.com/sourcebroker/deployer-extended#buffer-start
+task('buffer:start', function () {
+    if (empty(get('deploy_path'))) {
+        throw new GracefulShutdownException('The "deploy_path" var is empty but its used for file operations so its dangerous state.');
+    }
+    if (empty(get('random'))) {
+        throw new GracefulShutdownException('The "random" config var is empty. Its needed for file operation.');
+    }
+    if (preg_match('/^[a-zA-Z0-9]$/', get('random'))) {
+        throw new GracefulShutdownException('The "random" config var should be only /^[a-zA-Z0-9]$/ because its used in filenames.');
+    }
+    foreach (['release', 'current'] as $overwriteRelease) {
+        $overwriteReleasePath = get('deploy_path') . '/' . $overwriteRelease . '/';
+        if (test("[ -e $overwriteReleasePath ]")) {
+            $tempPath = $overwriteReleasePath . get('random');
+            run('[ -e ' . $tempPath . ' ] || mkdir -p ' . $tempPath);
+            $entrypointInjectStartComment = "\n\n// deployer extended buffering request code START\n";
+            $entrypointInjectEndComment = "\n// deployer extended buffering request code END\n\n";
+            foreach ((array)get('buffer_config') as $inject) {
+                if (empty($inject['entrypoint_filename'])) {
+                    throw new GracefulShutdownException('entrypoint_filename not set for buffer_data');
+                }
+                // general inject settings
+                $entrypointFilename = $inject['entrypoint_filename'];
+                if (empty($inject['entrypoint_needle'])) {
+                    $entrypointNeedle = '<?php';
+                } else {
+                    $entrypointNeedle = $inject['entrypoint_needle'];
+                }
+
+                // Buffering php requests
+                if (empty($inject['requestbuffer_sleep'])) {
+                    $requestBufferSleep = 200000; // 200ms
+                } else {
+                    $requestBufferSleep = (int)$inject['requestbuffer_sleep'];
+                }
+                if (empty($inject['requestbuffer_flag_filename'])) {
+                    $requestBufferFlagFilename = '.flag.requestbuffer';
+                } else {
+                    $requestBufferFlagFilename = $inject['requestbuffer_flag_filename'];
+                }
+                if (empty($inject['requestbuffer_duration'])) {
+                    $requestBufferDuration = 60;
+                } else {
+                    $requestBufferDuration = (int)$inject['requestbuffer_duration'];
+                }
+
+                // Action if php choose old release to serve request
+                if (empty($inject['oldrelease_flag__filename'])) {
+                    $oldReleaseFlagFilename = '.flag.oldrelease';
+                } else {
+                    $oldReleaseFlagFilename = $inject['oldrelease_flag_filename'];
+                }
+                if (empty($inject['oldrelease_redirect_sleep'])) {
+                    $oldReleaseRedirectSleep = 1000000; // 1s
+                } else {
+                    $oldReleaseRedirectSleep = (int)$inject['oldrelease_redirect_sleep'];
+                }
+                // Clearstatcache for n seconds after deploy
+                if (empty($inject['clearstatcache_flag_filename'])) {
+                    $clearStatCacheFlagFilename = '.flag.clearstatcache';
+                } else {
+                    $clearStatCacheFlagFilename = $inject['clearstatcache_flag_filename'];
+                }
+                if (empty($inject['clearstatcache_duration'])) {
+                    $clearStatCacheDuration = 150; // 120s is php default -> http://php.net/realpath-cache-ttl
+                } else {
+                    $clearStatCacheDuration = (int)$inject['clearstatcache_duration'];
+                }
+
+                if (empty($inject['entrypoint_inject'])) {
+                    $entrypointInject =
+                        "// Buffering php requests\n" .
+                        "isset(\$_SERVER['HTTP_X_DEPLOYER_DEPLOYMENT']) && \$_SERVER['HTTP_X_DEPLOYER_DEPLOYMENT'] == '{{random}}' ? \$deployerExtendedEnableBufferLock = false: \$deployerExtendedEnableBufferLock = true;\n" .
+                        "isset(\$_ENV['DEPLOYER_DEPLOYMENT']) && \$_ENV['DEPLOYER_DEPLOYMENT'] == '{{random}}' ? \$deployerExtendedEnableBufferLock = false: \$deployerExtendedEnableBufferLock = true;\n" .
+                        "clearstatcache(true, __DIR__ . '/$requestBufferFlagFilename');\n" .
+                        "while (file_exists(__DIR__ . '/$requestBufferFlagFilename') && \$deployerExtendedEnableBufferLock) {\n" .
+                        "    usleep($requestBufferSleep);\n" .
+                        "    clearstatcache(true);\n" .
+                        "    if(time() - @filectime(__DIR__ . '/$requestBufferFlagFilename') > $requestBufferDuration) @unlink(__DIR__ . '/$requestBufferFlagFilename');\n" .
+                        "}\n" .
+                        "// Clearstatcache for n seconds after deploy" .
+                        "clearstatcache(true, __DIR__ . '/$clearStatCacheFlagFilename')\n" .
+                        "if (file_exists(__DIR__ . '/$clearStatCacheFlagFilename')) {\n" .
+                        "    clearstatcache(true);\n" .
+                        "    if(time() - @filectime(__DIR__ . '/$clearStatCacheFlagFilename') > $clearStatCacheDuration) @unlink(__DIR__ . '/$clearStatCacheFlagFilename');\n" .
+                        "}\n" .
+                        "// Action if php choose old release to serve request" .
+                        "clearstatcache(true, __DIR__ . '/$oldReleaseFlagFilename')\n" .
+                        "if(file_exists(__DIR__ . '/$oldReleaseFlagFilename')) {\n" .
+                        "    clearstatcache(true);\n" .
+                        "    usleep($oldReleaseRedirectSleep);\n" .
+                        "    if(!empty(\$_SERVER['REQUEST_SCHEME']) && !empty(\$_SERVER['SERVER_NAME'])) {\n" .
+                        "      header('Location: ' . \$_SERVER['REQUEST_SCHEME'] . '://' . \$_SERVER['SERVER_NAME'] . !empty(\$_SERVER['REQUEST_URI']) ? \$_SERVER['REQUEST_URI'] : '', true, 307);\n" .
+                        "    } else {\n" .
+                        "      exit();\n" .
+                        "    }\n" .
+                        "}";
+                } else {
+                    $entrypointInject = $inject['entrypoint_inject'];
+                }
+                $entrypointFileContent = trim(run('cd {{release_path}} && [ -f ' . $entrypointFilename . ' ] && cat ' . $entrypointFilename . ' || echo ""'));
+                if (strpos($entrypointFileContent, $entrypointInjectStartComment) === false) {
+                    if (!empty($entrypointFileContent)) {
+                        $pos = strpos($entrypointFileContent, $entrypointNeedle);
+                        if ($pos !== false) {
+                            $content = substr_replace(
+                                $entrypointFileContent,
+                                $entrypointNeedle .
+                                $entrypointInjectStartComment . $entrypointInject . $entrypointInjectEndComment,
+                                $pos,
+                                strlen($entrypointNeedle)
+                            );
+                            $entrypointAbsolutePath = $overwriteReleasePath . $entrypointFilename;
+                            run('cp -p ' . $entrypointAbsolutePath . ' ' . $tempPath . '/');
+                            run('echo ' . escapeshellarg($content) . ' > ' . $tempPath . '/' . basename($entrypointAbsolutePath));
+                            run('mv -fT ' . $tempPath . '/' . basename($entrypointAbsolutePath) . ' ' . $entrypointAbsolutePath);
+                        } else {
+                            throw new GracefulShutdownException('Can not find needle to inject with inclusion.');
+                        }
+                    } else {
+                        throw new GracefulShutdownException('Can not find file to overwrite or the file is empty. File that was read is: ' . $overwriteReleasePath . $entrypointFilename);
+                    }
+                }
+                if (test('[ -e ' . $overwriteReleasePath . dirname($entrypointFilename) . ' ]')) {
+                    run('cd ' . $overwriteReleasePath . ' && touch ' . (dirname($entrypointFilename) ? dirname($entrypointFilename) . '/' : '') . $requestBufferFlagFilename);
+                    run('cd ' . $overwriteReleasePath . ' && touch ' . (dirname($entrypointFilename) ? dirname($entrypointFilename) . '/' : '') . $clearStatCacheFlagFilename);
+                }
+            }
+            run('rmdir ' . $tempPath);
+        }
+    }
+})->desc('Start buffering requests to application entry points');

deployer/tasks/buffer_stop.php

@@ -0,0 +1,43 @@
+<?php
+
+namespace Deployer;
+
+// Read more on https://github.com/sourcebroker/deployer-extended#buffer-stop
+use Deployer\Exception\GracefulShutdownException;
+
+task('buffer:stop', function () {
+    if (empty(get('deploy_path'))) {
+        throw new GracefulShutdownException('The "deploy_path" var is empty but its used for file operations so its dangerous state.');
+    }
+    $overwriteReleases = ['current'];
+    $releasesList = get('releases_list');
+    // Add .flag.oldrelease for cases when php cache will decide to run code in old release path
+    if (isset($releasesList[1])) {
+        $overwriteReleases[] = 'releases/' . $releasesList[1];
+        foreach (get('buffer_config') as $inject) {
+            if (empty($inject['entrypoint_filename'])) {
+                throw new GracefulShutdownException('entrypoint_filename not set for buffer_data');
+            }
+            $overwriteReleasePath = get('deploy_path') . '/releases/' . $releasesList[1];
+            $oldReleaseFlagFilename = empty($inject['oldrelease_flag__filename']) ?
+                '.flag.oldrelease' : $inject['oldrelease_flag_filename'];
+            $entrypointDirectory = dirname($inject['entrypoint_filename']) === '.' ? '' : dirname($inject['entrypoint_filename']);
+            if (test('[ -e ' . $overwriteReleasePath . '/' . $entrypointDirectory . ' ]')) {
+                run('touch ' . $overwriteReleasePath . '/' . $entrypointDirectory . '/' . $oldReleaseFlagFilename);
+            }
+        }
+    }
+    // Remove .flag.requestbuffer also from previous release because it can be still read because php cache
+    foreach ($overwriteReleases as $overwriteRelease) {
+        $overwriteReleasePath = get('deploy_path') . '/' . $overwriteRelease;
+        foreach (get('buffer_config') as $inject) {
+            if (empty($inject['entrypoint_filename'])) {
+                throw new GracefulShutdownException('entrypoint_filename not set for buffer_data');
+            }
+            $activateBufferFlagFilename = empty($inject['requestbuffer_flag_filename']) ?
+                '.flag.requestbuffer' : $inject['requestbuffer_flag_filename'];
+            $entrypointDirectory = dirname($inject['entrypoint_filename']) === '.' ? '' : dirname($inject['entrypoint_filename']) . '/';
+            run('cd ' . $overwriteReleasePath . ' && rm -f ' . $entrypointDirectory . $activateBufferFlagFilename);
+        }
+    }
+})->desc('Stop buffering requests to application entry points');

recipe/db_init.php → deployer/tasks/db_init.php

@@ -2,22 +2,22 @@
 
 namespace Deployer;
 
-use Xima\XimaDeployerExtendedTypo3\Utility\OptionUtility;
+use SourceBroker\DeployerExtendedDatabase\Utility\OptionUtility;
 
 /**
  * Check for missing database: Run database updateschema + import database of base branch
  */
 task('db:init', function () {
     $optionUtility = new OptionUtility(input()->getOption('options'));
-    $baseBranch = $optionUtility->getOption('base_branch');
+    $baseBranch = $optionUtility->getOption('txBaseBranch');
 
     // abort if feature branch has already been configured
     if (!$baseBranch || !get('argument_host')) {
         return;
     }
 
     $activePath = get('deploy_path') . '/' . (test('[ -L {{deploy_path}}/release ]') ? 'release' : 'current');
-    $hasPageTable = (boolean)run('cd ' . $activePath . ' && echo "SHOW TABLES LIKE \'pages\';" | {{bin/php}} {{bin/typo3cms}} database:import');
+    $hasPageTable = (boolean)run('cd ' . $activePath . ' && echo "SHOW TABLES LIKE \'pages\';" | {{bin/php}} {{local/bin/typo3}} database:import');
 
     // no database import if pages table exists
     if ($hasPageTable) {
@@ -28,7 +28,7 @@
     $baseStage = str_replace(strtolower(get('branch')), strtolower($baseBranch), $targetHost);
 
     // update schema (db:import would fail with empty database)
-    run('cd ' . $activePath . ' && {{bin/php}} {{bin/typo3cms}} database:updateschema');
+    run('cd ' . $activePath . ' && {{bin/php}} {{local/bin/typo3}} database:updateschema');
 
     // abort in case base is target
     if ($baseStage === $targetHost) {