Automatically download missing packages from repo.magento.com every day (#215)

* Additional packages

* Additional packages

* work

* work

* work

* work

* work

* work

* work

* work

Author: fballiano

.github/workflows/download-missing-packages.yml

@@ -0,0 +1,74 @@
+name: Download missing packages from repo.magento.com
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 0 * * *'
+
+jobs:
+  download-packages:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup PHP
+        uses: shivammathur/setup-php@v2
+        with:
+          php-version: '8.3'
+          tools: composer
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+
+      - name: Configure Git
+        run: |
+          git config user.name "GitHub Actions Bot"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+      - name: Run mirror script
+        run: node src/make/mirror.js --outputDir=build
+
+      - name: Run download script
+        run: php bin/download-all-missing-packages-from-repo-magento-com.php
+        env:
+          COMPOSER_AUTH: ${{ secrets.COMPOSER_AUTH }}
+
+      - name: Check for changes
+        id: git-check
+        run: |
+          git add resource/additional-packages
+          if git diff --staged --quiet; then
+            echo "No changes detected in resource/additional-packages"
+            echo "changes=false" >> $GITHUB_OUTPUT
+          else
+            echo "Changes detected in resource/additional-packages"
+            echo "changes=true" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Create Pull Request
+        if: steps.git-check.outputs.changes == 'true'
+        run: |
+          # Create a new branch with timestamp
+          BRANCH_NAME="update-packages-$(date +%Y%m%d-%H%M%S)"
+          git checkout -b $BRANCH_NAME
+
+          # Commit changes
+          git commit -m "Add new packages from repo.magento.com"
+
+          # Push to the repository
+          git push origin $BRANCH_NAME
+
+          # Create PR using GitHub CLI
+          gh pr create \
+            --title "Add new packages from repo.magento.com" \
+            --body "This PR adds new packages downloaded from repo.magento.com by the automated workflow. Please rebuild the mirror after merging." \
+            --repo "mage-os/generate-mirror-repo-js" \
+            --base main \
+            --head $BRANCH_NAME
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

bin/download-all-missing-packages-from-repo-magento-com.php

@@ -0,0 +1,70 @@
+#!/usr/bin/env php
+<?php
+
+/**
+ * Script to download the latest Magento packages
+ * This script:
+ * 1. Finds the latest version for each major.minor release in resource/history/magento/magento2-base
+ * 2. Uses composer to create a project with each version
+ * 3. Uses the download-missing-packages-from-repo-magento-com.php script to download missing packages
+ * 4. Removes the project directory
+ */
+
+// Set error reporting
+error_reporting(E_ALL);
+ini_set('display_errors', 1);
+
+// Directory containing version files
+$versionDir = "resource/history/magento/magento2-base";
+
+// Get all version files
+$versionFiles = glob("$versionDir/*.json");
+if (empty($versionFiles)) {
+    echo "Error: No version files found in $versionDir\n";
+    exit(1);
+}
+
+// Collect all versions
+$versions = [];
+foreach ($versionFiles as $file) {
+    $versions[] = basename($file, '.json');
+}
+
+usort($versions, function($a, $b) {
+    return version_compare($a, $b);
+});
+
+$latestVersions = [];
+foreach ($versions as $version) {
+    $versionAndPatch = explode('-', $version);
+    $latestVersions[$versionAndPatch[0]] = $version;
+}
+
+// Process each version
+foreach ($latestVersions as $version) {
+    echo "Processing version: $version\n";
+
+    // Create project directory using composer
+    echo "Creating Magento project with version $version...\n";
+    $command = "composer create-project --repository-url=https://repo.magento.com/ magento/project-community-edition:$version --ignore-platform-reqs --no-progress -q -n --no-plugins";
+    passthru($command, $returnCode);
+
+    if ($returnCode !== 0) {
+        echo "Error: Composer command failed with return code $returnCode\n";
+        continue;
+    }
+
+    // Download missing packages
+    echo "Downloading missing packages...\n";
+    $command = "php bin/download-missing-packages-from-repo-magento-com.php project-community-edition/composer.lock build resource/additional-packages";
+    passthru($command, $returnCode);
+
+    // Remove project directory
+    echo "Cleaning up project directory...\n";
+    $command = "rm -rf project-community-edition";
+    passthru($command);
+
+    echo "\n\n";
+}
+
+echo "Done!\n";