feat: V3 Granular Catalog Architecture - Atomic Writes & Race Condition Prevention

.github/workflows/auto-release.yml

@@ -34,27 +34,34 @@ jobs:
       matrix: ${{ steps.detect.outputs.matrix }}
     steps:
       - uses: actions/checkout@v6
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
         with:
-          fetch-depth: 2  # Need at least 2 commits to diff
+          python-version: '3.12'
+          cache: 'pip'
+          cache-dependency-path: 'requirements/base.txt'
+
+      - name: Install dependencies
+        run: pip install -r requirements/base.txt
 
       - name: 🔍 Detect Changed Exporters
         id: detect
         env:
           INPUT_EXPORTERS: ${{ inputs.exporters }}
+          CATALOG_URL: https://sckyzo.github.io/monitoring-hub/catalog/index.json
         run: |
-          echo "::group::🔍 Detecting modified exporters"
+          echo "::group::🔍 Detecting exporters to build"
 
           # Check if workflow was manually triggered
           if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
             echo "Manual trigger detected"
-            echo "Input length: ${#INPUT_EXPORTERS}"
-            echo "Input value: '$INPUT_EXPORTERS'"
 
-            # Check if specific exporters were provided (non-empty input)
-            if [ "${#INPUT_EXPORTERS}" -gt 0 ]; then
+            # Check if specific exporters were provided
+            if [ -n "$INPUT_EXPORTERS" ]; then
               echo "Building specific exporters: $INPUT_EXPORTERS"
 
-              # Convert comma-separated list to JSON array (trim spaces and filter empty)
+              # Convert comma-separated list to JSON array
               EXPORTERS_JSON=$(echo "$INPUT_EXPORTERS" | jq -R -c \
                 'split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))')
 
@@ -66,58 +73,36 @@ jobs:
               echo "::endgroup::"
               exit 0
             else
-              # No specific exporters - build ALL
-              echo "Building ALL exporters"
-
-              # List all exporters (directories in exporters/)
-              ALL_EXPORTERS=$(ls -1 exporters/ | jq -R -s -c 'split("\n") | map(select(length > 0))')
-
-              echo "exporters=$ALL_EXPORTERS" >> $GITHUB_OUTPUT
-              echo "matrix={\"exporter\":$ALL_EXPORTERS}" >> $GITHUB_OUTPUT
-
-              COUNT=$(echo "$ALL_EXPORTERS" | jq '. | length')
-              echo "✓ Building ALL exporters (total: $COUNT)"
-              echo "::endgroup::"
-              exit 0
+              # No specific exporters - force rebuild ALL
+              echo "Force rebuild ALL exporters"
+              export FORCE_REBUILD=true
             fi
           fi
 
-          # Auto-detect changes from git diff (normal push workflow)
-          CHANGED_FILES=$(git diff --name-only HEAD~1 HEAD | grep '^exporters/' || true)
+          # Use state_manager for smart detection
+          echo "Using state_manager for change detection..."
+          python3 core/engine/state_manager.py
 
-          if [ -z "$CHANGED_FILES" ]; then
-            echo "No changes detected in exporters/"
-            echo "exporters=[]" >> $GITHUB_OUTPUT
+          # Read outputs from state_manager
+          EXPORTERS_JSON=$(grep "^exporters=" $GITHUB_OUTPUT | cut -d= -f2)
+          BUILD_NEEDED=$(grep "^build_needed=" $GITHUB_OUTPUT | cut -d= -f2)
+
+          if [ "$BUILD_NEEDED" = "false" ]; then
+            echo "ℹ️  No exporters need building (all up to date)"
             echo "matrix={\"exporter\":[]}" >> $GITHUB_OUTPUT
             echo "::endgroup::"
             exit 0
           fi
 
-          # Extract unique exporter names from changed paths
-          CHANGED_EXPORTERS=$(echo "$CHANGED_FILES" | cut -d'/' -f2 | sort -u)
-
-          # Convert to JSON array
-          EXPORTERS_JSON=$(echo "$CHANGED_EXPORTERS" | jq -R -s -c 'split("\n") | map(select(length > 0))')
-
-          echo "Changed exporters: $EXPORTERS_JSON"
-
-          # Count exporters
-          COUNT=$(echo "$EXPORTERS_JSON" | jq '. | length')
-          echo "Found $COUNT modified exporter(s)"
-
-          # Set outputs
-          echo "exporters=$EXPORTERS_JSON" >> $GITHUB_OUTPUT
+          # Generate matrix
           echo "matrix={\"exporter\":$EXPORTERS_JSON}" >> $GITHUB_OUTPUT
 
-          # Display summary
-          if [ "$COUNT" -gt 0 ]; then
-            echo "✓ Will trigger release for: $(echo "$EXPORTERS_JSON" | jq -r 'join(", ")')"
-          else
-            echo "ℹ️  No exporters detected"
-          fi
+          COUNT=$(echo "$EXPORTERS_JSON" | jq '. | length')
+          echo "✓ Detected $COUNT exporter(s) to build: $(echo "$EXPORTERS_JSON" | jq -r 'join(", ")')"
 
           echo "::endgroup::"
 
+
       - name: 📊 Generate Summary
         if: steps.detect.outputs.exporters != '[]'
         env:

.github/workflows/build-pr.yml

@@ -13,9 +13,52 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  generate-artifacts:
+  detect-changes:
+    name: 🔍 Detect Modified Exporters
     runs-on: ubuntu-latest
-    timeout-minutes: 30
+    timeout-minutes: 5
+    outputs:
+      exporters: ${{ steps.detect.outputs.exporters }}
+      has_changes: ${{ steps.detect.outputs.has_changes }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - name: 🔍 Detect Changed Exporters
+        id: detect
+        run: |
+          echo "::group::🔍 Detecting modified exporters in PR"
+
+          # Get changed files in this PR
+          git fetch origin ${{ github.base_ref }}
+          CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD | grep '^exporters/' || true)
+
+          if [ -z "$CHANGED_FILES" ]; then
+            echo "ℹ️  No changes in exporters/"
+            echo "exporters=[]" >> $GITHUB_OUTPUT
+            echo "has_changes=false" >> $GITHUB_OUTPUT
+            echo "::endgroup::"
+            exit 0
+          fi
+
+          # Extract unique exporter names
+          EXPORTERS=$(echo "$CHANGED_FILES" | cut -d'/' -f2 | sort -u | jq -R -s -c 'split("\n") | map(select(length > 0))')
+
+          echo "exporters=$EXPORTERS" >> $GITHUB_OUTPUT
+          echo "has_changes=true" >> $GITHUB_OUTPUT
+
+          COUNT=$(echo "$EXPORTERS" | jq '. | length')
+          echo "✓ Found $COUNT modified exporter(s): $(echo "$EXPORTERS" | jq -r 'join(", ")')"
+
+          echo "::endgroup::"
+
+  validate-manifests:
+    name: ✅ Validate Manifests
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    needs: detect-changes
+    if: needs.detect-changes.outputs.has_changes == 'true'
     steps:
       - uses: actions/checkout@v6
 
@@ -29,31 +72,63 @@ jobs:
       - name: Install dependencies
         run: pip install -r requirements/base.txt
 
-      - name: Build Exporters
+      - name: Validate Modified Exporters
+        env:
+          EXPORTERS: ${{ needs.detect-changes.outputs.exporters }}
         run: |
+          echo "::group::✅ Validating manifests"
           export PYTHONPATH=$GITHUB_WORKSPACE
-          for exporter in exporters/*/; do
-            if [ -f "${exporter}manifest.yaml" ]; then
-              name=$(basename "$exporter")
-              echo "Building $name..."
-              python3 -m core.engine.builder --manifest "${exporter}manifest.yaml" --output-dir "build/$name"
+
+          echo "$EXPORTERS" | jq -r '.[]' | while read exporter; do
+            manifest="exporters/$exporter/manifest.yaml"
+
+            if [ ! -f "$manifest" ]; then
+              echo "⚠️  Skipping $exporter (no manifest)"
+              continue
             fi
+
+            echo "Validating $exporter..."
+            python3 -c "
+from core.engine.schema import ExporterManifestSchema
+import yaml
+
+with open('$manifest') as f:
+    data = yaml.safe_load(f)
+
+schema = ExporterManifestSchema()
+errors = schema.validate(data)
+
+if errors:
+    print('❌ Validation failed for $exporter:')
+    for field, msgs in errors.items():
+        for msg in msgs if isinstance(msgs, list) else [msgs]:
+            print(f'  - {field}: {msg}')
+    exit(1)
+else:
+    print('✓ Valid')
+"
           done
+
+          echo "::endgroup::"
+
+      - name: Validate URLs
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          EXPORTERS: ${{ needs.detect-changes.outputs.exporters }}
+        run: |
+          echo "::group::🔗 Validating download URLs"
 
-      - name: Upload Artifacts
-        uses: actions/upload-artifact@v6
-        with:
-          name: generated-build-files
-          path: build/
-          retention-days: 7
+          echo "$EXPORTERS" | jq -r '.[]' | while read exporter; do
+            echo "Checking $exporter..."
+            python3 core/scripts/validate_urls.py "$exporter" || echo "⚠️  URL validation failed for $exporter (non-blocking)"
+          done
 
-  # Ceinture et Bretelles : Test d'intégration complet sur un exporter témoin
-  integration-test:
+          echo "::endgroup::"
+
+  canary-build:
+    name: 🧪 Canary Build (node_exporter)
     runs-on: ubuntu-latest
     timeout-minutes: 20
-    needs: generate-artifacts
+    needs: detect-changes
     steps:
       - uses: actions/checkout@v6
 
@@ -67,21 +142,23 @@ jobs:
       - name: Install dependencies
         run: pip install -r requirements/base.txt
 
-      - name: Full Pipeline Test (Node Exporter)
+      - name: Test Full Pipeline
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           chmod +x core/scripts/*.sh
-          # On utilise node_exporter comme "Canary" pour valider la chaîne
+          echo "Testing full pipeline with node_exporter as canary..."
           ./core/scripts/local_test.sh node_exporter --validate
 
-  # Test DEB Build and Installation
-  deb-canary-test:
+  deb-canary:
+    name: 🧪 DEB Build Test (${{ matrix.dist }})
     runs-on: ubuntu-latest
     timeout-minutes: 20
-    needs: generate-artifacts
+    needs: detect-changes
     strategy:
+      fail-fast: false
       matrix:
         dist: [ubuntu-22.04, debian-12]
-        arch: [amd64]
     steps:
       - uses: actions/checkout@v6
 
@@ -99,22 +176,52 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           DIST: ${{ matrix.dist }}
-          ARCH: ${{ matrix.arch }}
         run: |
           chmod +x core/scripts/*.sh
-          echo "Testing DEB build for node_exporter on $DIST ($ARCH)..."
-          ./core/scripts/test_deb_build.sh node_exporter "$DIST" "$ARCH"
+          echo "Testing DEB build for node_exporter on $DIST..."
+          ./core/scripts/test_deb_build.sh node_exporter "$DIST" amd64
 
-      - name: Validate DEB Package
+      - name: Validate Package
         run: |
-          deb_file=$(find build -name "*.deb" | grep node-exporter | head -1)
+          deb_file=$(find build -name "*.deb" | head -1)
 
           if [ -z "$deb_file" ]; then
-            echo "ERROR: No DEB file found"
+            echo "❌ No DEB file found"
             exit 1
           fi
 
-          echo "Validating DEB: $deb_file"
+          echo "Validating: $deb_file"
           dpkg-deb --info "$deb_file"
           dpkg-deb --contents "$deb_file" | grep -E "(usr/bin|systemd)"
-          echo "✓ DEB validation passed!"
+          echo "✓ DEB validation passed"
+
+  summary:
+    name: 📊 Summary
+    runs-on: ubuntu-latest
+    needs: [detect-changes, validate-manifests, canary-build, deb-canary]
+    if: always()
+    steps:
+      - name: Generate Summary
+        env:
+          EXPORTERS: ${{ needs.detect-changes.outputs.exporters }}
+          HAS_CHANGES: ${{ needs.detect-changes.outputs.has_changes }}
+          VALIDATE_STATUS: ${{ needs.validate-manifests.result }}
+          CANARY_STATUS: ${{ needs.canary-build.result }}
+          DEB_STATUS: ${{ needs.deb-canary.result }}
+        run: |
+          echo "## 🔨 PR Build Summary" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+
+          if [ "$HAS_CHANGES" = "true" ]; then
+            echo "**Modified exporters:**" >> $GITHUB_STEP_SUMMARY
+            echo "$EXPORTERS" | jq -r '.[] | "- " + .' >> $GITHUB_STEP_SUMMARY
+            echo "" >> $GITHUB_STEP_SUMMARY
+          else
+            echo "ℹ️  No exporters modified in this PR" >> $GITHUB_STEP_SUMMARY
+            echo "" >> $GITHUB_STEP_SUMMARY
+          fi
+
+          echo "**Validation Results:**" >> $GITHUB_STEP_SUMMARY
+          echo "- Manifest validation: $VALIDATE_STATUS" >> $GITHUB_STEP_SUMMARY
+          echo "- Canary build (RPM+Docker): $CANARY_STATUS" >> $GITHUB_STEP_SUMMARY
+          echo "- DEB build test: $DEB_STATUS" >> $GITHUB_STEP_SUMMARY

.github/workflows/full-build.yml

@@ -711,9 +711,10 @@ jobs:
           echo "::endgroup::"
 
           echo "::group::🌐 Generating portal"
-          python3 -m core.engine.site_generator \
+          python3 -m core.engine.site_generator_v2 \
             --output index.html \
             --repo-dir . \
+            --catalog-dir catalog \
             --release-urls-dir release-urls
           echo "::endgroup::"
 
@@ -723,7 +724,6 @@ jobs:
 
           # Copy generated files
           cp index.html gh-pages-dist/
-          cp catalog.json gh-pages-dist/ 2>/dev/null || true
           cp -r catalog gh-pages-dist/ 2>/dev/null || true
 
           # Copy security stats if exists
@@ -734,7 +734,7 @@ jobs:
           cd gh-pages-dist
           git config user.name "Monitoring Hub Bot"
           git config user.email "bot@monitoring-hub.local"
-          git add index.html catalog.json catalog/ security-stats.json 2>/dev/null || git add index.html catalog.json catalog/
+          git add index.html catalog/ security-stats.json 2>/dev/null || git add index.html catalog/
 
           if git diff --staged --quiet; then
             echo "No changes to commit"
@@ -757,7 +757,4 @@ jobs:
           if [ -f "catalog/index.json" ]; then
             count=$(jq '.exporters | length' catalog/index.json)
             echo "**Total exporters:** $count" >> $GITHUB_STEP_SUMMARY
-          elif [ -f "catalog.json" ]; then
-            count=$(jq '.exporters | length' catalog.json)
-            echo "**Total exporters:** $count" >> $GITHUB_STEP_SUMMARY
           fi