Skip to content

Commit cb9e568

Browse files
SckyzOSckyzO
committed
feat: V3 Granular Catalog Architecture - Atomic Writes & Race Condition Prevention
Complete V3 catalog architecture refactoring with atomic writes, race condition prevention, and security hardening. ## Key Changes ### Architecture V3 - Granular catalog with atomic per-artifact JSON files - Lightweight catalog/index.json (10KB vs 400KB) - Two-phase metadata generation (artifact → aggregation) - Race condition prevention via atomic writes - Legacy catalog.json removed ### Security Fixes (3 HIGH vulnerabilities) - Path traversal protection on --exporter parameter - Path traversal protection on --output parameter - URL sanitization with proper urlparse validation ### Quality Improvements - Architecture validation in builder - Improved error messages with hints - Tarfile extraction with filter="data" (Python 3.12+) - Comprehensive test suite for V3 schemas ### Workflow Optimizations - Unified publish-metadata with if: always() - State-based incremental builds - Simplified full-build workflow - Documentation updates ## Test Results - All security scans passed (CodeQL, Bandit, Trivy) - 0 open security alerts - All CI checks successful Closes #27, #18, #1, #2, #23
1 parent 106b762 commit cb9e568

27 files changed

+4796
-439
lines changed

‎.github/workflows/auto-release.yml‎

Lines changed: 30 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -34,27 +34,34 @@ jobs:
3434
matrix: ${{ steps.detect.outputs.matrix }}
3535
steps:
3636
- uses: actions/checkout@v6
37+
38+
- name: Set up Python
39+
uses: actions/setup-python@v6
3740
with:
38-
fetch-depth: 2 # Need at least 2 commits to diff
41+
python-version: '3.12'
42+
cache: 'pip'
43+
cache-dependency-path: 'requirements/base.txt'
44+
45+
- name: Install dependencies
46+
run: pip install -r requirements/base.txt
3947

4048
- name: 🔍 Detect Changed Exporters
4149
id: detect
4250
env:
4351
INPUT_EXPORTERS: ${{ inputs.exporters }}
52+
CATALOG_URL: https://sckyzo.github.io/monitoring-hub/catalog/index.json
4453
run: |
45-
echo "::group::🔍 Detecting modified exporters"
54+
echo "::group::🔍 Detecting exporters to build"
4655
4756
# Check if workflow was manually triggered
4857
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
4958
echo "Manual trigger detected"
50-
echo "Input length: ${#INPUT_EXPORTERS}"
51-
echo "Input value: '$INPUT_EXPORTERS'"
5259
53-
# Check if specific exporters were provided (non-empty input)
54-
if [ "${#INPUT_EXPORTERS}" -gt 0 ]; then
60+
# Check if specific exporters were provided
61+
if [ -n "$INPUT_EXPORTERS" ]; then
5562
echo "Building specific exporters: $INPUT_EXPORTERS"
5663
57-
# Convert comma-separated list to JSON array (trim spaces and filter empty)
64+
# Convert comma-separated list to JSON array
5865
EXPORTERS_JSON=$(echo "$INPUT_EXPORTERS" | jq -R -c \
5966
'split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))')
6067
@@ -66,58 +73,36 @@ jobs:
6673
echo "::endgroup::"
6774
exit 0
6875
else
69-
# No specific exporters - build ALL
70-
echo "Building ALL exporters"
71-
72-
# List all exporters (directories in exporters/)
73-
ALL_EXPORTERS=$(ls -1 exporters/ | jq -R -s -c 'split("\n") | map(select(length > 0))')
74-
75-
echo "exporters=$ALL_EXPORTERS" >> $GITHUB_OUTPUT
76-
echo "matrix={\"exporter\":$ALL_EXPORTERS}" >> $GITHUB_OUTPUT
77-
78-
COUNT=$(echo "$ALL_EXPORTERS" | jq '. | length')
79-
echo "✓ Building ALL exporters (total: $COUNT)"
80-
echo "::endgroup::"
81-
exit 0
76+
# No specific exporters - force rebuild ALL
77+
echo "Force rebuild ALL exporters"
78+
export FORCE_REBUILD=true
8279
fi
8380
fi
8481
85-
# Auto-detect changes from git diff (normal push workflow)
86-
CHANGED_FILES=$(git diff --name-only HEAD~1 HEAD | grep '^exporters/' || true)
82+
# Use state_manager for smart detection
83+
echo "Using state_manager for change detection..."
84+
python3 core/engine/state_manager.py
8785
88-
if [ -z "$CHANGED_FILES" ]; then
89-
echo "No changes detected in exporters/"
90-
echo "exporters=[]" >> $GITHUB_OUTPUT
86+
# Read outputs from state_manager
87+
EXPORTERS_JSON=$(grep "^exporters=" $GITHUB_OUTPUT | cut -d= -f2)
88+
BUILD_NEEDED=$(grep "^build_needed=" $GITHUB_OUTPUT | cut -d= -f2)
89+
90+
if [ "$BUILD_NEEDED" = "false" ]; then
91+
echo "ℹ️ No exporters need building (all up to date)"
9192
echo "matrix={\"exporter\":[]}" >> $GITHUB_OUTPUT
9293
echo "::endgroup::"
9394
exit 0
9495
fi
9596
96-
# Extract unique exporter names from changed paths
97-
CHANGED_EXPORTERS=$(echo "$CHANGED_FILES" | cut -d'/' -f2 | sort -u)
98-
99-
# Convert to JSON array
100-
EXPORTERS_JSON=$(echo "$CHANGED_EXPORTERS" | jq -R -s -c 'split("\n") | map(select(length > 0))')
101-
102-
echo "Changed exporters: $EXPORTERS_JSON"
103-
104-
# Count exporters
105-
COUNT=$(echo "$EXPORTERS_JSON" | jq '. | length')
106-
echo "Found $COUNT modified exporter(s)"
107-
108-
# Set outputs
109-
echo "exporters=$EXPORTERS_JSON" >> $GITHUB_OUTPUT
97+
# Generate matrix
11098
echo "matrix={\"exporter\":$EXPORTERS_JSON}" >> $GITHUB_OUTPUT
11199
112-
# Display summary
113-
if [ "$COUNT" -gt 0 ]; then
114-
echo "✓ Will trigger release for: $(echo "$EXPORTERS_JSON" | jq -r 'join(", ")')"
115-
else
116-
echo "ℹ️ No exporters detected"
117-
fi
100+
COUNT=$(echo "$EXPORTERS_JSON" | jq '. | length')
101+
echo "✓ Detected $COUNT exporter(s) to build: $(echo "$EXPORTERS_JSON" | jq -r 'join(", ")')"
118102
119103
echo "::endgroup::"
120104
105+
121106
- name: 📊 Generate Summary
122107
if: steps.detect.outputs.exporters != '[]'
123108
env:

‎.github/workflows/build-pr.yml‎

Lines changed: 139 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,52 @@ concurrency:
1313
cancel-in-progress: true
1414

1515
jobs:
16-
generate-artifacts:
16+
detect-changes:
17+
name: 🔍 Detect Modified Exporters
1718
runs-on: ubuntu-latest
18-
timeout-minutes: 30
19+
timeout-minutes: 5
20+
outputs:
21+
exporters: ${{ steps.detect.outputs.exporters }}
22+
has_changes: ${{ steps.detect.outputs.has_changes }}
23+
steps:
24+
- uses: actions/checkout@v6
25+
with:
26+
fetch-depth: 0
27+
28+
- name: 🔍 Detect Changed Exporters
29+
id: detect
30+
run: |
31+
echo "::group::🔍 Detecting modified exporters in PR"
32+
33+
# Get changed files in this PR
34+
git fetch origin ${{ github.base_ref }}
35+
CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD | grep '^exporters/' || true)
36+
37+
if [ -z "$CHANGED_FILES" ]; then
38+
echo "ℹ️ No changes in exporters/"
39+
echo "exporters=[]" >> $GITHUB_OUTPUT
40+
echo "has_changes=false" >> $GITHUB_OUTPUT
41+
echo "::endgroup::"
42+
exit 0
43+
fi
44+
45+
# Extract unique exporter names
46+
EXPORTERS=$(echo "$CHANGED_FILES" | cut -d'/' -f2 | sort -u | jq -R -s -c 'split("\n") | map(select(length > 0))')
47+
48+
echo "exporters=$EXPORTERS" >> $GITHUB_OUTPUT
49+
echo "has_changes=true" >> $GITHUB_OUTPUT
50+
51+
COUNT=$(echo "$EXPORTERS" | jq '. | length')
52+
echo "✓ Found $COUNT modified exporter(s): $(echo "$EXPORTERS" | jq -r 'join(", ")')"
53+
54+
echo "::endgroup::"
55+
56+
validate-manifests:
57+
name: ✅ Validate Manifests
58+
runs-on: ubuntu-latest
59+
timeout-minutes: 10
60+
needs: detect-changes
61+
if: needs.detect-changes.outputs.has_changes == 'true'
1962
steps:
2063
- uses: actions/checkout@v6
2164

@@ -29,31 +72,63 @@ jobs:
2972
- name: Install dependencies
3073
run: pip install -r requirements/base.txt
3174

32-
- name: Build Exporters
75+
- name: Validate Modified Exporters
76+
env:
77+
EXPORTERS: ${{ needs.detect-changes.outputs.exporters }}
3378
run: |
79+
echo "::group::✅ Validating manifests"
3480
export PYTHONPATH=$GITHUB_WORKSPACE
35-
for exporter in exporters/*/; do
36-
if [ -f "${exporter}manifest.yaml" ]; then
37-
name=$(basename "$exporter")
38-
echo "Building $name..."
39-
python3 -m core.engine.builder --manifest "${exporter}manifest.yaml" --output-dir "build/$name"
81+
82+
echo "$EXPORTERS" | jq -r '.[]' | while read exporter; do
83+
manifest="exporters/$exporter/manifest.yaml"
84+
85+
if [ ! -f "$manifest" ]; then
86+
echo "⚠️ Skipping $exporter (no manifest)"
87+
continue
4088
fi
89+
90+
echo "Validating $exporter..."
91+
python3 -c "
92+
from core.engine.schema import ExporterManifestSchema
93+
import yaml
94+
95+
with open('$manifest') as f:
96+
data = yaml.safe_load(f)
97+
98+
schema = ExporterManifestSchema()
99+
errors = schema.validate(data)
100+
101+
if errors:
102+
print('❌ Validation failed for $exporter:')
103+
for field, msgs in errors.items():
104+
for msg in msgs if isinstance(msgs, list) else [msgs]:
105+
print(f' - {field}: {msg}')
106+
exit(1)
107+
else:
108+
print('✓ Valid')
109+
"
41110
done
111+
112+
echo "::endgroup::"
113+
114+
- name: Validate URLs
42115
env:
43-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
116+
EXPORTERS: ${{ needs.detect-changes.outputs.exporters }}
117+
run: |
118+
echo "::group::🔗 Validating download URLs"
44119
45-
- name: Upload Artifacts
46-
uses: actions/upload-artifact@v6
47-
with:
48-
name: generated-build-files
49-
path: build/
50-
retention-days: 7
120+
echo "$EXPORTERS" | jq -r '.[]' | while read exporter; do
121+
echo "Checking $exporter..."
122+
python3 core/scripts/validate_urls.py "$exporter" || echo "⚠️ URL validation failed for $exporter (non-blocking)"
123+
done
51124
52-
# Ceinture et Bretelles : Test d'intégration complet sur un exporter témoin
53-
integration-test:
125+
echo "::endgroup::"
126+
127+
canary-build:
128+
name: 🧪 Canary Build (node_exporter)
54129
runs-on: ubuntu-latest
55130
timeout-minutes: 20
56-
needs: generate-artifacts
131+
needs: detect-changes
57132
steps:
58133
- uses: actions/checkout@v6
59134

@@ -67,21 +142,23 @@ jobs:
67142
- name: Install dependencies
68143
run: pip install -r requirements/base.txt
69144

70-
- name: Full Pipeline Test (Node Exporter)
145+
- name: Test Full Pipeline
146+
env:
147+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
71148
run: |
72149
chmod +x core/scripts/*.sh
73-
# On utilise node_exporter comme "Canary" pour valider la chaîne
150+
echo "Testing full pipeline with node_exporter as canary..."
74151
./core/scripts/local_test.sh node_exporter --validate
75152
76-
# Test DEB Build and Installation
77-
deb-canary-test:
153+
deb-canary:
154+
name: 🧪 DEB Build Test (${{ matrix.dist }})
78155
runs-on: ubuntu-latest
79156
timeout-minutes: 20
80-
needs: generate-artifacts
157+
needs: detect-changes
81158
strategy:
159+
fail-fast: false
82160
matrix:
83161
dist: [ubuntu-22.04, debian-12]
84-
arch: [amd64]
85162
steps:
86163
- uses: actions/checkout@v6
87164

@@ -99,22 +176,52 @@ jobs:
99176
env:
100177
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
101178
DIST: ${{ matrix.dist }}
102-
ARCH: ${{ matrix.arch }}
103179
run: |
104180
chmod +x core/scripts/*.sh
105-
echo "Testing DEB build for node_exporter on $DIST ($ARCH)..."
106-
./core/scripts/test_deb_build.sh node_exporter "$DIST" "$ARCH"
181+
echo "Testing DEB build for node_exporter on $DIST..."
182+
./core/scripts/test_deb_build.sh node_exporter "$DIST" amd64
107183
108-
- name: Validate DEB Package
184+
- name: Validate Package
109185
run: |
110-
deb_file=$(find build -name "*.deb" | grep node-exporter | head -1)
186+
deb_file=$(find build -name "*.deb" | head -1)
111187
112188
if [ -z "$deb_file" ]; then
113-
echo "ERROR: No DEB file found"
189+
echo "❌ No DEB file found"
114190
exit 1
115191
fi
116192
117-
echo "Validating DEB: $deb_file"
193+
echo "Validating: $deb_file"
118194
dpkg-deb --info "$deb_file"
119195
dpkg-deb --contents "$deb_file" | grep -E "(usr/bin|systemd)"
120-
echo "✓ DEB validation passed!"
196+
echo "✓ DEB validation passed"
197+
198+
summary:
199+
name: 📊 Summary
200+
runs-on: ubuntu-latest
201+
needs: [detect-changes, validate-manifests, canary-build, deb-canary]
202+
if: always()
203+
steps:
204+
- name: Generate Summary
205+
env:
206+
EXPORTERS: ${{ needs.detect-changes.outputs.exporters }}
207+
HAS_CHANGES: ${{ needs.detect-changes.outputs.has_changes }}
208+
VALIDATE_STATUS: ${{ needs.validate-manifests.result }}
209+
CANARY_STATUS: ${{ needs.canary-build.result }}
210+
DEB_STATUS: ${{ needs.deb-canary.result }}
211+
run: |
212+
echo "## 🔨 PR Build Summary" >> $GITHUB_STEP_SUMMARY
213+
echo "" >> $GITHUB_STEP_SUMMARY
214+
215+
if [ "$HAS_CHANGES" = "true" ]; then
216+
echo "**Modified exporters:**" >> $GITHUB_STEP_SUMMARY
217+
echo "$EXPORTERS" | jq -r '.[] | "- " + .' >> $GITHUB_STEP_SUMMARY
218+
echo "" >> $GITHUB_STEP_SUMMARY
219+
else
220+
echo "ℹ️ No exporters modified in this PR" >> $GITHUB_STEP_SUMMARY
221+
echo "" >> $GITHUB_STEP_SUMMARY
222+
fi
223+
224+
echo "**Validation Results:**" >> $GITHUB_STEP_SUMMARY
225+
echo "- Manifest validation: $VALIDATE_STATUS" >> $GITHUB_STEP_SUMMARY
226+
echo "- Canary build (RPM+Docker): $CANARY_STATUS" >> $GITHUB_STEP_SUMMARY
227+
echo "- DEB build test: $DEB_STATUS" >> $GITHUB_STEP_SUMMARY

‎.github/workflows/full-build.yml‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -711,9 +711,10 @@ jobs:
711711
echo "::endgroup::"
712712
713713
echo "::group::🌐 Generating portal"
714-
python3 -m core.engine.site_generator \
714+
python3 -m core.engine.site_generator_v2 \
715715
--output index.html \
716716
--repo-dir . \
717+
--catalog-dir catalog \
717718
--release-urls-dir release-urls
718719
echo "::endgroup::"
719720
@@ -723,7 +724,6 @@ jobs:
723724
724725
# Copy generated files
725726
cp index.html gh-pages-dist/
726-
cp catalog.json gh-pages-dist/ 2>/dev/null || true
727727
cp -r catalog gh-pages-dist/ 2>/dev/null || true
728728
729729
# Copy security stats if exists
@@ -734,7 +734,7 @@ jobs:
734734
cd gh-pages-dist
735735
git config user.name "Monitoring Hub Bot"
736736
git config user.email "bot@monitoring-hub.local"
737-
git add index.html catalog.json catalog/ security-stats.json 2>/dev/null || git add index.html catalog.json catalog/
737+
git add index.html catalog/ security-stats.json 2>/dev/null || git add index.html catalog/
738738
739739
if git diff --staged --quiet; then
740740
echo "No changes to commit"
@@ -757,7 +757,4 @@ jobs:
757757
if [ -f "catalog/index.json" ]; then
758758
count=$(jq '.exporters | length' catalog/index.json)
759759
echo "**Total exporters:** $count" >> $GITHUB_STEP_SUMMARY
760-
elif [ -f "catalog.json" ]; then
761-
count=$(jq '.exporters | length' catalog.json)
762-
echo "**Total exporters:** $count" >> $GITHUB_STEP_SUMMARY
763760
fi

0 commit comments

Comments
 (0)