Merge pull request #996 from splunk/data_source_improvement

Data source improvement

Author: P4T12ICK

.github/validate_dataset_ymls.py

@@ -0,0 +1,123 @@
+name: Validate Attack Data
+
+on:
+  pull_request:
+    branches: [ master, main ]
+    types: [opened, synchronize, reopened]
+    paths:
+      - 'datasets/**/*.yml'
+      - 'datasets/**/*.yaml'
+      - 'bin/validate.py'
+      - 'bin/dataset_schema.json'
+      - 'bin/requirements.txt'
+  push:
+    branches: [ master, main ]
+    paths:
+      - 'datasets/**/*.yml'
+      - 'datasets/**/*.yaml'
+      - 'bin/validate.py'
+      - 'bin/dataset_schema.json'
+      - 'bin/requirements.txt'
+
+permissions:
+  contents: read
+  issues: write
+  pull-requests: write
+
+jobs:
+  validate-attack-data:
+    runs-on: ubuntu-latest
+    
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+      with:
+        lfs: true
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.9'
+        cache: 'pip'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r bin/requirements.txt
+
+    # Validate all YAML files
+    - name: Validate all YAML files
+      run: |
+        python bin/validate.py
+      env:
+        PYTHONPATH: ${{ github.workspace }}/bin
+
+    # PR-specific success/failure handling
+    - name: Comment PR on validation failure
+      if: failure() && github.event_name == 'pull_request'
+      uses: actions/github-script@v7
+      with:
+        script: |
+          const { owner, repo, number } = context.issue;
+          
+          const body = `❌ **Attack Data Validation Failed**
+          
+          The YAML files in this PR do not pass validation. Please check the workflow logs for detailed error messages and fix the issues before merging.
+          
+          [View workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})`;
+          
+          await github.rest.issues.createComment({
+            owner,
+            repo,
+            issue_number: number,
+            body: body
+          });
+
+    - name: Comment PR on validation success
+      if: success() && github.event_name == 'pull_request'
+      uses: actions/github-script@v7
+      with:
+        script: |
+          const { owner, repo, number } = context.issue;
+          
+          const body = `✅ **Attack Data Validation Passed**
+          
+          All YAML files in this PR have been successfully validated against the schema.
+          
+          Ready for review and merge! 🚀`;
+          
+          await github.rest.issues.createComment({
+            owner,
+            repo,
+            issue_number: number,
+            body: body
+          });
+
+    # Push-specific failure handling (create issue)
+    - name: Create issue on validation failure (Push)
+      if: failure() && github.event_name == 'push'
+      uses: actions/github-script@v7
+      with:
+        script: |
+          const title = `🚨 Attack Data Validation Failed - ${new Date().toISOString().split('T')[0]}`;
+          const body = `**Validation failed on push to ${context.ref}**
+          
+          Commit: ${context.sha}
+          
+          The YAML files in the datasets directory do not pass validation. This indicates that invalid data has been merged into the main branch.
+          
+          **Action Required:**
+          1. Review the [failed workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
+          2. Fix the validation errors
+          3. Create a hotfix PR to resolve the issues
+          `;
+          
+          await github.rest.issues.create({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            title: title,
+            body: body,
+            labels: ['bug', 'validation-failure', 'high-priority']
+          });
+
+