Merge pull request #62 from DXC-DP-Monitoring/staging

Merge Green Branch (#61)

Author: fadwael1

.github/workflows/cd.yml

@@ -5,6 +5,10 @@ on:
     branches: [ main ]
   workflow_dispatch: {}
 
+permissions:
+  contents: read
+  packages: write
+
 env:
   VERSION: v1.${{ github.run_number }}
 
@@ -14,12 +18,24 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Build Docker (prod)
-        run: docker build -t ghcr.io/${{ github.repository }}:${{ env.VERSION }} .
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
       - name: Login to GHCR
-        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
-      - name: Push image
-        run: |
-          docker push ghcr.io/${{ github.repository }}:${{ env.VERSION }}
-          docker tag ghcr.io/${{ github.repository }}:${{ env.VERSION }} ghcr.io/${{ github.repository }}:latest
-          docker push ghcr.io/${{ github.repository }}:latest
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push (cached)
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}:${{ env.VERSION }}
+            ghcr.io/${{ github.repository }}:latest
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/ci.yml

@@ -14,18 +14,30 @@ on:
       - 'images/**'
       - 'docs/**'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 env:
   NIFIPULSE_AUTO_ENV: "1"
 
 jobs:
   build-test-dev:
-    if: github.ref == 'refs/heads/dev'
+    if: github.ref == 'refs/heads/dev' && !(
+         github.event_name == 'push' && contains(github.event.head_commit.message, '[skip ci]') ||
+         github.event_name == 'pull_request' && (contains(github.event.pull_request.title, '[skip ci]') || contains(github.event.pull_request.body, '[skip ci]'))
+        )
     runs-on: ubuntu-latest
+    timeout-minutes: 15
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
           python-version: '3.11'
+          cache: 'pip'
+          cache-dependency-path: |
+            **/pyproject.toml
+            **/requirements*.txt
       - name: Install deps
         run: |
           pip install -U pip
@@ -45,7 +57,10 @@ jobs:
           fi
 
   integration-test-staging:
-    if: github.event.pull_request.base.ref == 'staging'
+    if: github.event.pull_request.base.ref == 'staging' && !(
+         github.event_name == 'push' && contains(github.event.head_commit.message, '[skip ci]') ||
+         github.event_name == 'pull_request' && (contains(github.event.pull_request.title, '[skip ci]') || contains(github.event.pull_request.body, '[skip ci]'))
+        )
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4

.github/workflows/tests.yml

@@ -16,6 +16,10 @@ on:
 
 jobs:
   unit-integration:
+    if: !(
+         github.event_name == 'push' && contains(github.event.head_commit.message, '[skip ci]') ||
+         github.event_name == 'pull_request' && (contains(github.event.pull_request.title, '[skip ci]') || contains(github.event.pull_request.body, '[skip ci]'))
+        )
     runs-on: ubuntu-latest
     services:
       postgres:

Readme.md

@@ -48,6 +48,28 @@ After running `docker compose up` run:
     LIMIT 20;"
 ---
 
+## CI usage and “[skip ci]”
+Our CI runs on every push to `dev` and on PRs to `staging`/`main`. To reduce waste:
+- CI auto-skips for docs and images via `paths-ignore`.
+- Optional: add `[skip ci]` in the commit message (push), or in the PR title/body (pull_request) for one-off skips.
+
+Use `[skip ci]` only for:
+- Docs-only edits (README, Alerting.md, state_of_art PDFs).
+- Image asset updates under `images/`.
+- Non-code text changes (typos, formatting).
+
+Do NOT use `[skip ci]` when changing:
+- Python code (`nifipulse/**`), tests (`tests/**`), dependencies (`pyproject.toml`, `requirements*.txt`).
+- CI/CD files (`.github/workflows/**`), Dockerfile/compose, SQL schema, Prometheus/Grafana configs.
+
+Examples:
+- `git commit -m "docs: update Alerting.md [skip ci]" && git push`
+- PR title: `docs: update dashboards [skip ci]`
+
+CI details:
+- Concurrency cancels superseded runs to save compute.
+- Pip caching speeds installs; lint is advisory (`continue-on-error: true`) until we enforce style.
+
 ## Architecture
 ![nifi_architecture](images/nifi_archi.png) 
 

nifipulse/cli.py

@@ -14,8 +14,8 @@ def main():
     parser.add_argument(
         "--interval",
         type=int,
-        default=5,
-        help="Seconds between polling cycles (default 5)."
+        default=60,
+        help="Seconds between polling cycles (default 60)."
     )
     args = parser.parse_args()
     nifipulse(poll_count=args.poll, interval=args.interval)

nifipulse/extract_metrics.py

@@ -9,7 +9,7 @@
 from nifipulse.data_normalisation import process_data
 from nifipulse.load_postgres import load_postgres
 
-def nifipulse(poll_count=10, interval=5):
+def nifipulse(poll_count=10, interval=60):
     """
     poll_count: number of cycles to poll (0 = run forever)
     interval: seconds to wait between cycles