Implement CI ci.yml tests.yml update config.py load_postgres.py pyproject.toml

Author: aMAAmina

.github/workflows/ci.yml

@@ -0,0 +1,91 @@
+name: CI Pipeline
+
+on:
+  push:
+    branches:
+      - dev
+  pull_request:
+    branches:
+      - staging
+      - main
+
+env:
+  NIFIPULSE_AUTO_ENV: "1"
+  PGUSER: postgres
+  PGPASSWORD: postgres
+  PGHOST: localhost
+  PGPORT: "5432"
+  PGDATABASE: metrics_db
+
+jobs:
+  build-test-dev:
+    if: github.ref == 'refs/heads/dev'
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      # Install dependencies (change this depending on your stack)
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          pip install -U pip
+          pip install -e .
+          pip install flake8
+
+      - name: Lint
+        run: |
+          pip install flake8
+          flake8 .
+
+      - name: Docker build (dev)
+        run: |
+          docker build -t nifipulse-dev .
+
+  integration-test-staging:
+    if: github.event.pull_request.base.ref == 'staging'
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Docker build
+        run: docker build -t nifipulse-staging .
+
+      - name: Docker compose up
+        run: docker compose -f docker-compose.yml up -d
+
+      - name: Wait for services
+        run: sleep 15
+
+      - name: Integration tests
+        run: |
+          pip install -r tests/requirements.txt
+          pytest tests/integration
+
+  release-main:
+    if: github.event.pull_request.base.ref == 'main'
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Docker build (prod)
+        run: docker build -t nifipulse-prod .
+
+      - name: Tag release
+        run: |
+          VERSION="v1.${{ github.run_number }}"
+          echo "Tagging version: $VERSION"
+          git tag $VERSION
+          git push origin $VERSION
+
+      - name: Push to container registry
+        run: |
+          echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin
+          docker tag nifipulse-prod myregistry/nifipulse:$VERSION
+          docker push myregistry/nifipulse:$VERSION

.github/workflows/tests.yml

@@ -0,0 +1,74 @@
+name: Tests
+
+on:
+  push:
+    branches: [ dev ]
+  pull_request:
+    branches: [ dev, staging, main ]
+
+jobs:
+  unit-integration:
+    runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres:15-alpine
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: metrics_db
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd="pg_isready -U postgres -d metrics_db"
+          --health-interval=5s
+          --health-timeout=5s
+          --health-retries=12
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install project and test deps
+        run: |
+          pip install -U pip
+          pip install -e .
+          pip install pytest "sqlalchemy>=2" psycopg2-binary pandas requests
+
+      - name: Wait for Postgres
+        run: |
+          python - <<'PY'
+          import time, psycopg2
+          for i in range(30):
+              try:
+                  psycopg2.connect(host="localhost", port=5432, user="postgres", password="postgres", dbname="metrics_db").close()
+                  print("Postgres ready"); break
+              except Exception as e:
+                  print("Waiting...", e); time.sleep(2)
+          else:
+              raise SystemExit("Postgres not ready")
+          PY
+
+      - name: Initialize schema (SQLAlchemy)
+        run: |
+          python - <<'PY'
+          from sqlalchemy import create_engine, text
+          ddl = """
+          CREATE TABLE IF NOT EXISTS dim_instance (instance_id SERIAL PRIMARY KEY, instance_name TEXT UNIQUE NOT NULL);
+          CREATE TABLE IF NOT EXISTS dim_metric (metric_id SERIAL PRIMARY KEY, metric_name TEXT UNIQUE NOT NULL, original_unit TEXT);
+          CREATE TABLE IF NOT EXISTS dim_component (component_id SERIAL PRIMARY KEY, component_name TEXT NOT NULL, component_type TEXT NOT NULL, UNIQUE (component_name, component_type));
+          CREATE TABLE IF NOT EXISTS dim_date (date_id SERIAL PRIMARY KEY, timestamp_utc TIMESTAMPTZ UNIQUE NOT NULL, year INT, month INT, day INT, hour INT, minute INT, second INT);
+          CREATE TABLE IF NOT EXISTS fact_metrics (fact_id SERIAL PRIMARY KEY, date_id INT REFERENCES dim_date(date_id), instance_id INT REFERENCES dim_instance(instance_id), metric_id INT REFERENCES dim_metric(metric_id), component_id INT REFERENCES dim_component(component_id), value DOUBLE PRECISION, UNIQUE (date_id, instance_id, metric_id, component_id));
+          """
+          eng = create_engine("postgresql+psycopg2://postgres:postgres@localhost:5432/metrics_db")
+          with eng.begin() as c:
+              for stmt in [s.strip() for s in ddl.split(';') if s.strip()]:
+                  c.execute(text(stmt))
+          print("Schema ready")
+          PY
+
+      - name: Run tests
+        run: pytest -q

nifipulse/config.py

@@ -1,9 +1,21 @@
+import os
+from urllib.parse import quote_plus
+
 class BaseConfig:
     # Shared defaults
     RESULTS_DIR = "results"
     PROM_URL = "http://localhost:9090/api/v1/query"
     CSV_SINK = "results/prometheus_metrics_log.csv"
     CLEAN_DATA = "results/nifi_metrics_propre.csv" 
+    # Build DSN from env (override PGHOST to 'postgres' when running inside Docker)
+    PW = quote_plus(os.getenv('PGPASSWORD', 'postgres'))
+    PG_DSN = os.getenv(
+        "PG_DSN",
+        "postgresql+psycopg2://"
+        f"{os.getenv('PGUSER','postgres')}:{PW}@"
+        f"{os.getenv('PGHOST','localhost')}:{os.getenv('PGPORT','5432')}/"
+        f"{os.getenv('PGDATABASE','metrics_db')}"
+    )
 class DevConfig(BaseConfig):
     pass
 
@@ -26,4 +38,22 @@ def set_env(env_name):
     try:
         env = ENV_MAP[env_name]
     except KeyError:
-        raise ValueError(f"Unknown environment: {env_name}")
+        raise ValueError(f"Unknown environment: {env_name}")
+
+def set_env_from_branch(branch_name: str):
+    """
+    Map Git branch to environment: main->prod, staging->staged, dev->dev
+    """
+    mapping = {"main": "prod", "staging": "staged", "dev": "dev"}
+    set_env(mapping.get(branch_name, "prod"))
+
+def auto_set_env():
+    name = os.getenv("NIFIPULSE_ENV")
+    if name:
+        set_env(name)
+        return
+    branch = os.getenv("GITHUB_REF_NAME") or (os.getenv("GITHUB_REF","").split("/")[-1] or "main")
+    set_env_from_branch(branch)
+
+if os.getenv("NIFIPULSE_AUTO_ENV") == "1":
+    auto_set_env()

nifipulse/load_postgres.py

@@ -1,17 +1,18 @@
 import pandas as pd
 from sqlalchemy import create_engine, text
+from nifipulse import config
 
 def load_postgres(clean_data):
     print(" inserting into PostgreSQL ... ")
     df = pd.read_csv(clean_data, parse_dates=['timestamp_utc'])
     df['timestamp_utc'] = pd.to_datetime(df['timestamp_utc'], utc=True)
 
     #  Connexion PostgreSQL
-
+    # Use DSN from config (works on host and in Docker; override via env)
     engine = create_engine(
-        "postgresql+psycopg2://postgres:postgres@localhost:5432/metrics_db"
+        config.env.PG_DSN,
+        connect_args={"options": "-c client_encoding=UTF8"}
     )
-    
 
     print("- Connexion PostgreSQL OK")
 

pyproject.toml

@@ -23,4 +23,14 @@ build-backend = "setuptools.build_meta"
 include = ["nifipulse*"]
 
 [tool.setuptools.package-data]
-nifipulse = ["metrics_list.txt"]
+nifipulse = ["metrics_list.txt"]
+
+[project.optional-dependencies]
+dev = [
+  "pytest",
+  "flake8",
+  "black",
+  "mypy",
+  "types-requests",
+  "python-dotenv"
+]